From 0446c45d8e48d0389c19dcc161578caf3922e3a0 Mon Sep 17 00:00:00 2001 From: wehub-resource-sync Date: Mon, 13 Jul 2026 12:12:13 +0800 Subject: [PATCH] chore: import upstream snapshot with attribution --- .env.txt | 4 + .gitattributes | 12 + .../DISCUSSION_TEMPLATE/feature-requests.yml | 59 + .github/FUNDING.yml | 7 + .github/ISSUE_TEMPLATE/bug_report.yml | 127 + .github/ISSUE_TEMPLATE/config.yml | 8 + .github/pull_request_template.md | 19 + .github/workflows/docker-release.yml | 100 + .github/workflows/docs/ARCHITECTURE.md | 917 ++ .github/workflows/docs/README.md | 1029 ++ .github/workflows/docs/WORKFLOW_REFERENCE.md | 287 + .github/workflows/main.yml | 46 + .github/workflows/release.yml | 113 + .github/workflows/release.yml.backup | 142 + .github/workflows/security.yml | 65 + .github/workflows/test-release.yml.disabled | 116 + .gitignore | 313 + CHANGELOG.md | 1777 +++ CODE_OF_CONDUCT.md | 131 + CONTRIBUTING.md | 102 + CONTRIBUTORS.md | 93 + Dockerfile | 229 + JOURNAL.md | 339 + LICENSE | 69 + MANIFEST.in | 2 + MISSION.md | 46 + PROGRESSIVE_CRAWLING.md | 320 + README-first.md | 809 ++ README.md | 1274 ++ README.wehub.md | 7 + ROADMAP.md | 503 + SECURITY-CREDITS.md | 20 + SECURITY.md | 134 + SPONSORS.md | 65 + cliff.toml | 24 + crawl4ai/__init__.py | 236 + crawl4ai/__version__.py | 8 + crawl4ai/adaptive_crawler.py | 1923 +++ crawl4ai/antibot_detector.py | 281 + crawl4ai/async_configs.py | 2527 ++++ crawl4ai/async_crawler_strategy.py | 2830 ++++ crawl4ai/async_database.py | 678 + crawl4ai/async_dispatcher.py | 772 + crawl4ai/async_logger.py | 386 + crawl4ai/async_url_seeder.py | 1794 +++ crawl4ai/async_webcrawler.py | 1249 ++ crawl4ai/browser_adapter.py | 413 + crawl4ai/browser_manager.py | 2096 +++ crawl4ai/browser_profiler.py | 1403 ++ crawl4ai/cache_context.py | 117 + crawl4ai/cache_validator.py | 270 + crawl4ai/chunking_strategy.py | 255 + crawl4ai/cli.py | 1654 +++ crawl4ai/cloud/__init__.py | 16 + crawl4ai/cloud/cli.py | 473 + crawl4ai/components/crawler_monitor.py | 869 ++ crawl4ai/config.py | 155 + crawl4ai/content_filter_strategy.py | 1110 ++ crawl4ai/content_scraping_strategy.py | 1014 ++ crawl4ai/crawlers/__init__.py | 0 crawl4ai/crawlers/amazon_product/__init__.py | 0 crawl4ai/crawlers/amazon_product/crawler.py | 20 + crawl4ai/crawlers/google_search/__init__.py | 0 crawl4ai/crawlers/google_search/crawler.py | 131 + crawl4ai/crawlers/google_search/script.js | 115 + crawl4ai/deep_crawling/__init__.py | 47 + crawl4ai/deep_crawling/base_strategy.py | 159 + crawl4ai/deep_crawling/bff_strategy.py | 429 + crawl4ai/deep_crawling/bfs_strategy.py | 420 + crawl4ai/deep_crawling/crazy.py | 432 + crawl4ai/deep_crawling/dfs_strategy.py | 331 + crawl4ai/deep_crawling/filters.py | 691 + crawl4ai/deep_crawling/scorers.py | 519 + crawl4ai/docker_client.py | 219 + crawl4ai/domain_mapper.py | 1132 ++ crawl4ai/extraction_strategy.py | 2827 ++++ crawl4ai/html2text/__init__.py | 1215 ++ crawl4ai/html2text/__main__.py | 3 + crawl4ai/html2text/_typing.py | 3 + crawl4ai/html2text/cli.py | 330 + crawl4ai/html2text/config.py | 172 + crawl4ai/html2text/elements.py | 18 + crawl4ai/html2text/utils.py | 304 + crawl4ai/hub.py | 69 + crawl4ai/install.py | 212 + crawl4ai/js_snippet/__init__.py | 18 + crawl4ai/js_snippet/flatten_shadow_dom.js | 104 + crawl4ai/js_snippet/navigator_overrider.js | 25 + crawl4ai/js_snippet/remove_consent_popups.js | 710 + .../js_snippet/remove_overlay_elements.js | 120 + .../js_snippet/update_image_dimensions.js | 54 + crawl4ai/legacy/__init__.py | 0 crawl4ai/legacy/cli.py | 123 + crawl4ai/legacy/crawler_strategy.py | 394 + crawl4ai/legacy/database.py | 180 + crawl4ai/legacy/docs_manager.py | 75 + crawl4ai/legacy/llmtxt.py | 546 + crawl4ai/legacy/version_manager.py | 29 + crawl4ai/legacy/web_crawler.py | 294 + crawl4ai/link_preview.py | 411 + crawl4ai/markdown_generation_strategy.py | 260 + crawl4ai/migrations.py | 194 + crawl4ai/model_loader.py | 296 + crawl4ai/models.py | 407 + crawl4ai/processors/pdf/__init__.py | 196 + crawl4ai/processors/pdf/processor.py | 487 + crawl4ai/processors/pdf/utils.py | 350 + crawl4ai/prompts.py | 1693 +++ crawl4ai/proxy_strategy.py | 341 + crawl4ai/script/__init__.py | 35 + crawl4ai/script/c4a_compile.py | 398 + crawl4ai/script/c4a_result.py | 219 + crawl4ai/script/c4ai_script.py | 690 + crawl4ai/ssl_certificate.py | 204 + crawl4ai/table_extraction.py | 1402 ++ crawl4ai/types.py | 195 + crawl4ai/user_agent_generator.py | 428 + crawl4ai/utils.py | 3740 +++++ deploy/docker/.dockerignore | 31 + deploy/docker/.llm.env.example | 32 + deploy/docker/ARCHITECTURE.md | 1149 ++ deploy/docker/MIGRATION.md | 179 + deploy/docker/README.md | 1033 ++ deploy/docker/SECURITY-VERIFY.md | 153 + deploy/docker/STRESS_TEST_PIPELINE.md | 241 + deploy/docker/WEBHOOK_EXAMPLES.md | 378 + deploy/docker/api.py | 1026 ++ deploy/docker/artifacts.py | 166 + deploy/docker/auth.py | 146 + deploy/docker/auth_gate.py | 126 + deploy/docker/c4ai-code-context.md | 11632 ++++++++++++++++ deploy/docker/c4ai-doc-context.md | 8913 ++++++++++++ deploy/docker/config.yml | 133 + deploy/docker/crawler_pool.py | 220 + deploy/docker/egress_broker.py | 209 + deploy/docker/egress_proxy.py | 212 + deploy/docker/entrypoint.sh | 37 + deploy/docker/governor.py | 111 + deploy/docker/hook_registry.py | 244 + deploy/docker/job.py | 150 + deploy/docker/llm_broker.py | 64 + deploy/docker/mcp_bridge.py | 273 + deploy/docker/monitor.py | 391 + deploy/docker/monitor_routes.py | 409 + deploy/docker/redis_config.py | 75 + deploy/docker/requirements.txt | 16 + deploy/docker/schemas.py | 130 + deploy/docker/server.py | 1106 ++ deploy/docker/static/assets/crawl4ai-logo.jpg | Bin 0 -> 5920 bytes deploy/docker/static/assets/crawl4ai-logo.png | Bin 0 -> 1622 bytes deploy/docker/static/assets/logo.png | Bin 0 -> 11243 bytes deploy/docker/static/monitor/index.html | 1129 ++ deploy/docker/static/playground/index.html | 1036 ++ deploy/docker/supervisord.conf | 34 + deploy/docker/test-websocket.py | 34 + deploy/docker/tests/conftest.py | 168 + deploy/docker/tests/demo_monitor_dashboard.py | 164 + deploy/docker/tests/requirements.txt | 2 + deploy/docker/tests/run_security_tests.py | 196 + deploy/docker/tests/test_1_basic.py | 138 + deploy/docker/tests/test_2_memory.py | 205 + deploy/docker/tests/test_3_pool.py | 229 + deploy/docker/tests/test_4_concurrent.py | 236 + deploy/docker/tests/test_5_pool_stress.py | 267 + deploy/docker/tests/test_6_multi_endpoint.py | 234 + deploy/docker/tests/test_7_cleanup.py | 199 + deploy/docker/tests/test_monitor_demo.py | 57 + deploy/docker/tests/test_security_2026_04.py | 312 + .../docker/tests/test_security_2026_04_b2.py | 247 + .../tests/test_security_artifact_store.py | 108 + deploy/docker/tests/test_security_authz.py | 116 + .../tests/test_security_container_posture.py | 124 + .../tests/test_security_default_posture.py | 254 + .../tests/test_security_download_traversal.py | 100 + .../tests/test_security_egress_proxy.py | 132 + deploy/docker/tests/test_security_fixes.py | 274 + .../docker/tests/test_security_headers_xss.py | 124 + .../docker/tests/test_security_llm_broker.py | 78 + .../tests/test_security_resource_caps.py | 175 + .../docker/tests/test_security_ssrf_crawl.py | 314 + .../docker/tests/test_security_ssrf_egress.py | 142 + .../tests/test_security_trust_boundary.py | 236 + .../tests/test_security_webhook_pinning.py | 82 + deploy/docker/utils.py | 434 + deploy/docker/webhook.py | 248 + deploy/docker/work_queue.py | 110 + docker-compose.yml | 68 + docs/RELEASE_NOTES_v0.8.0.md | 243 + docs/apps/iseeyou/llms-full.txt | 7715 ++++++++++ ...wl4ai_Linkedin_Data_Discovery_Part_1.ipynb | 1323 ++ ...wl4ai_Linkedin_Data_Discovery_Part_2.ipynb | 5859 ++++++++ docs/apps/linkdin/README.md | 131 + docs/apps/linkdin/c4ai_discover.py | 446 + docs/apps/linkdin/c4ai_insights.py | 381 + docs/apps/linkdin/samples/companies.jsonl | 9 + docs/apps/linkdin/samples/people.jsonl | 108 + docs/apps/linkdin/schemas/company_card.json | 51 + docs/apps/linkdin/schemas/people_card.json | 41 + docs/apps/linkdin/snippets/company.html | 138 + docs/apps/linkdin/snippets/people.html | 93 + docs/apps/linkdin/templates/ai.js | 50 + .../templates/graph_view_template.html | 1168 ++ docs/assets/pitch-dark.png | Bin 0 -> 33656 bytes docs/assets/pitch-dark.svg | 64 + docs/assets/powered-by-dark.svg | 25 + docs/assets/powered-by-disco.svg | 64 + docs/assets/powered-by-light.svg | 21 + docs/assets/powered-by-night.svg | 28 + docs/assets/sponsors/aleph_null.svg | 25 + docs/assets/sponsors/aleph_null_light.svg | 25 + docs/assets/sponsors/massive.svg | 10 + docs/assets/sponsors/massive_light.svg | 10 + docs/assets/sponsors/nst-light.svg | 1 + docs/assets/sponsors/thor_data.svg | 18 + docs/assets/sponsors/thor_data_light.svg | 18 + docs/blog/release-v0.7.0.md | 343 + docs/blog/release-v0.7.1.md | 43 + docs/blog/release-v0.7.3.md | 350 + docs/blog/release-v0.7.4.md | 270 + docs/blog/release-v0.7.5.md | 318 + docs/blog/release-v0.7.6.md | 314 + docs/blog/release-v0.7.7.md | 626 + docs/blog/release-v0.7.8.md | 327 + docs/blog/release-v0.8.0.md | 243 + docs/blog/release-v0.8.5.md | 384 + docs/blog/release-v0.8.7.md | 113 + docs/blog/release-v0.8.8.md | 43 + docs/blog/release-v0.8.9.md | 37 + docs/blog/release-v0.9.0.md | 78 + docs/blog/release-v0.9.1.md | 90 + docs/codebase/browser.md | 51 + docs/codebase/cli.md | 40 + docs/deprecated/docker-deployment.md | 189 + docs/examples/README_BUILTIN_BROWSER.md | 123 + docs/examples/adaptive_crawling/README.md | 85 + .../advanced_configuration.py | 207 + .../examples/adaptive_crawling/basic_usage.py | 76 + .../adaptive_crawling/custom_strategies.py | 373 + .../embedding_configuration.py | 206 + .../adaptive_crawling/embedding_strategy.py | 109 + .../embedding_vs_statistical.py | 167 + .../adaptive_crawling/export_import_kb.py | 232 + .../adaptive_crawling/llm_config_example.py | 154 + .../amazon_product_extraction_direct_url.py | 110 + .../amazon_product_extraction_using_hooks.py | 150 + ...product_extraction_using_use_javascript.py | 126 + docs/examples/arun_vs_arun_many.py | 79 + docs/examples/assets/audio.mp3 | Bin 0 -> 519360 bytes docs/examples/assets/basic.png | Bin 0 -> 381423 bytes docs/examples/assets/cosine_extraction.png | Bin 0 -> 412569 bytes docs/examples/assets/css_js.png | Bin 0 -> 549639 bytes docs/examples/assets/css_selector.png | Bin 0 -> 383806 bytes docs/examples/assets/exec_script.png | Bin 0 -> 480629 bytes .../examples/assets/instagram_grid_result.png | Bin 0 -> 6890454 bytes docs/examples/assets/llm_extraction.png | Bin 0 -> 488016 bytes .../assets/semantic_extraction_cosine.png | Bin 0 -> 429401 bytes .../assets/semantic_extraction_llm.png | Bin 0 -> 497058 bytes .../assets/virtual_scroll_append_only.html | 132 + .../assets/virtual_scroll_instagram_grid.html | 158 + .../assets/virtual_scroll_news_feed.html | 210 + .../assets/virtual_scroll_twitter_like.html | 122 + .../async_webcrawler_multiple_urls_example.py | 55 + docs/examples/browser_optimization_example.py | 126 + docs/examples/builtin_browser_example.py | 86 + .../c4a_script/amazon_example/README.md | 171 + .../amazon_example/amazon_r2d2_search.py | 202 + .../amazon_example/extracted_products.json | 114 + .../generated_product_schema.json | 47 + .../amazon_example/generated_search_script.js | 9 + .../c4a_script/amazon_example/header.html | 214 + .../c4a_script/amazon_example/product.html | 206 + .../examples/c4a_script/api_usage_examples.py | 217 + .../c4a_script/c4a_script_hello_world.py | 53 + .../c4a_script_hello_world_error.py | 53 + docs/examples/c4a_script/demo_c4a_crawl4ai.py | 285 + .../c4a_script/generate_script_hello_world.py | 89 + .../github_search/extracted_repositories.json | 111 + .../generated_result_schema.json | 66 + .../github_search/generated_search_script.js | 39 + .../github_search/github_search_crawler.py | 211 + .../c4a_script/github_search/result.html | 54 + .../c4a_script/github_search/search_form.html | 336 + .../c4a_script/script_samples/add_to_cart.c4a | 7 + .../script_samples/advanced_control_flow.c4a | 43 + .../script_samples/conditional_login.c4a | 8 + .../script_samples/data_extraction.c4a | 56 + .../script_samples/fill_contact.c4a | 8 + .../script_samples/load_more_content.c4a | 7 + .../c4a_script/script_samples/login_flow.c4a | 36 + .../script_samples/multi_step_workflow.c4a | 106 + .../script_samples/navigate_tabs.c4a | 8 + .../c4a_script/script_samples/quick_login.c4a | 8 + .../script_samples/responsive_actions.c4a | 7 + .../script_samples/scroll_and_click.c4a | 8 + .../script_samples/search_product.c4a | 7 + .../c4a_script/script_samples/simple_form.c4a | 19 + .../script_samples/smart_form_fill.c4a | 11 + docs/examples/c4a_script/tutorial/README.md | 396 + .../tutorial/assets/DankMono-Bold.woff2 | Bin 0 -> 33480 bytes .../tutorial/assets/DankMono-Italic.woff2 | Bin 0 -> 32468 bytes .../tutorial/assets/DankMono-Regular.woff2 | Bin 0 -> 30528 bytes .../c4a_script/tutorial/assets/app.css | 906 ++ .../c4a_script/tutorial/assets/app.js | 1485 ++ .../tutorial/assets/blockly-manager.js | 591 + .../tutorial/assets/blockly-theme.css | 238 + .../c4a_script/tutorial/assets/c4a-blocks.js | 549 + .../tutorial/assets/c4a-generator.js | 261 + .../c4a_script/tutorial/assets/styles.css | 531 + .../c4a_script/tutorial/blockly-demo.c4a | 21 + docs/examples/c4a_script/tutorial/index.html | 205 + .../c4a_script/tutorial/playground/app.js | 604 + .../c4a_script/tutorial/playground/index.html | 328 + .../c4a_script/tutorial/playground/styles.css | 627 + .../c4a_script/tutorial/requirements.txt | 2 + docs/examples/c4a_script/tutorial/server.py | 304 + .../c4a_script/tutorial/test_blockly.html | 69 + .../solve_aws_waf.py | 62 + .../solve_cloudflare_challenge.py | 60 + .../solve_cloudflare_turnstile.py | 64 + .../solve_recaptcha_v2.py | 67 + .../solve_recaptcha_v3.py | 75 + .../solve_aws_waf.py | 36 + .../solve_cloudflare_challenge.py | 36 + .../solve_cloudflare_turnstile.py | 36 + .../solve_recaptcha_v2.py | 36 + .../solve_recaptcha_v3.py | 36 + docs/examples/chainlit.md | 3 + docs/examples/cli/browser.yml | 13 + docs/examples/cli/crawler.yml | 13 + docs/examples/cli/css_schema.json | 27 + docs/examples/cli/extract.yml | 11 + docs/examples/cli/extract_css.yml | 3 + docs/examples/cli/llm_schema.json | 26 + .../cloud_browser/scrapeless_browser.py | 61 + docs/examples/crawlai_vs_firecrawl.py | 70 + docs/examples/crawler_monitor_example.py | 209 + docs/examples/crypto_analysis_example.py | 445 + docs/examples/deep_crawl_cancellation.py | 427 + docs/examples/deep_crawl_crash_recovery.py | 297 + docs/examples/deepcrawl_example.py | 498 + docs/examples/demo_multi_config_clean.py | 303 + docs/examples/dfs_crawl_demo.py | 39 + docs/examples/dispatcher_example.py | 136 + docs/examples/docker/demo_docker_api.py | 1317 ++ docs/examples/docker/demo_docker_polling.py | 149 + docs/examples/docker_client_hooks_example.py | 522 + docs/examples/docker_config_obj.py | 249 + docs/examples/docker_example.py | 407 + docs/examples/docker_hooks_examples.py | 627 + docs/examples/docker_python_rest_api.py | 214 + docs/examples/docker_python_sdk.py | 35 + docs/examples/docker_webhook_example.py | 461 + .../domain_mapper/domain_mapper_demo.py | 82 + .../extraction_strategies_examples.py | 125 + .../full_page_screenshot_and_pdf_export.md | 108 + docs/examples/hello_world.py | 29 + docs/examples/hello_world_undetected.py | 57 + docs/examples/hooks_example.py | 118 + docs/examples/identity_based_browsing.py | 108 + docs/examples/language_support_example.py | 50 + docs/examples/link_head_extraction_example.py | 376 + .../examples/llm_extraction_openai_pricing.py | 55 + docs/examples/llm_markdown_generator.py | 87 + docs/examples/llm_table_extraction_example.py | 356 + .../markdown/content_source_example.py | 64 + .../markdown/content_source_short_example.py | 42 + .../network_console_capture_example.py | 477 + docs/examples/nst_proxy/api_proxy_example.py | 48 + docs/examples/nst_proxy/auth_proxy_example.py | 31 + .../examples/nst_proxy/basic_proxy_example.py | 29 + docs/examples/nst_proxy/nstproxy_example.py | 39 + docs/examples/prefetch_two_phase_crawl.py | 279 + docs/examples/proxy_rotation_demo.py | 161 + docs/examples/quickstart.ipynb | 664 + docs/examples/quickstart.py | 562 + docs/examples/quickstart_examples_set_1.py | 412 + docs/examples/quickstart_examples_set_2.py | 562 + docs/examples/regex_extraction_quickstart.py | 143 + docs/examples/research_assistant.py | 172 + docs/examples/rest_call.py | 54 + docs/examples/sample_ecommerce.html | 106 + .../scraping_strategies_performance.py | 136 + docs/examples/serp_api_project_11_feb.py | 308 + docs/examples/session_id_example.py | 38 + docs/examples/shadow_dom_crawling.py | 77 + docs/examples/simple_anti_bot_examples.py | 59 + docs/examples/ssl_example.py | 51 + docs/examples/stealth_mode_example.py | 522 + docs/examples/stealth_mode_quick_start.py | 215 + docs/examples/stealth_test_simple.py | 62 + docs/examples/storage_state_tutorial.md | 225 + docs/examples/summarize_page.py | 53 + docs/examples/table_extraction_example.py | 276 + docs/examples/tutorial_dynamic_clicks.md | 117 + docs/examples/tutorial_v0.5.py | 460 + .../undetectability/undetected_basic_test.py | 74 + .../undetectability/undetected_bot_test.py | 155 + .../undetected_cloudflare_test.py | 164 + .../undetected_vs_regular_comparison.py | 184 + docs/examples/undetected_simple_demo.py | 118 + .../Crawl4AI_URL_Seeder_Tutorial.ipynb | 1171 ++ .../bbc_sport_research_assistant.py | 807 ++ .../url_seeder/convert_tutorial_to_colab.py | 155 + .../url_seeder/tutorial_url_seeder.md | 1035 ++ docs/examples/url_seeder/url_seeder_demo.py | 263 + .../url_seeder/url_seeder_quick_demo.py | 128 + docs/examples/use_geo_location.py | 70 + docs/examples/virtual_scroll_example.py | 367 + docs/examples/website-to-api/.gitignore | 221 + docs/examples/website-to-api/README.md | 252 + docs/examples/website-to-api/api_server.py | 363 + docs/examples/website-to-api/app.py | 49 + .../website-to-api/assets/crawl4ai_logo.jpg | Bin 0 -> 5920 bytes docs/examples/website-to-api/requirements.txt | 5 + .../examples/website-to-api/static/index.html | 201 + docs/examples/website-to-api/static/script.js | 401 + .../examples/website-to-api/static/styles.css | 765 + docs/examples/website-to-api/test_api.py | 28 + docs/examples/website-to-api/test_models.py | 67 + .../website-to-api/web_scraper_lib.py | 397 + docs/md_v2/CONTRIBUTING.md | 102 + docs/md_v2/advanced/adaptive-strategies.md | 400 + docs/md_v2/advanced/advanced-features.md | 446 + docs/md_v2/advanced/anti-bot-and-fallback.md | 263 + docs/md_v2/advanced/crawl-dispatcher.md | 12 + docs/md_v2/advanced/file-downloading.md | 118 + docs/md_v2/advanced/hooks-auth.md | 254 + .../md_v2/advanced/identity-based-crawling.md | 413 + docs/md_v2/advanced/lazy-loading.md | 104 + docs/md_v2/advanced/multi-url-crawling.md | 604 + .../md_v2/advanced/network-console-capture.md | 205 + docs/md_v2/advanced/pdf-parsing.md | 201 + docs/md_v2/advanced/proxy-security.md | 308 + docs/md_v2/advanced/session-management.md | 267 + docs/md_v2/advanced/ssl-certificate.md | 179 + docs/md_v2/advanced/undetected-browser.md | 395 + docs/md_v2/advanced/virtual-scroll.md | 309 + docs/md_v2/api/adaptive-crawler.md | 244 + docs/md_v2/api/arun.md | 309 + docs/md_v2/api/arun_many.md | 192 + docs/md_v2/api/async-webcrawler.md | 311 + docs/md_v2/api/c4a-script-reference.md | 992 ++ docs/md_v2/api/crawl-result.md | 431 + docs/md_v2/api/digest.md | 181 + docs/md_v2/api/parameters.md | 534 + docs/md_v2/api/strategies.md | 401 + docs/md_v2/apps/assets/DankMono-Bold.woff2 | Bin 0 -> 33480 bytes docs/md_v2/apps/assets/DankMono-Italic.woff2 | Bin 0 -> 32468 bytes docs/md_v2/apps/assets/DankMono-Regular.woff2 | Bin 0 -> 30528 bytes docs/md_v2/apps/c4a-script/README.md | 396 + .../c4a-script/assets/DankMono-Bold.woff2 | Bin 0 -> 33480 bytes .../c4a-script/assets/DankMono-Italic.woff2 | Bin 0 -> 32468 bytes .../c4a-script/assets/DankMono-Regular.woff2 | Bin 0 -> 30528 bytes docs/md_v2/apps/c4a-script/assets/app.css | 906 ++ docs/md_v2/apps/c4a-script/assets/app.js | 1485 ++ .../apps/c4a-script/assets/blockly-manager.js | 591 + .../apps/c4a-script/assets/blockly-theme.css | 238 + .../apps/c4a-script/assets/c4a-blocks.js | 549 + .../apps/c4a-script/assets/c4a-generator.js | 261 + docs/md_v2/apps/c4a-script/assets/styles.css | 531 + docs/md_v2/apps/c4a-script/blockly-demo.c4a | 21 + docs/md_v2/apps/c4a-script/index.html | 205 + docs/md_v2/apps/c4a-script/playground/app.js | 604 + .../apps/c4a-script/playground/index.html | 328 + .../apps/c4a-script/playground/styles.css | 627 + docs/md_v2/apps/c4a-script/requirements.txt | 2 + docs/md_v2/apps/c4a-script/server.py | 304 + docs/md_v2/apps/c4a-script/test_blockly.html | 69 + docs/md_v2/apps/crawl4ai-assistant/README.md | 124 + .../assets/DankMono-Bold.woff2 | Bin 0 -> 33480 bytes .../assets/DankMono-Italic.woff2 | Bin 0 -> 32468 bytes .../assets/DankMono-Regular.woff2 | Bin 0 -> 30528 bytes .../apps/crawl4ai-assistant/assistant.css | 1551 +++ .../background/service-worker.js | 39 + .../crawl4ai-assistant/content/click2crawl.js | 1968 +++ .../crawl4ai-assistant/content/content.js | 78 + .../content/contentAnalyzer.js | 623 + .../content/markdownConverter.js | 718 + .../content/markdownExtraction.js | 701 + .../content/markdownPreviewModal.js | 300 + .../crawl4ai-assistant/content/overlay.css | 2842 ++++ .../content/scriptBuilder.js | 2515 ++++ .../content/shared/utils.js | 253 + .../crawl4ai-assistant-v1.2.1.zip | Bin 0 -> 138534 bytes .../crawl4ai-assistant-v1.3.0.zip | Bin 0 -> 213054 bytes .../apps/crawl4ai-assistant/icons/favicon.ico | Bin 0 -> 3449 bytes .../crawl4ai-assistant/icons/icon-128.png | Bin 0 -> 1622 bytes .../apps/crawl4ai-assistant/icons/icon-16.png | Bin 0 -> 1622 bytes .../apps/crawl4ai-assistant/icons/icon-48.png | Bin 0 -> 1622 bytes docs/md_v2/apps/crawl4ai-assistant/index.html | 974 ++ .../crawl4ai-assistant/libs/marked.min.js | 69 + .../apps/crawl4ai-assistant/manifest.json | 54 + .../popup/icons/favicon.ico | Bin 0 -> 3449 bytes .../popup/icons/icon-128.png | Bin 0 -> 1622 bytes .../popup/icons/icon-16.png | Bin 0 -> 1622 bytes .../popup/icons/icon-48.png | Bin 0 -> 1622 bytes .../apps/crawl4ai-assistant/popup/popup.css | 330 + .../apps/crawl4ai-assistant/popup/popup.html | 111 + .../apps/crawl4ai-assistant/popup/popup.js | 146 + docs/md_v2/apps/index.md | 302 + docs/md_v2/apps/llmtxt/build.md | 75 + docs/md_v2/apps/llmtxt/index.html | 135 + docs/md_v2/apps/llmtxt/llmtxt.css | 603 + docs/md_v2/apps/llmtxt/llmtxt.js | 580 + docs/md_v2/apps/llmtxt/why.md | 37 + docs/md_v2/ask_ai/ask-ai.css | 444 + docs/md_v2/ask_ai/ask-ai.js | 607 + docs/md_v2/ask_ai/index.html | 64 + docs/md_v2/assets/DankMono-Bold.woff2 | Bin 0 -> 33480 bytes docs/md_v2/assets/DankMono-Italic.woff2 | Bin 0 -> 32468 bytes docs/md_v2/assets/DankMono-Regular.woff2 | Bin 0 -> 30528 bytes docs/md_v2/assets/Monaco.woff | Bin 0 -> 20096 bytes docs/md_v2/assets/copy_code.js | 62 + docs/md_v2/assets/crawl4ai-skill.zip | Bin 0 -> 78441 bytes docs/md_v2/assets/dmvendor.css | 127 + docs/md_v2/assets/docs.zip | Bin 0 -> 64674 bytes docs/md_v2/assets/feedback-overrides.css | 37 + docs/md_v2/assets/floating_ask_ai_button.js | 39 + docs/md_v2/assets/github_stats.js | 119 + docs/md_v2/assets/gtag.js | 5 + docs/md_v2/assets/highlight.css | 0 docs/md_v2/assets/highlight.min.js | 1213 ++ docs/md_v2/assets/highlight_init.js | 6 + docs/md_v2/assets/images/dispatcher.png | Bin 0 -> 487245 bytes docs/md_v2/assets/images/logo.png | Bin 0 -> 1622 bytes docs/md_v2/assets/layout.css | 576 + docs/md_v2/assets/llm.txt/diagrams/cli.txt | 425 + .../llm.txt/diagrams/config_objects.txt | 1421 ++ .../deep_crawl_advanced_filters_scorers.txt | 401 + .../assets/llm.txt/diagrams/deep_crawling.txt | 428 + docs/md_v2/assets/llm.txt/diagrams/docker.txt | 603 + .../llm.txt/diagrams/extraction-llm.txt | 478 + .../llm.txt/diagrams/extraction-no-llm.txt | 478 + .../diagrams/http_based_crawler_strategy.txt | 472 + .../assets/llm.txt/diagrams/installation.txt | 368 + .../assets/llm.txt/diagrams/llms-diagram.txt | 5912 ++++++++ .../llm.txt/diagrams/multi_urls_crawling.txt | 392 + .../llm.txt/diagrams/simple_crawling.txt | 411 + .../assets/llm.txt/diagrams/url_seeder.txt | 441 + docs/md_v2/assets/llm.txt/txt/cli.txt | 295 + .../assets/llm.txt/txt/config_objects.txt | 1171 ++ .../deep_crawl_advanced_filters_scorers.txt | 446 + .../assets/llm.txt/txt/deep_crawling.txt | 348 + docs/md_v2/assets/llm.txt/txt/docker.txt | 826 ++ .../assets/llm.txt/txt/extraction-llm.txt | 903 ++ .../assets/llm.txt/txt/extraction-no-llm.txt | 835 ++ .../txt/http_based_crawler_strategy.txt | 388 + .../md_v2/assets/llm.txt/txt/installation.txt | 231 + .../assets/llm.txt/txt/llms-full-v0.1.1.txt | 7715 ++++++++++ docs/md_v2/assets/llm.txt/txt/llms-full.txt | 7715 ++++++++++ .../llm.txt/txt/multi_urls_crawling.txt | 339 + .../assets/llm.txt/txt/simple_crawling.txt | 365 + docs/md_v2/assets/llm.txt/txt/url_seeder.txt | 655 + docs/md_v2/assets/mobile_menu.js | 106 + docs/md_v2/assets/page_actions.css | 376 + docs/md_v2/assets/page_actions.js | 427 + docs/md_v2/assets/selection_ask_ai.js | 186 + docs/md_v2/assets/styles.css | 262 + docs/md_v2/assets/test/toc.js | 144 + docs/md_v2/assets/toc.js | 144 + docs/md_v2/basic/installation.md | 137 + .../articles/adaptive-crawling-revolution.md | 369 + docs/md_v2/blog/articles/dockerize_hooks.md | 46 + .../blog/articles/llm-context-revolution.md | 217 + .../articles/virtual-scroll-revolution.md | 355 + docs/md_v2/blog/index.md | 90 + docs/md_v2/blog/index.md.bak | 144 + docs/md_v2/blog/releases/0.4.0.md | 62 + docs/md_v2/blog/releases/0.4.1.md | 145 + docs/md_v2/blog/releases/0.4.2.md | 86 + docs/md_v2/blog/releases/0.5.0.md | 524 + docs/md_v2/blog/releases/0.6.0.md | 143 + docs/md_v2/blog/releases/0.7.0.md | 343 + docs/md_v2/blog/releases/0.7.1.md | 43 + docs/md_v2/blog/releases/0.7.2.md | 98 + docs/md_v2/blog/releases/0.7.3.md | 170 + docs/md_v2/blog/releases/0.7.6.md | 314 + docs/md_v2/blog/releases/v0.4.3b1.md | 138 + docs/md_v2/blog/releases/v0.7.5.md | 318 + docs/md_v2/blog/releases/v0.7.7.md | 626 + docs/md_v2/blog/releases/v0.7.8.md | 327 + docs/md_v2/blog/releases/v0.8.0.md | 243 + docs/md_v2/blog/releases/v0.8.5.md | 384 + docs/md_v2/blog/releases/v0.9.1.md | 90 + docs/md_v2/branding/index.md | 1371 ++ docs/md_v2/complete-sdk-reference.md | 5273 +++++++ docs/md_v2/core/adaptive-crawling.md | 375 + docs/md_v2/core/ask-ai.md | 74 + docs/md_v2/core/browser-crawler-config.md | 487 + docs/md_v2/core/c4a-script.md | 393 + docs/md_v2/core/cache-modes.md | 69 + docs/md_v2/core/cli.md | 307 + docs/md_v2/core/content-selection.md | 550 + docs/md_v2/core/crawler-result.md | 343 + docs/md_v2/core/deep-crawling.md | 907 ++ docs/md_v2/core/domain-mapping.md | 343 + docs/md_v2/core/examples.md | 134 + docs/md_v2/core/fit-markdown.md | 245 + docs/md_v2/core/installation.md | 129 + docs/md_v2/core/link-media.md | 698 + docs/md_v2/core/llmtxt.md | 61 + docs/md_v2/core/local-files.md | 158 + docs/md_v2/core/markdown-generation.md | 504 + docs/md_v2/core/page-interaction.md | 432 + docs/md_v2/core/quickstart.md | 465 + docs/md_v2/core/self-hosting.md | 2641 ++++ docs/md_v2/core/simple-crawling.md | 152 + docs/md_v2/core/table_extraction.md | 807 ++ docs/md_v2/core/url-seeding.md | 1173 ++ docs/md_v2/extraction/chunking.md | 144 + docs/md_v2/extraction/clustring-strategies.md | 222 + docs/md_v2/extraction/llm-strategies.md | 330 + docs/md_v2/extraction/no-llm-strategies.md | 946 ++ docs/md_v2/favicon.ico | Bin 0 -> 3449 bytes docs/md_v2/img/favicon-32x32.png | Bin 0 -> 1622 bytes docs/md_v2/img/favicon-x-32x32.png | Bin 0 -> 1484 bytes docs/md_v2/img/favicon.ico | Bin 0 -> 3449 bytes docs/md_v2/index.md | 191 + docs/md_v2/marketplace/README.md | 66 + docs/md_v2/marketplace/admin/admin.css | 759 + docs/md_v2/marketplace/admin/admin.js | 933 ++ docs/md_v2/marketplace/admin/index.html | 215 + docs/md_v2/marketplace/app-detail.css | 683 + docs/md_v2/marketplace/app-detail.html | 175 + docs/md_v2/marketplace/app-detail.js | 318 + docs/md_v2/marketplace/backend/.env.example | 14 + docs/md_v2/marketplace/backend/config.py | 59 + docs/md_v2/marketplace/backend/database.py | 117 + docs/md_v2/marketplace/backend/dummy_data.py | 267 + .../marketplace/backend/requirements.txt | 5 + docs/md_v2/marketplace/backend/schema.yaml | 75 + docs/md_v2/marketplace/backend/server.py | 497 + .../marketplace/backend/uploads/.gitignore | 2 + .../md_v2/marketplace/frontend/app-detail.css | 462 + .../marketplace/frontend/app-detail.html | 239 + docs/md_v2/marketplace/frontend/app-detail.js | 335 + docs/md_v2/marketplace/frontend/index.html | 147 + .../marketplace/frontend/marketplace.css | 957 ++ .../md_v2/marketplace/frontend/marketplace.js | 395 + docs/md_v2/marketplace/index.html | 147 + docs/md_v2/marketplace/marketplace.css | 994 ++ docs/md_v2/marketplace/marketplace.js | 412 + docs/md_v2/migration/table_extraction_v073.md | 376 + .../webscraping-strategy-migration.md | 92 + docs/md_v2/overrides/main.html | 47 + docs/md_v2/privacy.md | 102 + docs/md_v2/stats.md | 524 + docs/md_v2/support.md | 63 + docs/md_v2/terms.md | 108 + docs/migration/v0.8.0-upgrade-guide.md | 301 + ...rawl4AI_v0.3.72_Release_Announcement.ipynb | 235 + .../crawl4ai_v0_7_0_showcase.py | 1584 +++ docs/releases_review/demo_v0.7.0.py | 408 + docs/releases_review/demo_v0.7.5.py | 338 + docs/releases_review/demo_v0.7.6.py | 359 + docs/releases_review/demo_v0.7.7.py | 628 + docs/releases_review/demo_v0.7.8.py | 910 ++ docs/releases_review/demo_v0.8.0.py | 633 + docs/releases_review/demo_v0.8.5.py | 913 ++ docs/releases_review/demo_v0.9.1.py | 272 + docs/releases_review/v0.3.74.overview.py | 276 + .../v0.7.5_docker_hooks_demo.py | 655 + .../v0.7.5_video_walkthrough.ipynb | 1516 ++ docs/releases_review/v0_4_24_walkthrough.py | 464 + .../releases_review/v0_4_3b2_features_demo.py | 349 + docs/releases_review/v0_7_0_features_demo.py | 280 + docs/security/GHSA-DRAFT-RCE-LFI.md | 171 + docs/snippets/deep_crawl/1.intro.py | 78 + docs/snippets/deep_crawl/2.filters.py | 162 + docs/tutorials/coming_soon.md | 0 mkdocs.yml | 129 + prompts/prompt_net_requests.md | 489 + pyproject.toml | 100 + requirements.txt | 36 + sbom/README.md | 13 + sbom/sbom.cdx.json | 1 + setup.cfg | 2 + setup.py | 65 + test_llm_webhook_feature.py | 401 + test_webhook_implementation.py | 307 + tests/WEBHOOK_TEST_README.md | 251 + tests/__init__.py | 0 tests/adaptive/compare_performance.py | 98 + tests/adaptive/test_adaptive_crawler.py | 293 + tests/adaptive/test_confidence_debug.py | 182 + tests/adaptive/test_embedding_performance.py | 254 + tests/adaptive/test_embedding_strategy.py | 634 + tests/adaptive/test_llm_embedding.py | 154 + tests/adaptive/test_query_llm_config.py | 284 + .../test_domain_mapper_adversarial.py | 136 + tests/async/sample_wikipedia.html | 2179 +++ tests/async/test_0.4.2_browser_manager.py | 160 + tests/async/test_0.4.2_config_params.py | 211 + tests/async/test_async_doanloader.py | 247 + tests/async/test_basic_crawling.py | 90 + tests/async/test_browser_lifecycle.py | 972 ++ tests/async/test_browser_memory.py | 1169 ++ tests/async/test_browser_recycle_v2.py | 386 + tests/async/test_caching.py | 87 + ...test_chunking_and_extraction_strategies.py | 86 + tests/async/test_content_extraction.py | 108 + tests/async/test_content_filter_bm25.py | 183 + tests/async/test_content_filter_prune.py | 170 + tests/async/test_content_scraper_strategy.py | 216 + tests/async/test_crawler_strategy.py | 73 + tests/async/test_database_operations.py | 90 + tests/async/test_dispatchers.py | 170 + tests/async/test_edge_cases.py | 133 + tests/async/test_error_handling.py | 78 + ...on_scraping_methods_performance.configs.py | 705 + tests/async/test_http_file_download.py | 423 + tests/async/test_markdown_genertor.py | 181 + tests/async/test_parameters_and_options.py | 116 + tests/async/test_performance.py | 79 + tests/async/test_redirect_url_resolution.py | 118 + tests/async/test_screenshot.py | 122 + .../async_assistant/test_extract_pipeline.py | 381 + .../test_extract_pipeline_v2.py | 386 + tests/browser/docker/__init__.py | 4 + tests/browser/docker/test_docker_browser.py | 651 + tests/browser/manager/demo_browser_manager.py | 525 + tests/browser/test_browser_context_id.py | 489 + tests/browser/test_browser_manager.py | 190 + tests/browser/test_browser_manager_close.py | 35 + tests/browser/test_builtin_browser.py | 809 ++ tests/browser/test_builtin_strategy.py | 160 + tests/browser/test_cdp_cleanup_reuse.py | 281 + tests/browser/test_cdp_strategy.py | 470 + tests/browser/test_combined.py | 77 + tests/browser/test_context_leak_fix.py | 358 + tests/browser/test_init_script_dedup.py | 399 + tests/browser/test_launch_standalone.py | 17 + .../browser/test_page_reuse_race_condition.py | 606 + tests/browser/test_parallel_crawling.py | 902 ++ tests/browser/test_playwright_strategy.py | 316 + tests/browser/test_profile_shrink.py | 1053 ++ tests/browser/test_profiles.py | 180 + tests/browser/test_repro_1640.py | 424 + tests/browser/test_resource_filtering.py | 178 + tests/cache_validation/__init__.py | 1 + tests/cache_validation/conftest.py | 40 + tests/cache_validation/test_end_to_end.py | 449 + .../cache_validation/test_head_fingerprint.py | 97 + tests/cache_validation/test_real_domains.py | 354 + tests/check_dependencies.py | 344 + tests/cli/test_cli.py | 257 + tests/deep_crawling/__init__.py | 0 .../test_deep_crawl_cancellation.py | 597 + .../test_deep_crawl_contextvar.py | 340 + tests/deep_crawling/test_deep_crawl_resume.py | 839 ++ .../test_deep_crawl_resume_integration.py | 162 + tests/deep_crwaling/test_filter.py | 75 + tests/docker/simple_api_test.py | 365 + tests/docker/test_config_object.py | 113 + tests/docker/test_docker.py | 186 + tests/docker/test_dockerclient.py | 34 + tests/docker/test_filter_deep_crawl.py | 245 + tests/docker/test_hooks_client.py | 372 + tests/docker/test_hooks_comprehensive.py | 558 + tests/docker/test_hooks_utility.py | 193 + tests/docker/test_llm_params.py | 349 + tests/docker/test_pool_release.py | 155 + tests/docker/test_rest_api_deep_crawl.py | 596 + tests/docker/test_serialization.py | 255 + tests/docker/test_server.py | 146 + tests/docker/test_server_requests.py | 890 ++ tests/docker/test_server_token.py | 212 + tests/docker_example.py | 397 + tests/general/generate_dummy_site.py | 335 + ...t_acyn_crawl_wuth_http_crawler_strategy.py | 56 + tests/general/test_advanced_deep_crawl.py | 46 + tests/general/test_async_crawler_strategy.py | 382 + .../general/test_async_markdown_generator.py | 171 + tests/general/test_async_url_seeder_bm25.py | 711 + tests/general/test_async_webcrawler.py | 228 + tests/general/test_bff_scoring.py | 117 + tests/general/test_cache_context.py | 85 + .../general/test_content_source_parameter.py | 106 + tests/general/test_crawlers.py | 17 + tests/general/test_deep_crawl.py | 46 + tests/general/test_deep_crawl_filters.py | 279 + tests/general/test_deep_crawl_scorers.py | 179 + tests/general/test_download_file.py | 34 + tests/general/test_flatten_shadow_dom.py | 84 + tests/general/test_generate_schema_usage.py | 654 + tests/general/test_http_crawler_strategy.py | 116 + tests/general/test_llm_filter.py | 86 + tests/general/test_max_scroll.py | 115 + tests/general/test_mhtml.py | 213 + tests/general/test_network_console_capture.py | 185 + tests/general/test_persistent_context.py | 43 + tests/general/test_robot_parser.py | 159 + tests/general/test_schema_builder.py | 112 + tests/general/test_stream.py | 50 + tests/general/test_stream_dispatch.py | 39 + tests/general/test_strip_markdown_fences.py | 321 + tests/general/test_url_pattern.py | 85 + .../test_url_seeder_for_only_sitemap.py | 32 + tests/general/tets_robot.py | 62 + tests/hub/test_simple.py | 34 + tests/integration/test_domain_mapper_e2e.py | 138 + tests/loggers/test_logger.py | 80 + tests/mcp/test_mcp_socket.py | 119 + tests/mcp/test_mcp_sse.py | 11 + tests/memory/README.md | 315 + tests/memory/benchmark_report.py | 887 ++ tests/memory/cap_test.py | 34 + tests/memory/requirements.txt | 4 + tests/memory/run_benchmark.py | 259 + tests/memory/test_crawler_monitor.py | 168 + tests/memory/test_dispatcher_stress.py | 410 + tests/memory/test_docker_config_gen.py | 36 + tests/memory/test_stress_api.py | 520 + tests/memory/test_stress_api_xs.py | 203 + tests/memory/test_stress_docker_api.py | 129 + tests/memory/test_stress_sdk.py | 500 + tests/profiler/test_create_profile.py | 32 + tests/profiler/test_keyboard_handle.py | 55 + tests/proxy/test_antibot_detector.py | 334 + tests/proxy/test_chanel_cdp_proxy.py | 112 + tests/proxy/test_persistent_proxy.py | 68 + tests/proxy/test_proxy_config.py | 582 + tests/proxy/test_proxy_deprecation.py | 42 + tests/proxy/test_proxy_regression.py | 96 + tests/proxy/test_proxy_verify.py | 109 + tests/proxy/test_sticky_sessions.py | 569 + tests/regression/__init__.py | 1 + tests/regression/conftest.py | 628 + tests/regression/test_reg_browser.py | 561 + tests/regression/test_reg_config.py | 776 ++ tests/regression/test_reg_content.py | 512 + tests/regression/test_reg_core_crawl.py | 405 + tests/regression/test_reg_deep_crawl.py | 633 + tests/regression/test_reg_domain_mapper.py | 270 + tests/regression/test_reg_edge_cases.py | 359 + tests/regression/test_reg_extraction.py | 608 + tests/regression/test_reg_utils.py | 500 + tests/releases/test_release_0.6.4.py | 151 + tests/releases/test_release_0.7.0.py | 317 + tests/test_arun_many.py | 42 + tests/test_async_logger_stderr.py | 156 + tests/test_bug_batch_1622_1786_1796.py | 314 + tests/test_cdp_changes.py | 506 + tests/test_cli_docs.py | 44 + tests/test_cloud_bugs_batch.py | 479 + tests/test_config_defaults.py | 263 + tests/test_config_matching_only.py | 131 + tests/test_config_selection.py | 87 + tests/test_docker.py | 299 + tests/test_docker_api_with_llm_provider.py | 122 + tests/test_eval_security_adversarial.py | 838 ++ tests/test_http_timeout_unit_1894.py | 78 + tests/test_issue_1043_mermaid_svg.py | 229 + tests/test_issue_1213_bm25_dedup.py | 178 + tests/test_issue_1370_1818_1762_1509.py | 512 + tests/test_issue_1484_css_selector.py | 167 + tests/test_issue_1594_mcp_sse.py | 78 + tests/test_issue_1611_llm_provider.py | 114 + ...test_issue_1748_screenshot_scroll_delay.py | 261 + ...st_issue_1750_screenshot_scan_full_page.py | 249 + tests/test_issue_1837_config_list.py | 140 + tests/test_issue_1842_browser_none.py | 181 + tests/test_issue_1848_logger_serialize.py | 222 + tests/test_issue_1850_mcp_sse.py | 172 + tests/test_link_extractor.py | 262 + ...test_llm_extraction_parallel_issue_1055.py | 220 + tests/test_llm_simple_url.py | 170 + tests/test_llmtxt.py | 52 + tests/test_main.py | 276 + ...test_markdown_generator_validation_1880.py | 158 + tests/test_memory_macos.py | 71 + tests/test_merge_head_data_scoring.py | 183 + tests/test_multi_config.py | 117 + tests/test_normalize_url.py | 91 + tests/test_pr_1290_1668.py | 167 + tests/test_pr_1435_redirected_status_code.py | 72 + tests/test_pr_1463_device_scale_factor.py | 62 + tests/test_pr_1795_1798_1734.py | 222 + tests/test_prefetch_integration.py | 236 + tests/test_prefetch_mode.py | 275 + tests/test_prefetch_regression.py | 232 + .../test_preserve_https_for_internal_links.py | 175 + tests/test_pruning_preserve_whitelist_1900.py | 258 + tests/test_pyopenssl_security_fix.py | 168 + tests/test_pyopenssl_update.py | 184 + tests/test_raw_html_browser.py | 172 + tests/test_raw_html_edge_cases.py | 563 + tests/test_raw_html_redirected_url.py | 299 + tests/test_scraping_strategy.py | 26 + tests/test_source_sibling_selector.py | 396 + tests/test_table_gfm_compliance.py | 247 + tests/test_type_annotations.py | 194 + tests/test_virtual_scroll.py | 197 + tests/test_web_crawler.py | 149 + tests/unit/test_domain_mapper_unit.py | 406 + tests/unit/test_resource_filtering_config.py | 100 + tests/unit/test_sitemap_namespace_parsing.py | 134 + uv.lock | 3049 ++++ 898 files changed, 328024 insertions(+) create mode 100644 .env.txt create mode 100644 .gitattributes create mode 100644 .github/DISCUSSION_TEMPLATE/feature-requests.yml create mode 100644 .github/FUNDING.yml create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/docker-release.yml create mode 100644 .github/workflows/docs/ARCHITECTURE.md create mode 100644 .github/workflows/docs/README.md create mode 100644 .github/workflows/docs/WORKFLOW_REFERENCE.md create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/release.yml.backup create mode 100644 .github/workflows/security.yml create mode 100644 .github/workflows/test-release.yml.disabled create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 CONTRIBUTORS.md create mode 100644 Dockerfile create mode 100644 JOURNAL.md create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 MISSION.md create mode 100644 PROGRESSIVE_CRAWLING.md create mode 100644 README-first.md create mode 100644 README.md create mode 100644 README.wehub.md create mode 100644 ROADMAP.md create mode 100644 SECURITY-CREDITS.md create mode 100644 SECURITY.md create mode 100644 SPONSORS.md create mode 100644 cliff.toml create mode 100644 crawl4ai/__init__.py create mode 100644 crawl4ai/__version__.py create mode 100644 crawl4ai/adaptive_crawler.py create mode 100644 crawl4ai/antibot_detector.py create mode 100644 crawl4ai/async_configs.py create mode 100644 crawl4ai/async_crawler_strategy.py create mode 100644 crawl4ai/async_database.py create mode 100644 crawl4ai/async_dispatcher.py create mode 100644 crawl4ai/async_logger.py create mode 100644 crawl4ai/async_url_seeder.py create mode 100644 crawl4ai/async_webcrawler.py create mode 100644 crawl4ai/browser_adapter.py create mode 100644 crawl4ai/browser_manager.py create mode 100644 crawl4ai/browser_profiler.py create mode 100644 crawl4ai/cache_context.py create mode 100644 crawl4ai/cache_validator.py create mode 100644 crawl4ai/chunking_strategy.py create mode 100644 crawl4ai/cli.py create mode 100644 crawl4ai/cloud/__init__.py create mode 100644 crawl4ai/cloud/cli.py create mode 100644 crawl4ai/components/crawler_monitor.py create mode 100644 crawl4ai/config.py create mode 100644 crawl4ai/content_filter_strategy.py create mode 100644 crawl4ai/content_scraping_strategy.py create mode 100644 crawl4ai/crawlers/__init__.py create mode 100644 crawl4ai/crawlers/amazon_product/__init__.py create mode 100644 crawl4ai/crawlers/amazon_product/crawler.py create mode 100644 crawl4ai/crawlers/google_search/__init__.py create mode 100644 crawl4ai/crawlers/google_search/crawler.py create mode 100644 crawl4ai/crawlers/google_search/script.js create mode 100644 crawl4ai/deep_crawling/__init__.py create mode 100644 crawl4ai/deep_crawling/base_strategy.py create mode 100644 crawl4ai/deep_crawling/bff_strategy.py create mode 100644 crawl4ai/deep_crawling/bfs_strategy.py create mode 100644 crawl4ai/deep_crawling/crazy.py create mode 100644 crawl4ai/deep_crawling/dfs_strategy.py create mode 100644 crawl4ai/deep_crawling/filters.py create mode 100644 crawl4ai/deep_crawling/scorers.py create mode 100644 crawl4ai/docker_client.py create mode 100644 crawl4ai/domain_mapper.py create mode 100644 crawl4ai/extraction_strategy.py create mode 100644 crawl4ai/html2text/__init__.py create mode 100644 crawl4ai/html2text/__main__.py create mode 100644 crawl4ai/html2text/_typing.py create mode 100644 crawl4ai/html2text/cli.py create mode 100644 crawl4ai/html2text/config.py create mode 100644 crawl4ai/html2text/elements.py create mode 100644 crawl4ai/html2text/utils.py create mode 100644 crawl4ai/hub.py create mode 100644 crawl4ai/install.py create mode 100644 crawl4ai/js_snippet/__init__.py create mode 100644 crawl4ai/js_snippet/flatten_shadow_dom.js create mode 100644 crawl4ai/js_snippet/navigator_overrider.js create mode 100644 crawl4ai/js_snippet/remove_consent_popups.js create mode 100644 crawl4ai/js_snippet/remove_overlay_elements.js create mode 100644 crawl4ai/js_snippet/update_image_dimensions.js create mode 100644 crawl4ai/legacy/__init__.py create mode 100644 crawl4ai/legacy/cli.py create mode 100644 crawl4ai/legacy/crawler_strategy.py create mode 100644 crawl4ai/legacy/database.py create mode 100644 crawl4ai/legacy/docs_manager.py create mode 100644 crawl4ai/legacy/llmtxt.py create mode 100644 crawl4ai/legacy/version_manager.py create mode 100644 crawl4ai/legacy/web_crawler.py create mode 100644 crawl4ai/link_preview.py create mode 100644 crawl4ai/markdown_generation_strategy.py create mode 100644 crawl4ai/migrations.py create mode 100644 crawl4ai/model_loader.py create mode 100644 crawl4ai/models.py create mode 100644 crawl4ai/processors/pdf/__init__.py create mode 100644 crawl4ai/processors/pdf/processor.py create mode 100644 crawl4ai/processors/pdf/utils.py create mode 100644 crawl4ai/prompts.py create mode 100644 crawl4ai/proxy_strategy.py create mode 100644 crawl4ai/script/__init__.py create mode 100644 crawl4ai/script/c4a_compile.py create mode 100644 crawl4ai/script/c4a_result.py create mode 100644 crawl4ai/script/c4ai_script.py create mode 100644 crawl4ai/ssl_certificate.py create mode 100644 crawl4ai/table_extraction.py create mode 100644 crawl4ai/types.py create mode 100644 crawl4ai/user_agent_generator.py create mode 100644 crawl4ai/utils.py create mode 100644 deploy/docker/.dockerignore create mode 100644 deploy/docker/.llm.env.example create mode 100644 deploy/docker/ARCHITECTURE.md create mode 100644 deploy/docker/MIGRATION.md create mode 100644 deploy/docker/README.md create mode 100644 deploy/docker/SECURITY-VERIFY.md create mode 100644 deploy/docker/STRESS_TEST_PIPELINE.md create mode 100644 deploy/docker/WEBHOOK_EXAMPLES.md create mode 100644 deploy/docker/api.py create mode 100644 deploy/docker/artifacts.py create mode 100644 deploy/docker/auth.py create mode 100644 deploy/docker/auth_gate.py create mode 100644 deploy/docker/c4ai-code-context.md create mode 100644 deploy/docker/c4ai-doc-context.md create mode 100644 deploy/docker/config.yml create mode 100644 deploy/docker/crawler_pool.py create mode 100644 deploy/docker/egress_broker.py create mode 100644 deploy/docker/egress_proxy.py create mode 100644 deploy/docker/entrypoint.sh create mode 100644 deploy/docker/governor.py create mode 100644 deploy/docker/hook_registry.py create mode 100644 deploy/docker/job.py create mode 100644 deploy/docker/llm_broker.py create mode 100644 deploy/docker/mcp_bridge.py create mode 100644 deploy/docker/monitor.py create mode 100644 deploy/docker/monitor_routes.py create mode 100644 deploy/docker/redis_config.py create mode 100644 deploy/docker/requirements.txt create mode 100644 deploy/docker/schemas.py create mode 100644 deploy/docker/server.py create mode 100644 deploy/docker/static/assets/crawl4ai-logo.jpg create mode 100644 deploy/docker/static/assets/crawl4ai-logo.png create mode 100644 deploy/docker/static/assets/logo.png create mode 100644 deploy/docker/static/monitor/index.html create mode 100644 deploy/docker/static/playground/index.html create mode 100644 deploy/docker/supervisord.conf create mode 100755 deploy/docker/test-websocket.py create mode 100644 deploy/docker/tests/conftest.py create mode 100755 deploy/docker/tests/demo_monitor_dashboard.py create mode 100644 deploy/docker/tests/requirements.txt create mode 100755 deploy/docker/tests/run_security_tests.py create mode 100755 deploy/docker/tests/test_1_basic.py create mode 100755 deploy/docker/tests/test_2_memory.py create mode 100755 deploy/docker/tests/test_3_pool.py create mode 100755 deploy/docker/tests/test_4_concurrent.py create mode 100755 deploy/docker/tests/test_5_pool_stress.py create mode 100755 deploy/docker/tests/test_6_multi_endpoint.py create mode 100755 deploy/docker/tests/test_7_cleanup.py create mode 100644 deploy/docker/tests/test_monitor_demo.py create mode 100644 deploy/docker/tests/test_security_2026_04.py create mode 100644 deploy/docker/tests/test_security_2026_04_b2.py create mode 100644 deploy/docker/tests/test_security_artifact_store.py create mode 100644 deploy/docker/tests/test_security_authz.py create mode 100644 deploy/docker/tests/test_security_container_posture.py create mode 100644 deploy/docker/tests/test_security_default_posture.py create mode 100644 deploy/docker/tests/test_security_download_traversal.py create mode 100644 deploy/docker/tests/test_security_egress_proxy.py create mode 100644 deploy/docker/tests/test_security_fixes.py create mode 100644 deploy/docker/tests/test_security_headers_xss.py create mode 100644 deploy/docker/tests/test_security_llm_broker.py create mode 100644 deploy/docker/tests/test_security_resource_caps.py create mode 100644 deploy/docker/tests/test_security_ssrf_crawl.py create mode 100644 deploy/docker/tests/test_security_ssrf_egress.py create mode 100644 deploy/docker/tests/test_security_trust_boundary.py create mode 100644 deploy/docker/tests/test_security_webhook_pinning.py create mode 100644 deploy/docker/utils.py create mode 100644 deploy/docker/webhook.py create mode 100644 deploy/docker/work_queue.py create mode 100644 docker-compose.yml create mode 100644 docs/RELEASE_NOTES_v0.8.0.md create mode 100644 docs/apps/iseeyou/llms-full.txt create mode 100644 docs/apps/linkdin/Crawl4ai_Linkedin_Data_Discovery_Part_1.ipynb create mode 100644 docs/apps/linkdin/Crawl4ai_Linkedin_Data_Discovery_Part_2.ipynb create mode 100644 docs/apps/linkdin/README.md create mode 100644 docs/apps/linkdin/c4ai_discover.py create mode 100644 docs/apps/linkdin/c4ai_insights.py create mode 100644 docs/apps/linkdin/samples/companies.jsonl create mode 100644 docs/apps/linkdin/samples/people.jsonl create mode 100644 docs/apps/linkdin/schemas/company_card.json create mode 100644 docs/apps/linkdin/schemas/people_card.json create mode 100644 docs/apps/linkdin/snippets/company.html create mode 100644 docs/apps/linkdin/snippets/people.html create mode 100644 docs/apps/linkdin/templates/ai.js create mode 100644 docs/apps/linkdin/templates/graph_view_template.html create mode 100644 docs/assets/pitch-dark.png create mode 100644 docs/assets/pitch-dark.svg create mode 100644 docs/assets/powered-by-dark.svg create mode 100644 docs/assets/powered-by-disco.svg create mode 100644 docs/assets/powered-by-light.svg create mode 100644 docs/assets/powered-by-night.svg create mode 100644 docs/assets/sponsors/aleph_null.svg create mode 100644 docs/assets/sponsors/aleph_null_light.svg create mode 100644 docs/assets/sponsors/massive.svg create mode 100644 docs/assets/sponsors/massive_light.svg create mode 100644 docs/assets/sponsors/nst-light.svg create mode 100644 docs/assets/sponsors/thor_data.svg create mode 100644 docs/assets/sponsors/thor_data_light.svg create mode 100644 docs/blog/release-v0.7.0.md create mode 100644 docs/blog/release-v0.7.1.md create mode 100644 docs/blog/release-v0.7.3.md create mode 100644 docs/blog/release-v0.7.4.md create mode 100644 docs/blog/release-v0.7.5.md create mode 100644 docs/blog/release-v0.7.6.md create mode 100644 docs/blog/release-v0.7.7.md create mode 100644 docs/blog/release-v0.7.8.md create mode 100644 docs/blog/release-v0.8.0.md create mode 100644 docs/blog/release-v0.8.5.md create mode 100644 docs/blog/release-v0.8.7.md create mode 100644 docs/blog/release-v0.8.8.md create mode 100644 docs/blog/release-v0.8.9.md create mode 100644 docs/blog/release-v0.9.0.md create mode 100644 docs/blog/release-v0.9.1.md create mode 100644 docs/codebase/browser.md create mode 100644 docs/codebase/cli.md create mode 100644 docs/deprecated/docker-deployment.md create mode 100644 docs/examples/README_BUILTIN_BROWSER.md create mode 100644 docs/examples/adaptive_crawling/README.md create mode 100644 docs/examples/adaptive_crawling/advanced_configuration.py create mode 100644 docs/examples/adaptive_crawling/basic_usage.py create mode 100644 docs/examples/adaptive_crawling/custom_strategies.py create mode 100644 docs/examples/adaptive_crawling/embedding_configuration.py create mode 100644 docs/examples/adaptive_crawling/embedding_strategy.py create mode 100644 docs/examples/adaptive_crawling/embedding_vs_statistical.py create mode 100644 docs/examples/adaptive_crawling/export_import_kb.py create mode 100644 docs/examples/adaptive_crawling/llm_config_example.py create mode 100644 docs/examples/amazon_product_extraction_direct_url.py create mode 100644 docs/examples/amazon_product_extraction_using_hooks.py create mode 100644 docs/examples/amazon_product_extraction_using_use_javascript.py create mode 100644 docs/examples/arun_vs_arun_many.py create mode 100644 docs/examples/assets/audio.mp3 create mode 100644 docs/examples/assets/basic.png create mode 100644 docs/examples/assets/cosine_extraction.png create mode 100644 docs/examples/assets/css_js.png create mode 100644 docs/examples/assets/css_selector.png create mode 100644 docs/examples/assets/exec_script.png create mode 100644 docs/examples/assets/instagram_grid_result.png create mode 100644 docs/examples/assets/llm_extraction.png create mode 100644 docs/examples/assets/semantic_extraction_cosine.png create mode 100644 docs/examples/assets/semantic_extraction_llm.png create mode 100644 docs/examples/assets/virtual_scroll_append_only.html create mode 100644 docs/examples/assets/virtual_scroll_instagram_grid.html create mode 100644 docs/examples/assets/virtual_scroll_news_feed.html create mode 100644 docs/examples/assets/virtual_scroll_twitter_like.html create mode 100644 docs/examples/async_webcrawler_multiple_urls_example.py create mode 100644 docs/examples/browser_optimization_example.py create mode 100644 docs/examples/builtin_browser_example.py create mode 100644 docs/examples/c4a_script/amazon_example/README.md create mode 100644 docs/examples/c4a_script/amazon_example/amazon_r2d2_search.py create mode 100644 docs/examples/c4a_script/amazon_example/extracted_products.json create mode 100644 docs/examples/c4a_script/amazon_example/generated_product_schema.json create mode 100644 docs/examples/c4a_script/amazon_example/generated_search_script.js create mode 100644 docs/examples/c4a_script/amazon_example/header.html create mode 100644 docs/examples/c4a_script/amazon_example/product.html create mode 100644 docs/examples/c4a_script/api_usage_examples.py create mode 100644 docs/examples/c4a_script/c4a_script_hello_world.py create mode 100644 docs/examples/c4a_script/c4a_script_hello_world_error.py create mode 100644 docs/examples/c4a_script/demo_c4a_crawl4ai.py create mode 100644 docs/examples/c4a_script/generate_script_hello_world.py create mode 100644 docs/examples/c4a_script/github_search/extracted_repositories.json create mode 100644 docs/examples/c4a_script/github_search/generated_result_schema.json create mode 100644 docs/examples/c4a_script/github_search/generated_search_script.js create mode 100644 docs/examples/c4a_script/github_search/github_search_crawler.py create mode 100644 docs/examples/c4a_script/github_search/result.html create mode 100644 docs/examples/c4a_script/github_search/search_form.html create mode 100644 docs/examples/c4a_script/script_samples/add_to_cart.c4a create mode 100644 docs/examples/c4a_script/script_samples/advanced_control_flow.c4a create mode 100644 docs/examples/c4a_script/script_samples/conditional_login.c4a create mode 100644 docs/examples/c4a_script/script_samples/data_extraction.c4a create mode 100644 docs/examples/c4a_script/script_samples/fill_contact.c4a create mode 100644 docs/examples/c4a_script/script_samples/load_more_content.c4a create mode 100644 docs/examples/c4a_script/script_samples/login_flow.c4a create mode 100644 docs/examples/c4a_script/script_samples/multi_step_workflow.c4a create mode 100644 docs/examples/c4a_script/script_samples/navigate_tabs.c4a create mode 100644 docs/examples/c4a_script/script_samples/quick_login.c4a create mode 100644 docs/examples/c4a_script/script_samples/responsive_actions.c4a create mode 100644 docs/examples/c4a_script/script_samples/scroll_and_click.c4a create mode 100644 docs/examples/c4a_script/script_samples/search_product.c4a create mode 100644 docs/examples/c4a_script/script_samples/simple_form.c4a create mode 100644 docs/examples/c4a_script/script_samples/smart_form_fill.c4a create mode 100644 docs/examples/c4a_script/tutorial/README.md create mode 100644 docs/examples/c4a_script/tutorial/assets/DankMono-Bold.woff2 create mode 100644 docs/examples/c4a_script/tutorial/assets/DankMono-Italic.woff2 create mode 100644 docs/examples/c4a_script/tutorial/assets/DankMono-Regular.woff2 create mode 100644 docs/examples/c4a_script/tutorial/assets/app.css create mode 100644 docs/examples/c4a_script/tutorial/assets/app.js create mode 100644 docs/examples/c4a_script/tutorial/assets/blockly-manager.js create mode 100644 docs/examples/c4a_script/tutorial/assets/blockly-theme.css create mode 100644 docs/examples/c4a_script/tutorial/assets/c4a-blocks.js create mode 100644 docs/examples/c4a_script/tutorial/assets/c4a-generator.js create mode 100644 docs/examples/c4a_script/tutorial/assets/styles.css create mode 100644 docs/examples/c4a_script/tutorial/blockly-demo.c4a create mode 100644 docs/examples/c4a_script/tutorial/index.html create mode 100644 docs/examples/c4a_script/tutorial/playground/app.js create mode 100644 docs/examples/c4a_script/tutorial/playground/index.html create mode 100644 docs/examples/c4a_script/tutorial/playground/styles.css create mode 100644 docs/examples/c4a_script/tutorial/requirements.txt create mode 100644 docs/examples/c4a_script/tutorial/server.py create mode 100644 docs/examples/c4a_script/tutorial/test_blockly.html create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_aws_waf.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_challenge.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_turnstile.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v2.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v3.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_aws_waf.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_challenge.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_turnstile.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v2.py create mode 100644 docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v3.py create mode 100644 docs/examples/chainlit.md create mode 100644 docs/examples/cli/browser.yml create mode 100644 docs/examples/cli/crawler.yml create mode 100644 docs/examples/cli/css_schema.json create mode 100644 docs/examples/cli/extract.yml create mode 100644 docs/examples/cli/extract_css.yml create mode 100644 docs/examples/cli/llm_schema.json create mode 100644 docs/examples/cloud_browser/scrapeless_browser.py create mode 100644 docs/examples/crawlai_vs_firecrawl.py create mode 100644 docs/examples/crawler_monitor_example.py create mode 100644 docs/examples/crypto_analysis_example.py create mode 100644 docs/examples/deep_crawl_cancellation.py create mode 100644 docs/examples/deep_crawl_crash_recovery.py create mode 100644 docs/examples/deepcrawl_example.py create mode 100644 docs/examples/demo_multi_config_clean.py create mode 100644 docs/examples/dfs_crawl_demo.py create mode 100644 docs/examples/dispatcher_example.py create mode 100644 docs/examples/docker/demo_docker_api.py create mode 100644 docs/examples/docker/demo_docker_polling.py create mode 100644 docs/examples/docker_client_hooks_example.py create mode 100644 docs/examples/docker_config_obj.py create mode 100644 docs/examples/docker_example.py create mode 100644 docs/examples/docker_hooks_examples.py create mode 100644 docs/examples/docker_python_rest_api.py create mode 100644 docs/examples/docker_python_sdk.py create mode 100644 docs/examples/docker_webhook_example.py create mode 100644 docs/examples/domain_mapper/domain_mapper_demo.py create mode 100644 docs/examples/extraction_strategies_examples.py create mode 100644 docs/examples/full_page_screenshot_and_pdf_export.md create mode 100644 docs/examples/hello_world.py create mode 100644 docs/examples/hello_world_undetected.py create mode 100644 docs/examples/hooks_example.py create mode 100644 docs/examples/identity_based_browsing.py create mode 100644 docs/examples/language_support_example.py create mode 100644 docs/examples/link_head_extraction_example.py create mode 100644 docs/examples/llm_extraction_openai_pricing.py create mode 100644 docs/examples/llm_markdown_generator.py create mode 100644 docs/examples/llm_table_extraction_example.py create mode 100644 docs/examples/markdown/content_source_example.py create mode 100644 docs/examples/markdown/content_source_short_example.py create mode 100644 docs/examples/network_console_capture_example.py create mode 100644 docs/examples/nst_proxy/api_proxy_example.py create mode 100644 docs/examples/nst_proxy/auth_proxy_example.py create mode 100644 docs/examples/nst_proxy/basic_proxy_example.py create mode 100644 docs/examples/nst_proxy/nstproxy_example.py create mode 100644 docs/examples/prefetch_two_phase_crawl.py create mode 100644 docs/examples/proxy_rotation_demo.py create mode 100644 docs/examples/quickstart.ipynb create mode 100644 docs/examples/quickstart.py create mode 100644 docs/examples/quickstart_examples_set_1.py create mode 100644 docs/examples/quickstart_examples_set_2.py create mode 100644 docs/examples/regex_extraction_quickstart.py create mode 100644 docs/examples/research_assistant.py create mode 100644 docs/examples/rest_call.py create mode 100644 docs/examples/sample_ecommerce.html create mode 100644 docs/examples/scraping_strategies_performance.py create mode 100644 docs/examples/serp_api_project_11_feb.py create mode 100644 docs/examples/session_id_example.py create mode 100644 docs/examples/shadow_dom_crawling.py create mode 100644 docs/examples/simple_anti_bot_examples.py create mode 100644 docs/examples/ssl_example.py create mode 100644 docs/examples/stealth_mode_example.py create mode 100644 docs/examples/stealth_mode_quick_start.py create mode 100644 docs/examples/stealth_test_simple.py create mode 100644 docs/examples/storage_state_tutorial.md create mode 100644 docs/examples/summarize_page.py create mode 100644 docs/examples/table_extraction_example.py create mode 100644 docs/examples/tutorial_dynamic_clicks.md create mode 100644 docs/examples/tutorial_v0.5.py create mode 100644 docs/examples/undetectability/undetected_basic_test.py create mode 100644 docs/examples/undetectability/undetected_bot_test.py create mode 100644 docs/examples/undetectability/undetected_cloudflare_test.py create mode 100644 docs/examples/undetectability/undetected_vs_regular_comparison.py create mode 100644 docs/examples/undetected_simple_demo.py create mode 100644 docs/examples/url_seeder/Crawl4AI_URL_Seeder_Tutorial.ipynb create mode 100644 docs/examples/url_seeder/bbc_sport_research_assistant.py create mode 100644 docs/examples/url_seeder/convert_tutorial_to_colab.py create mode 100644 docs/examples/url_seeder/tutorial_url_seeder.md create mode 100644 docs/examples/url_seeder/url_seeder_demo.py create mode 100644 docs/examples/url_seeder/url_seeder_quick_demo.py create mode 100644 docs/examples/use_geo_location.py create mode 100644 docs/examples/virtual_scroll_example.py create mode 100644 docs/examples/website-to-api/.gitignore create mode 100644 docs/examples/website-to-api/README.md create mode 100644 docs/examples/website-to-api/api_server.py create mode 100644 docs/examples/website-to-api/app.py create mode 100644 docs/examples/website-to-api/assets/crawl4ai_logo.jpg create mode 100644 docs/examples/website-to-api/requirements.txt create mode 100644 docs/examples/website-to-api/static/index.html create mode 100644 docs/examples/website-to-api/static/script.js create mode 100644 docs/examples/website-to-api/static/styles.css create mode 100644 docs/examples/website-to-api/test_api.py create mode 100644 docs/examples/website-to-api/test_models.py create mode 100644 docs/examples/website-to-api/web_scraper_lib.py create mode 100644 docs/md_v2/CONTRIBUTING.md create mode 100644 docs/md_v2/advanced/adaptive-strategies.md create mode 100644 docs/md_v2/advanced/advanced-features.md create mode 100644 docs/md_v2/advanced/anti-bot-and-fallback.md create mode 100644 docs/md_v2/advanced/crawl-dispatcher.md create mode 100644 docs/md_v2/advanced/file-downloading.md create mode 100644 docs/md_v2/advanced/hooks-auth.md create mode 100644 docs/md_v2/advanced/identity-based-crawling.md create mode 100644 docs/md_v2/advanced/lazy-loading.md create mode 100644 docs/md_v2/advanced/multi-url-crawling.md create mode 100644 docs/md_v2/advanced/network-console-capture.md create mode 100644 docs/md_v2/advanced/pdf-parsing.md create mode 100644 docs/md_v2/advanced/proxy-security.md create mode 100644 docs/md_v2/advanced/session-management.md create mode 100644 docs/md_v2/advanced/ssl-certificate.md create mode 100644 docs/md_v2/advanced/undetected-browser.md create mode 100644 docs/md_v2/advanced/virtual-scroll.md create mode 100644 docs/md_v2/api/adaptive-crawler.md create mode 100644 docs/md_v2/api/arun.md create mode 100644 docs/md_v2/api/arun_many.md create mode 100644 docs/md_v2/api/async-webcrawler.md create mode 100644 docs/md_v2/api/c4a-script-reference.md create mode 100644 docs/md_v2/api/crawl-result.md create mode 100644 docs/md_v2/api/digest.md create mode 100644 docs/md_v2/api/parameters.md create mode 100644 docs/md_v2/api/strategies.md create mode 100644 docs/md_v2/apps/assets/DankMono-Bold.woff2 create mode 100644 docs/md_v2/apps/assets/DankMono-Italic.woff2 create mode 100644 docs/md_v2/apps/assets/DankMono-Regular.woff2 create mode 100644 docs/md_v2/apps/c4a-script/README.md create mode 100644 docs/md_v2/apps/c4a-script/assets/DankMono-Bold.woff2 create mode 100644 docs/md_v2/apps/c4a-script/assets/DankMono-Italic.woff2 create mode 100644 docs/md_v2/apps/c4a-script/assets/DankMono-Regular.woff2 create mode 100644 docs/md_v2/apps/c4a-script/assets/app.css create mode 100644 docs/md_v2/apps/c4a-script/assets/app.js create mode 100644 docs/md_v2/apps/c4a-script/assets/blockly-manager.js create mode 100644 docs/md_v2/apps/c4a-script/assets/blockly-theme.css create mode 100644 docs/md_v2/apps/c4a-script/assets/c4a-blocks.js create mode 100644 docs/md_v2/apps/c4a-script/assets/c4a-generator.js create mode 100644 docs/md_v2/apps/c4a-script/assets/styles.css create mode 100644 docs/md_v2/apps/c4a-script/blockly-demo.c4a create mode 100644 docs/md_v2/apps/c4a-script/index.html create mode 100644 docs/md_v2/apps/c4a-script/playground/app.js create mode 100644 docs/md_v2/apps/c4a-script/playground/index.html create mode 100644 docs/md_v2/apps/c4a-script/playground/styles.css create mode 100644 docs/md_v2/apps/c4a-script/requirements.txt create mode 100644 docs/md_v2/apps/c4a-script/server.py create mode 100644 docs/md_v2/apps/c4a-script/test_blockly.html create mode 100644 docs/md_v2/apps/crawl4ai-assistant/README.md create mode 100644 docs/md_v2/apps/crawl4ai-assistant/assets/DankMono-Bold.woff2 create mode 100644 docs/md_v2/apps/crawl4ai-assistant/assets/DankMono-Italic.woff2 create mode 100644 docs/md_v2/apps/crawl4ai-assistant/assets/DankMono-Regular.woff2 create mode 100644 docs/md_v2/apps/crawl4ai-assistant/assistant.css create mode 100644 docs/md_v2/apps/crawl4ai-assistant/background/service-worker.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/click2crawl.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/content.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/contentAnalyzer.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/markdownConverter.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/markdownExtraction.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/markdownPreviewModal.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/overlay.css create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/scriptBuilder.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/content/shared/utils.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/crawl4ai-assistant-v1.2.1.zip create mode 100644 docs/md_v2/apps/crawl4ai-assistant/crawl4ai-assistant-v1.3.0.zip create mode 100644 docs/md_v2/apps/crawl4ai-assistant/icons/favicon.ico create mode 100644 docs/md_v2/apps/crawl4ai-assistant/icons/icon-128.png create mode 100644 docs/md_v2/apps/crawl4ai-assistant/icons/icon-16.png create mode 100644 docs/md_v2/apps/crawl4ai-assistant/icons/icon-48.png create mode 100644 docs/md_v2/apps/crawl4ai-assistant/index.html create mode 100644 docs/md_v2/apps/crawl4ai-assistant/libs/marked.min.js create mode 100644 docs/md_v2/apps/crawl4ai-assistant/manifest.json create mode 100644 docs/md_v2/apps/crawl4ai-assistant/popup/icons/favicon.ico create mode 100644 docs/md_v2/apps/crawl4ai-assistant/popup/icons/icon-128.png create mode 100644 docs/md_v2/apps/crawl4ai-assistant/popup/icons/icon-16.png create mode 100644 docs/md_v2/apps/crawl4ai-assistant/popup/icons/icon-48.png create mode 100644 docs/md_v2/apps/crawl4ai-assistant/popup/popup.css create mode 100644 docs/md_v2/apps/crawl4ai-assistant/popup/popup.html create mode 100644 docs/md_v2/apps/crawl4ai-assistant/popup/popup.js create mode 100644 docs/md_v2/apps/index.md create mode 100644 docs/md_v2/apps/llmtxt/build.md create mode 100644 docs/md_v2/apps/llmtxt/index.html create mode 100644 docs/md_v2/apps/llmtxt/llmtxt.css create mode 100644 docs/md_v2/apps/llmtxt/llmtxt.js create mode 100644 docs/md_v2/apps/llmtxt/why.md create mode 100644 docs/md_v2/ask_ai/ask-ai.css create mode 100644 docs/md_v2/ask_ai/ask-ai.js create mode 100644 docs/md_v2/ask_ai/index.html create mode 100644 docs/md_v2/assets/DankMono-Bold.woff2 create mode 100644 docs/md_v2/assets/DankMono-Italic.woff2 create mode 100644 docs/md_v2/assets/DankMono-Regular.woff2 create mode 100644 docs/md_v2/assets/Monaco.woff create mode 100644 docs/md_v2/assets/copy_code.js create mode 100644 docs/md_v2/assets/crawl4ai-skill.zip create mode 100644 docs/md_v2/assets/dmvendor.css create mode 100644 docs/md_v2/assets/docs.zip create mode 100644 docs/md_v2/assets/feedback-overrides.css create mode 100644 docs/md_v2/assets/floating_ask_ai_button.js create mode 100644 docs/md_v2/assets/github_stats.js create mode 100644 docs/md_v2/assets/gtag.js create mode 100644 docs/md_v2/assets/highlight.css create mode 100644 docs/md_v2/assets/highlight.min.js create mode 100644 docs/md_v2/assets/highlight_init.js create mode 100644 docs/md_v2/assets/images/dispatcher.png create mode 100644 docs/md_v2/assets/images/logo.png create mode 100644 docs/md_v2/assets/layout.css create mode 100644 docs/md_v2/assets/llm.txt/diagrams/cli.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/config_objects.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/deep_crawl_advanced_filters_scorers.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/deep_crawling.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/docker.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/extraction-llm.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/extraction-no-llm.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/http_based_crawler_strategy.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/installation.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/llms-diagram.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/multi_urls_crawling.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/simple_crawling.txt create mode 100644 docs/md_v2/assets/llm.txt/diagrams/url_seeder.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/cli.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/config_objects.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/deep_crawl_advanced_filters_scorers.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/deep_crawling.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/docker.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/extraction-llm.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/extraction-no-llm.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/http_based_crawler_strategy.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/installation.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/llms-full-v0.1.1.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/llms-full.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/multi_urls_crawling.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/simple_crawling.txt create mode 100644 docs/md_v2/assets/llm.txt/txt/url_seeder.txt create mode 100644 docs/md_v2/assets/mobile_menu.js create mode 100644 docs/md_v2/assets/page_actions.css create mode 100644 docs/md_v2/assets/page_actions.js create mode 100644 docs/md_v2/assets/selection_ask_ai.js create mode 100644 docs/md_v2/assets/styles.css create mode 100644 docs/md_v2/assets/test/toc.js create mode 100644 docs/md_v2/assets/toc.js create mode 100644 docs/md_v2/basic/installation.md create mode 100644 docs/md_v2/blog/articles/adaptive-crawling-revolution.md create mode 100644 docs/md_v2/blog/articles/dockerize_hooks.md create mode 100644 docs/md_v2/blog/articles/llm-context-revolution.md create mode 100644 docs/md_v2/blog/articles/virtual-scroll-revolution.md create mode 100644 docs/md_v2/blog/index.md create mode 100644 docs/md_v2/blog/index.md.bak create mode 100644 docs/md_v2/blog/releases/0.4.0.md create mode 100644 docs/md_v2/blog/releases/0.4.1.md create mode 100644 docs/md_v2/blog/releases/0.4.2.md create mode 100644 docs/md_v2/blog/releases/0.5.0.md create mode 100644 docs/md_v2/blog/releases/0.6.0.md create mode 100644 docs/md_v2/blog/releases/0.7.0.md create mode 100644 docs/md_v2/blog/releases/0.7.1.md create mode 100644 docs/md_v2/blog/releases/0.7.2.md create mode 100644 docs/md_v2/blog/releases/0.7.3.md create mode 100644 docs/md_v2/blog/releases/0.7.6.md create mode 100644 docs/md_v2/blog/releases/v0.4.3b1.md create mode 100644 docs/md_v2/blog/releases/v0.7.5.md create mode 100644 docs/md_v2/blog/releases/v0.7.7.md create mode 100644 docs/md_v2/blog/releases/v0.7.8.md create mode 100644 docs/md_v2/blog/releases/v0.8.0.md create mode 100644 docs/md_v2/blog/releases/v0.8.5.md create mode 100644 docs/md_v2/blog/releases/v0.9.1.md create mode 100644 docs/md_v2/branding/index.md create mode 100644 docs/md_v2/complete-sdk-reference.md create mode 100644 docs/md_v2/core/adaptive-crawling.md create mode 100644 docs/md_v2/core/ask-ai.md create mode 100644 docs/md_v2/core/browser-crawler-config.md create mode 100644 docs/md_v2/core/c4a-script.md create mode 100644 docs/md_v2/core/cache-modes.md create mode 100644 docs/md_v2/core/cli.md create mode 100644 docs/md_v2/core/content-selection.md create mode 100644 docs/md_v2/core/crawler-result.md create mode 100644 docs/md_v2/core/deep-crawling.md create mode 100644 docs/md_v2/core/domain-mapping.md create mode 100644 docs/md_v2/core/examples.md create mode 100644 docs/md_v2/core/fit-markdown.md create mode 100644 docs/md_v2/core/installation.md create mode 100644 docs/md_v2/core/link-media.md create mode 100644 docs/md_v2/core/llmtxt.md create mode 100644 docs/md_v2/core/local-files.md create mode 100644 docs/md_v2/core/markdown-generation.md create mode 100644 docs/md_v2/core/page-interaction.md create mode 100644 docs/md_v2/core/quickstart.md create mode 100644 docs/md_v2/core/self-hosting.md create mode 100644 docs/md_v2/core/simple-crawling.md create mode 100644 docs/md_v2/core/table_extraction.md create mode 100644 docs/md_v2/core/url-seeding.md create mode 100644 docs/md_v2/extraction/chunking.md create mode 100644 docs/md_v2/extraction/clustring-strategies.md create mode 100644 docs/md_v2/extraction/llm-strategies.md create mode 100644 docs/md_v2/extraction/no-llm-strategies.md create mode 100644 docs/md_v2/favicon.ico create mode 100644 docs/md_v2/img/favicon-32x32.png create mode 100644 docs/md_v2/img/favicon-x-32x32.png create mode 100644 docs/md_v2/img/favicon.ico create mode 100644 docs/md_v2/index.md create mode 100644 docs/md_v2/marketplace/README.md create mode 100644 docs/md_v2/marketplace/admin/admin.css create mode 100644 docs/md_v2/marketplace/admin/admin.js create mode 100644 docs/md_v2/marketplace/admin/index.html create mode 100644 docs/md_v2/marketplace/app-detail.css create mode 100644 docs/md_v2/marketplace/app-detail.html create mode 100644 docs/md_v2/marketplace/app-detail.js create mode 100644 docs/md_v2/marketplace/backend/.env.example create mode 100644 docs/md_v2/marketplace/backend/config.py create mode 100644 docs/md_v2/marketplace/backend/database.py create mode 100644 docs/md_v2/marketplace/backend/dummy_data.py create mode 100644 docs/md_v2/marketplace/backend/requirements.txt create mode 100644 docs/md_v2/marketplace/backend/schema.yaml create mode 100644 docs/md_v2/marketplace/backend/server.py create mode 100644 docs/md_v2/marketplace/backend/uploads/.gitignore create mode 100644 docs/md_v2/marketplace/frontend/app-detail.css create mode 100644 docs/md_v2/marketplace/frontend/app-detail.html create mode 100644 docs/md_v2/marketplace/frontend/app-detail.js create mode 100644 docs/md_v2/marketplace/frontend/index.html create mode 100644 docs/md_v2/marketplace/frontend/marketplace.css create mode 100644 docs/md_v2/marketplace/frontend/marketplace.js create mode 100644 docs/md_v2/marketplace/index.html create mode 100644 docs/md_v2/marketplace/marketplace.css create mode 100644 docs/md_v2/marketplace/marketplace.js create mode 100644 docs/md_v2/migration/table_extraction_v073.md create mode 100644 docs/md_v2/migration/webscraping-strategy-migration.md create mode 100644 docs/md_v2/overrides/main.html create mode 100644 docs/md_v2/privacy.md create mode 100644 docs/md_v2/stats.md create mode 100644 docs/md_v2/support.md create mode 100644 docs/md_v2/terms.md create mode 100644 docs/migration/v0.8.0-upgrade-guide.md create mode 100644 docs/releases_review/Crawl4AI_v0.3.72_Release_Announcement.ipynb create mode 100644 docs/releases_review/crawl4ai_v0_7_0_showcase.py create mode 100644 docs/releases_review/demo_v0.7.0.py create mode 100644 docs/releases_review/demo_v0.7.5.py create mode 100644 docs/releases_review/demo_v0.7.6.py create mode 100644 docs/releases_review/demo_v0.7.7.py create mode 100644 docs/releases_review/demo_v0.7.8.py create mode 100644 docs/releases_review/demo_v0.8.0.py create mode 100644 docs/releases_review/demo_v0.8.5.py create mode 100644 docs/releases_review/demo_v0.9.1.py create mode 100644 docs/releases_review/v0.3.74.overview.py create mode 100644 docs/releases_review/v0.7.5_docker_hooks_demo.py create mode 100644 docs/releases_review/v0.7.5_video_walkthrough.ipynb create mode 100644 docs/releases_review/v0_4_24_walkthrough.py create mode 100644 docs/releases_review/v0_4_3b2_features_demo.py create mode 100644 docs/releases_review/v0_7_0_features_demo.py create mode 100644 docs/security/GHSA-DRAFT-RCE-LFI.md create mode 100644 docs/snippets/deep_crawl/1.intro.py create mode 100644 docs/snippets/deep_crawl/2.filters.py create mode 100644 docs/tutorials/coming_soon.md create mode 100644 mkdocs.yml create mode 100644 prompts/prompt_net_requests.md create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 sbom/README.md create mode 100644 sbom/sbom.cdx.json create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test_llm_webhook_feature.py create mode 100644 test_webhook_implementation.py create mode 100644 tests/WEBHOOK_TEST_README.md create mode 100644 tests/__init__.py create mode 100644 tests/adaptive/compare_performance.py create mode 100644 tests/adaptive/test_adaptive_crawler.py create mode 100644 tests/adaptive/test_confidence_debug.py create mode 100644 tests/adaptive/test_embedding_performance.py create mode 100644 tests/adaptive/test_embedding_strategy.py create mode 100644 tests/adaptive/test_llm_embedding.py create mode 100644 tests/adaptive/test_query_llm_config.py create mode 100644 tests/adversarial/test_domain_mapper_adversarial.py create mode 100644 tests/async/sample_wikipedia.html create mode 100644 tests/async/test_0.4.2_browser_manager.py create mode 100644 tests/async/test_0.4.2_config_params.py create mode 100644 tests/async/test_async_doanloader.py create mode 100644 tests/async/test_basic_crawling.py create mode 100644 tests/async/test_browser_lifecycle.py create mode 100644 tests/async/test_browser_memory.py create mode 100644 tests/async/test_browser_recycle_v2.py create mode 100644 tests/async/test_caching.py create mode 100644 tests/async/test_chunking_and_extraction_strategies.py create mode 100644 tests/async/test_content_extraction.py create mode 100644 tests/async/test_content_filter_bm25.py create mode 100644 tests/async/test_content_filter_prune.py create mode 100644 tests/async/test_content_scraper_strategy.py create mode 100644 tests/async/test_crawler_strategy.py create mode 100644 tests/async/test_database_operations.py create mode 100644 tests/async/test_dispatchers.py create mode 100644 tests/async/test_edge_cases.py create mode 100644 tests/async/test_error_handling.py create mode 100644 tests/async/test_evaluation_scraping_methods_performance.configs.py create mode 100644 tests/async/test_http_file_download.py create mode 100644 tests/async/test_markdown_genertor.py create mode 100644 tests/async/test_parameters_and_options.py create mode 100644 tests/async/test_performance.py create mode 100644 tests/async/test_redirect_url_resolution.py create mode 100644 tests/async/test_screenshot.py create mode 100644 tests/async_assistant/test_extract_pipeline.py create mode 100644 tests/async_assistant/test_extract_pipeline_v2.py create mode 100644 tests/browser/docker/__init__.py create mode 100644 tests/browser/docker/test_docker_browser.py create mode 100644 tests/browser/manager/demo_browser_manager.py create mode 100644 tests/browser/test_browser_context_id.py create mode 100644 tests/browser/test_browser_manager.py create mode 100644 tests/browser/test_browser_manager_close.py create mode 100644 tests/browser/test_builtin_browser.py create mode 100644 tests/browser/test_builtin_strategy.py create mode 100644 tests/browser/test_cdp_cleanup_reuse.py create mode 100644 tests/browser/test_cdp_strategy.py create mode 100644 tests/browser/test_combined.py create mode 100644 tests/browser/test_context_leak_fix.py create mode 100644 tests/browser/test_init_script_dedup.py create mode 100644 tests/browser/test_launch_standalone.py create mode 100644 tests/browser/test_page_reuse_race_condition.py create mode 100644 tests/browser/test_parallel_crawling.py create mode 100644 tests/browser/test_playwright_strategy.py create mode 100644 tests/browser/test_profile_shrink.py create mode 100644 tests/browser/test_profiles.py create mode 100644 tests/browser/test_repro_1640.py create mode 100644 tests/browser/test_resource_filtering.py create mode 100644 tests/cache_validation/__init__.py create mode 100644 tests/cache_validation/conftest.py create mode 100644 tests/cache_validation/test_end_to_end.py create mode 100644 tests/cache_validation/test_head_fingerprint.py create mode 100644 tests/cache_validation/test_real_domains.py create mode 100755 tests/check_dependencies.py create mode 100644 tests/cli/test_cli.py create mode 100644 tests/deep_crawling/__init__.py create mode 100644 tests/deep_crawling/test_deep_crawl_cancellation.py create mode 100644 tests/deep_crawling/test_deep_crawl_contextvar.py create mode 100644 tests/deep_crawling/test_deep_crawl_resume.py create mode 100644 tests/deep_crawling/test_deep_crawl_resume_integration.py create mode 100644 tests/deep_crwaling/test_filter.py create mode 100644 tests/docker/simple_api_test.py create mode 100644 tests/docker/test_config_object.py create mode 100644 tests/docker/test_docker.py create mode 100644 tests/docker/test_dockerclient.py create mode 100644 tests/docker/test_filter_deep_crawl.py create mode 100644 tests/docker/test_hooks_client.py create mode 100644 tests/docker/test_hooks_comprehensive.py create mode 100644 tests/docker/test_hooks_utility.py create mode 100755 tests/docker/test_llm_params.py create mode 100644 tests/docker/test_pool_release.py create mode 100644 tests/docker/test_rest_api_deep_crawl.py create mode 100644 tests/docker/test_serialization.py create mode 100644 tests/docker/test_server.py create mode 100644 tests/docker/test_server_requests.py create mode 100644 tests/docker/test_server_token.py create mode 100644 tests/docker_example.py create mode 100644 tests/general/generate_dummy_site.py create mode 100644 tests/general/test_acyn_crawl_wuth_http_crawler_strategy.py create mode 100644 tests/general/test_advanced_deep_crawl.py create mode 100644 tests/general/test_async_crawler_strategy.py create mode 100644 tests/general/test_async_markdown_generator.py create mode 100644 tests/general/test_async_url_seeder_bm25.py create mode 100644 tests/general/test_async_webcrawler.py create mode 100644 tests/general/test_bff_scoring.py create mode 100644 tests/general/test_cache_context.py create mode 100644 tests/general/test_content_source_parameter.py create mode 100644 tests/general/test_crawlers.py create mode 100644 tests/general/test_deep_crawl.py create mode 100644 tests/general/test_deep_crawl_filters.py create mode 100644 tests/general/test_deep_crawl_scorers.py create mode 100644 tests/general/test_download_file.py create mode 100644 tests/general/test_flatten_shadow_dom.py create mode 100644 tests/general/test_generate_schema_usage.py create mode 100644 tests/general/test_http_crawler_strategy.py create mode 100644 tests/general/test_llm_filter.py create mode 100644 tests/general/test_max_scroll.py create mode 100644 tests/general/test_mhtml.py create mode 100644 tests/general/test_network_console_capture.py create mode 100644 tests/general/test_persistent_context.py create mode 100644 tests/general/test_robot_parser.py create mode 100644 tests/general/test_schema_builder.py create mode 100644 tests/general/test_stream.py create mode 100644 tests/general/test_stream_dispatch.py create mode 100644 tests/general/test_strip_markdown_fences.py create mode 100644 tests/general/test_url_pattern.py create mode 100644 tests/general/test_url_seeder_for_only_sitemap.py create mode 100644 tests/general/tets_robot.py create mode 100644 tests/hub/test_simple.py create mode 100644 tests/integration/test_domain_mapper_e2e.py create mode 100644 tests/loggers/test_logger.py create mode 100644 tests/mcp/test_mcp_socket.py create mode 100644 tests/mcp/test_mcp_sse.py create mode 100644 tests/memory/README.md create mode 100755 tests/memory/benchmark_report.py create mode 100644 tests/memory/cap_test.py create mode 100644 tests/memory/requirements.txt create mode 100755 tests/memory/run_benchmark.py create mode 100644 tests/memory/test_crawler_monitor.py create mode 100644 tests/memory/test_dispatcher_stress.py create mode 100644 tests/memory/test_docker_config_gen.py create mode 100644 tests/memory/test_stress_api.py create mode 100644 tests/memory/test_stress_api_xs.py create mode 100644 tests/memory/test_stress_docker_api.py create mode 100644 tests/memory/test_stress_sdk.py create mode 100644 tests/profiler/test_create_profile.py create mode 100644 tests/profiler/test_keyboard_handle.py create mode 100644 tests/proxy/test_antibot_detector.py create mode 100644 tests/proxy/test_chanel_cdp_proxy.py create mode 100644 tests/proxy/test_persistent_proxy.py create mode 100644 tests/proxy/test_proxy_config.py create mode 100644 tests/proxy/test_proxy_deprecation.py create mode 100644 tests/proxy/test_proxy_regression.py create mode 100644 tests/proxy/test_proxy_verify.py create mode 100644 tests/proxy/test_sticky_sessions.py create mode 100644 tests/regression/__init__.py create mode 100644 tests/regression/conftest.py create mode 100644 tests/regression/test_reg_browser.py create mode 100644 tests/regression/test_reg_config.py create mode 100644 tests/regression/test_reg_content.py create mode 100644 tests/regression/test_reg_core_crawl.py create mode 100644 tests/regression/test_reg_deep_crawl.py create mode 100644 tests/regression/test_reg_domain_mapper.py create mode 100644 tests/regression/test_reg_edge_cases.py create mode 100644 tests/regression/test_reg_extraction.py create mode 100644 tests/regression/test_reg_utils.py create mode 100644 tests/releases/test_release_0.6.4.py create mode 100644 tests/releases/test_release_0.7.0.py create mode 100644 tests/test_arun_many.py create mode 100644 tests/test_async_logger_stderr.py create mode 100644 tests/test_bug_batch_1622_1786_1796.py create mode 100644 tests/test_cdp_changes.py create mode 100644 tests/test_cli_docs.py create mode 100644 tests/test_cloud_bugs_batch.py create mode 100644 tests/test_config_defaults.py create mode 100644 tests/test_config_matching_only.py create mode 100644 tests/test_config_selection.py create mode 100644 tests/test_docker.py create mode 100644 tests/test_docker_api_with_llm_provider.py create mode 100644 tests/test_eval_security_adversarial.py create mode 100644 tests/test_http_timeout_unit_1894.py create mode 100644 tests/test_issue_1043_mermaid_svg.py create mode 100644 tests/test_issue_1213_bm25_dedup.py create mode 100644 tests/test_issue_1370_1818_1762_1509.py create mode 100644 tests/test_issue_1484_css_selector.py create mode 100644 tests/test_issue_1594_mcp_sse.py create mode 100644 tests/test_issue_1611_llm_provider.py create mode 100644 tests/test_issue_1748_screenshot_scroll_delay.py create mode 100644 tests/test_issue_1750_screenshot_scan_full_page.py create mode 100644 tests/test_issue_1837_config_list.py create mode 100644 tests/test_issue_1842_browser_none.py create mode 100644 tests/test_issue_1848_logger_serialize.py create mode 100644 tests/test_issue_1850_mcp_sse.py create mode 100644 tests/test_link_extractor.py create mode 100644 tests/test_llm_extraction_parallel_issue_1055.py create mode 100644 tests/test_llm_simple_url.py create mode 100644 tests/test_llmtxt.py create mode 100644 tests/test_main.py create mode 100644 tests/test_markdown_generator_validation_1880.py create mode 100755 tests/test_memory_macos.py create mode 100644 tests/test_merge_head_data_scoring.py create mode 100644 tests/test_multi_config.py create mode 100644 tests/test_normalize_url.py create mode 100644 tests/test_pr_1290_1668.py create mode 100644 tests/test_pr_1435_redirected_status_code.py create mode 100644 tests/test_pr_1463_device_scale_factor.py create mode 100644 tests/test_pr_1795_1798_1734.py create mode 100644 tests/test_prefetch_integration.py create mode 100644 tests/test_prefetch_mode.py create mode 100644 tests/test_prefetch_regression.py create mode 100644 tests/test_preserve_https_for_internal_links.py create mode 100644 tests/test_pruning_preserve_whitelist_1900.py create mode 100644 tests/test_pyopenssl_security_fix.py create mode 100644 tests/test_pyopenssl_update.py create mode 100644 tests/test_raw_html_browser.py create mode 100644 tests/test_raw_html_edge_cases.py create mode 100644 tests/test_raw_html_redirected_url.py create mode 100644 tests/test_scraping_strategy.py create mode 100644 tests/test_source_sibling_selector.py create mode 100644 tests/test_table_gfm_compliance.py create mode 100644 tests/test_type_annotations.py create mode 100644 tests/test_virtual_scroll.py create mode 100644 tests/test_web_crawler.py create mode 100644 tests/unit/test_domain_mapper_unit.py create mode 100644 tests/unit/test_resource_filtering_config.py create mode 100644 tests/unit/test_sitemap_namespace_parsing.py create mode 100644 uv.lock diff --git a/.env.txt b/.env.txt new file mode 100644 index 0000000..07d266c --- /dev/null +++ b/.env.txt @@ -0,0 +1,4 @@ +GROQ_API_KEY = "YOUR_GROQ_API" +OPENAI_API_KEY = "YOUR_OPENAI_API" +ANTHROPIC_API_KEY = "YOUR_ANTHROPIC_API" +# You can add more API keys here \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0af13c5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +# Documentation +*.html linguist-documentation +docs/* linguist-documentation +docs/examples/* linguist-documentation +docs/md_v2/* linguist-documentation + +# Explicitly mark Python as the main language +*.py linguist-detectable=true +*.py linguist-language=Python + +# Exclude HTML from language statistics +*.html linguist-detectable=false diff --git a/.github/DISCUSSION_TEMPLATE/feature-requests.yml b/.github/DISCUSSION_TEMPLATE/feature-requests.yml new file mode 100644 index 0000000..24c2156 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/feature-requests.yml @@ -0,0 +1,59 @@ +title: "[Feature Request]: " +labels: ["⚙️ New"] +body: + - type: markdown + attributes: + value: | + Thank you for your interest in suggesting a new feature! Before you submit, please take a moment to check if already exists in + this discussions category to avoid duplicates. 😊 + + - type: textarea + id: needs_to_be_done + attributes: + label: What needs to be done? + description: Please describe the feature or functionality you'd like to see. + placeholder: "e.g., Return alt text along with images scraped from a webpages in Result" + validations: + required: true + + - type: textarea + id: problem_to_solve + attributes: + label: What problem does this solve? + description: Explain the pain point or issue this feature will help address. + placeholder: "e.g., Bypass Captchas added by cloudflare" + validations: + required: true + + - type: textarea + id: target_users + attributes: + label: Target users/beneficiaries + description: Who would benefit from this feature? (e.g., specific teams, developers, users, etc.) + placeholder: "e.g., Marketing teams, developers" + validations: + required: false + + - type: textarea + id: current_workarounds + attributes: + label: Current alternatives/workarounds + description: Are there any existing solutions or workarounds? How does this feature improve upon them? + placeholder: "e.g., Users manually select the css classes mapped to data fields to extract them" + validations: + required: false + + - type: markdown + attributes: + value: | + ### 💡 Implementation Ideas + + - type: textarea + id: proposed_approach + attributes: + label: Proposed approach + description: Share any ideas you have for how this feature could be implemented. Point out any challenges your foresee + and the success metrics for this feature + placeholder: "e.g., Implement a breadth first traversal algorithm for scraper" + validations: + required: false diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..0726594 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,7 @@ +# These are supported funding model platforms + +# GitHub Sponsors +github: unclecode + +# Custom links for enterprise inquiries (uncomment when ready) +# custom: ["https://crawl4ai.com/enterprise"] \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..0ff926b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,127 @@ +name: Bug Report +description: Report a bug with the Crawl4AI. +title: "[Bug]: " +labels: ["🐞 Bug","🩺 Needs Triage"] +body: + - type: input + id: crawl4ai_version + attributes: + label: crawl4ai version + description: Specify the version of crawl4ai you are using. + placeholder: "e.g., 2.0.0" + validations: + required: true + + - type: textarea + id: expected_behavior + attributes: + label: Expected Behavior + description: Describe what you expected to happen. + placeholder: "Provide a detailed explanation of the expected outcome." + validations: + required: true + + - type: textarea + id: current_behavior + attributes: + label: Current Behavior + description: Describe what is happening instead of the expected behavior. + placeholder: "Describe the actual result or issue you encountered." + validations: + required: true + + - type: dropdown + id: reproducible + attributes: + label: Is this reproducible? + description: Indicate whether this bug can be reproduced consistently. + options: + - "Yes" + - "No" + validations: + required: true + + - type: textarea + id: inputs + attributes: + label: Inputs Causing the Bug + description: Provide details about the inputs causing the issue. + placeholder: | + - URL(s): + - Settings used: + - Input data (if applicable): + render: bash + + - type: textarea + id: steps_to_reproduce + attributes: + label: Steps to Reproduce + description: Provide step-by-step instructions to reproduce the issue. + placeholder: | + 1. Go to... + 2. Click on... + 3. Observe the issue... + render: bash + + - type: textarea + id: code_snippets + attributes: + label: Code snippets + description: Provide code snippets(if any). Add comments as necessary + placeholder: print("Hello world") + render: python + + # Header Section with Title + - type: markdown + attributes: + value: | + ## Supporting Information + Please provide the following details to help us understand and resolve your issue. This will assist us in reproducing and diagnosing the problem + + - type: input + id: os + attributes: + label: OS + description: Please provide the operating system & distro where the issue occurs. + placeholder: "e.g., Windows, macOS, Linux" + validations: + required: true + + - type: input + id: python_version + attributes: + label: Python version + description: Specify the Python version being used. + placeholder: "e.g., 3.8.5" + validations: + required: true + + # Browser Field + - type: input + id: browser + attributes: + label: Browser + description: Provide the name of the browser you are using. + placeholder: "e.g., Chrome, Firefox, Safari" + validations: + required: false + + # Browser Version Field + - type: input + id: browser_version + attributes: + label: Browser version + description: Provide the version of the browser you are using. + placeholder: "e.g., 91.0.4472.124" + validations: + required: false + + # Error Logs Field (Text Area) + - type: textarea + id: error_logs + attributes: + label: Error logs & Screenshots (if applicable) + description: If you encountered any errors, please provide the error logs. Attach any relevant screenshots to help us understand the issue. + placeholder: "Paste error logs here and attach your screenshots" + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..5f877d1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Feature Requests + url: https://github.com/unclecode/crawl4ai/discussions/categories/feature-requests + about: "Suggest new features or enhancements for Crawl4AI" + - name: Forums - Q&A + url: https://github.com/unclecode/crawl4ai/discussions/categories/forums-q-a + about: "Ask questions or engage in general discussions about Crawl4AI" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..7366dad --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ +## Summary +Please include a summary of the change and/or which issues are fixed. + +eg: `Fixes #123` (Tag GitHub issue numbers in this format, so it automatically links the issues with your PR) + +## List of files changed and why +eg: quickstart.py - To update the example as per new changes + +## How Has This Been Tested? +Please describe the tests that you ran to verify your changes. + +## Checklist: + +- [ ] My code follows the style guidelines of this project +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] I have added/updated unit tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml new file mode 100644 index 0000000..23c7760 --- /dev/null +++ b/.github/workflows/docker-release.yml @@ -0,0 +1,100 @@ +name: Docker Release +on: + release: + types: [published] + push: + tags: + - 'docker-rebuild-v*' # Allow manual Docker rebuilds via tags + +jobs: + docker: + runs-on: ubuntu-latest + + steps: + - name: Free up disk space + run: | + echo "=== Disk space before cleanup ===" + df -h + + # Remove unnecessary tools and libraries (frees ~25GB) + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo rm -rf /usr/local/share/boost + sudo rm -rf /usr/share/swift + + # Clean apt cache + sudo apt-get clean + + echo "=== Disk space after cleanup ===" + df -h + + - name: Checkout code + uses: actions/checkout@v6 + + - name: Extract version from release or tag + id: get_version + run: | + if [ "${{ github.event_name }}" == "release" ]; then + # Triggered by release event + VERSION="${{ github.event.release.tag_name }}" + VERSION=${VERSION#v} # Remove 'v' prefix + else + # Triggered by docker-rebuild-v* tag + VERSION=${GITHUB_REF#refs/tags/docker-rebuild-v} + fi + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "Building Docker images for version: $VERSION" + + - name: Extract major and minor versions + id: versions + run: | + VERSION=${{ steps.get_version.outputs.VERSION }} + MAJOR=$(echo $VERSION | cut -d. -f1) + MINOR=$(echo $VERSION | cut -d. -f1-2) + echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT + echo "MINOR=$MINOR" >> $GITHUB_OUTPUT + echo "Semantic versions - Major: $MAJOR, Minor: $MINOR" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push Docker images + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: | + unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }} + unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }} + unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }} + unclecode/crawl4ai:latest + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Summary + run: | + echo "## 🐳 Docker Release Complete!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Published Images" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Platforms" >> $GITHUB_STEP_SUMMARY + echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY + echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 🚀 Pull Command" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/docs/ARCHITECTURE.md b/.github/workflows/docs/ARCHITECTURE.md new file mode 100644 index 0000000..aab2e8c --- /dev/null +++ b/.github/workflows/docs/ARCHITECTURE.md @@ -0,0 +1,917 @@ +# Workflow Architecture Documentation + +## Overview + +This document describes the technical architecture of the split release pipeline for Crawl4AI. + +--- + +## Architecture Diagram + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Developer │ +│ │ │ +│ ▼ │ +│ git tag v1.2.3 │ +│ git push --tags │ +└──────────────────────────────┬──────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ GitHub Repository │ +│ │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ Tag Event: v1.2.3 │ │ +│ └────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ release.yml (Release Pipeline) │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 1. Extract Version │ │ │ +│ │ │ v1.2.3 → 1.2.3 │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 2. Validate Version │ │ │ +│ │ │ Tag == __version__.py │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 3. Build Python Package │ │ │ +│ │ │ - Source dist (.tar.gz) │ │ │ +│ │ │ - Wheel (.whl) │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 4. Upload to PyPI │ │ │ +│ │ │ - Authenticate with token │ │ │ +│ │ │ - Upload dist/* │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 5. Create GitHub Release │ │ │ +│ │ │ - Tag: v1.2.3 │ │ │ +│ │ │ - Body: Install instructions │ │ │ +│ │ │ - Status: Published │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ Release Event: published (v1.2.3) │ │ +│ └────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ docker-release.yml (Docker Pipeline) │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 1. Extract Version from Release │ │ │ +│ │ │ github.event.release.tag_name → 1.2.3 │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 2. Parse Semantic Versions │ │ │ +│ │ │ 1.2.3 → Major: 1, Minor: 1.2 │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 3. Setup Multi-Arch Build │ │ │ +│ │ │ - Docker Buildx │ │ │ +│ │ │ - QEMU emulation │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 4. Authenticate Docker Hub │ │ │ +│ │ │ - Username: DOCKER_USERNAME │ │ │ +│ │ │ - Token: DOCKER_TOKEN │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 5. Build Multi-Arch Images │ │ │ +│ │ │ ┌────────────────┬────────────────┐ │ │ │ +│ │ │ │ linux/amd64 │ linux/arm64 │ │ │ │ +│ │ │ └────────────────┴────────────────┘ │ │ │ +│ │ │ Cache: GitHub Actions (type=gha) │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ │ ┌──────────────────────────────────────────────┐ │ │ +│ │ │ 6. Push to Docker Hub │ │ │ +│ │ │ Tags: │ │ │ +│ │ │ - unclecode/crawl4ai:1.2.3 │ │ │ +│ │ │ - unclecode/crawl4ai:1.2 │ │ │ +│ │ │ - unclecode/crawl4ai:1 │ │ │ +│ │ │ - unclecode/crawl4ai:latest │ │ │ +│ │ └──────────────────────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ External Services │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ PyPI │ │ Docker Hub │ │ GitHub │ │ +│ │ │ │ │ │ │ │ +│ │ crawl4ai │ │ unclecode/ │ │ Releases │ │ +│ │ 1.2.3 │ │ crawl4ai │ │ v1.2.3 │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Component Details + +### 1. Release Pipeline (release.yml) + +#### Purpose +Fast publication of Python package and GitHub release. + +#### Input +- **Trigger**: Git tag matching `v*` (excluding `test-v*`) +- **Example**: `v1.2.3` + +#### Processing Stages + +##### Stage 1: Version Extraction +```bash +Input: refs/tags/v1.2.3 +Output: VERSION=1.2.3 +``` + +**Implementation**: +```bash +TAG_VERSION=${GITHUB_REF#refs/tags/v} # Remove 'refs/tags/v' prefix +echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT +``` + +##### Stage 2: Version Validation +```bash +Input: TAG_VERSION=1.2.3 +Check: crawl4ai/__version__.py contains __version__ = "1.2.3" +Output: Pass/Fail +``` + +**Implementation**: +```bash +PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)") +if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then + exit 1 +fi +``` + +##### Stage 3: Package Build +```bash +Input: Source code + pyproject.toml +Output: dist/crawl4ai-1.2.3.tar.gz + dist/crawl4ai-1.2.3-py3-none-any.whl +``` + +**Implementation**: +```bash +python -m build +# Uses build backend defined in pyproject.toml +``` + +##### Stage 4: PyPI Upload +```bash +Input: dist/*.{tar.gz,whl} +Auth: PYPI_TOKEN +Output: Package published to PyPI +``` + +**Implementation**: +```bash +twine upload dist/* +# Environment: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} +``` + +##### Stage 5: GitHub Release Creation +```bash +Input: Tag: v1.2.3 + Body: Markdown content +Output: Published GitHub release +``` + +**Implementation**: +```yaml +uses: softprops/action-gh-release@v2 +with: + tag_name: v1.2.3 + name: Release v1.2.3 + body: | + Installation instructions and changelog + draft: false + prerelease: false +``` + +#### Output +- **PyPI Package**: https://pypi.org/project/crawl4ai/1.2.3/ +- **GitHub Release**: Published release on repository +- **Event**: `release.published` (triggers Docker workflow) + +#### Timeline +``` +0:00 - Tag pushed +0:01 - Checkout + Python setup +0:02 - Version validation +0:03 - Package build +0:04 - PyPI upload starts +0:06 - PyPI upload complete +0:07 - GitHub release created +0:08 - Workflow complete +``` + +--- + +### 2. Docker Release Pipeline (docker-release.yml) + +#### Purpose +Build and publish multi-architecture Docker images. + +#### Inputs + +##### Input 1: Release Event (Automatic) +```yaml +Event: release.published +Data: github.event.release.tag_name = "v1.2.3" +``` + +##### Input 2: Docker Rebuild Tag (Manual) +```yaml +Tag: docker-rebuild-v1.2.3 +``` + +#### Processing Stages + +##### Stage 1: Version Detection +```bash +# From release event: +VERSION = github.event.release.tag_name.strip("v") +# Result: "1.2.3" + +# From rebuild tag: +VERSION = GITHUB_REF.replace("refs/tags/docker-rebuild-v", "") +# Result: "1.2.3" +``` + +##### Stage 2: Semantic Version Parsing +```bash +Input: VERSION=1.2.3 +Output: MAJOR=1 + MINOR=1.2 + PATCH=3 (implicit) +``` + +**Implementation**: +```bash +MAJOR=$(echo $VERSION | cut -d. -f1) # Extract first component +MINOR=$(echo $VERSION | cut -d. -f1-2) # Extract first two components +``` + +##### Stage 3: Multi-Architecture Setup +```yaml +Setup: + - Docker Buildx (multi-platform builder) + - QEMU (ARM emulation on x86) + +Platforms: + - linux/amd64 (x86_64) + - linux/arm64 (aarch64) +``` + +**Architecture**: +``` +GitHub Runner (linux/amd64) + ├─ Buildx Builder + │ ├─ Native: Build linux/amd64 image + │ └─ QEMU: Emulate ARM to build linux/arm64 image + └─ Generate manifest list (points to both images) +``` + +##### Stage 4: Docker Hub Authentication +```bash +Input: DOCKER_USERNAME + DOCKER_TOKEN +Output: Authenticated Docker client +``` + +##### Stage 5: Build with Cache +```yaml +Cache Configuration: + cache-from: type=gha # Read from GitHub Actions cache + cache-to: type=gha,mode=max # Write all layers + +Cache Key Components: + - Workflow file path + - Branch name + - Architecture (amd64/arm64) +``` + +**Cache Hierarchy**: +``` +Cache Entry: main/docker-release.yml/linux-amd64 + ├─ Layer: sha256:abc123... (FROM python:3.12) + ├─ Layer: sha256:def456... (RUN apt-get update) + ├─ Layer: sha256:ghi789... (COPY requirements.txt) + ├─ Layer: sha256:jkl012... (RUN pip install) + └─ Layer: sha256:mno345... (COPY . /app) + +Cache Hit/Miss Logic: + - If layer input unchanged → cache hit → skip build + - If layer input changed → cache miss → rebuild + all subsequent layers +``` + +##### Stage 6: Tag Generation +```bash +Input: VERSION=1.2.3, MAJOR=1, MINOR=1.2 + +Output Tags: + - unclecode/crawl4ai:1.2.3 (exact version) + - unclecode/crawl4ai:1.2 (minor version) + - unclecode/crawl4ai:1 (major version) + - unclecode/crawl4ai:latest (latest stable) +``` + +**Tag Strategy**: +- All tags point to same image SHA +- Users can pin to desired stability level +- Pushing new version updates `1`, `1.2`, and `latest` automatically + +##### Stage 7: Push to Registry +```bash +For each tag: + For each platform (amd64, arm64): + Push image to Docker Hub + +Create manifest list: + Manifest: unclecode/crawl4ai:1.2.3 + ├─ linux/amd64: sha256:abc... + └─ linux/arm64: sha256:def... + +Docker CLI automatically selects correct platform on pull +``` + +#### Output +- **Docker Images**: 4 tags × 2 platforms = 8 image variants + 4 manifests +- **Docker Hub**: https://hub.docker.com/r/unclecode/crawl4ai/tags + +#### Timeline + +**Cold Cache (First Build)**: +``` +0:00 - Release event received +0:01 - Checkout + Buildx setup +0:02 - Docker Hub auth +0:03 - Start build (amd64) +0:08 - Complete amd64 build +0:09 - Start build (arm64) +0:14 - Complete arm64 build +0:15 - Generate manifests +0:16 - Push all tags +0:17 - Workflow complete +``` + +**Warm Cache (Code Change Only)**: +``` +0:00 - Release event received +0:01 - Checkout + Buildx setup +0:02 - Docker Hub auth +0:03 - Start build (amd64) - cache hit for layers 1-4 +0:04 - Complete amd64 build (only layer 5 rebuilt) +0:05 - Start build (arm64) - cache hit for layers 1-4 +0:06 - Complete arm64 build (only layer 5 rebuilt) +0:07 - Generate manifests +0:08 - Push all tags +0:09 - Workflow complete +``` + +--- + +## Data Flow + +### Version Information Flow + +``` +Developer + │ + ▼ +crawl4ai/__version__.py + __version__ = "1.2.3" + │ + ├─► Git Tag + │ v1.2.3 + │ │ + │ ▼ + │ release.yml + │ │ + │ ├─► Validation + │ │ ✓ Match + │ │ + │ ├─► PyPI Package + │ │ crawl4ai==1.2.3 + │ │ + │ └─► GitHub Release + │ v1.2.3 + │ │ + │ ▼ + │ docker-release.yml + │ │ + │ └─► Docker Tags + │ 1.2.3, 1.2, 1, latest + │ + └─► Package Metadata + pyproject.toml + version = "1.2.3" +``` + +### Secrets Flow + +``` +GitHub Secrets (Encrypted at Rest) + │ + ├─► PYPI_TOKEN + │ │ + │ ▼ + │ release.yml + │ │ + │ ▼ + │ TWINE_PASSWORD env var (masked in logs) + │ │ + │ ▼ + │ PyPI API (HTTPS) + │ + ├─► DOCKER_USERNAME + │ │ + │ ▼ + │ docker-release.yml + │ │ + │ ▼ + │ docker/login-action (masked in logs) + │ │ + │ ▼ + │ Docker Hub API (HTTPS) + │ + └─► DOCKER_TOKEN + │ + ▼ + docker-release.yml + │ + ▼ + docker/login-action (masked in logs) + │ + ▼ + Docker Hub API (HTTPS) +``` + +### Artifact Flow + +``` +Source Code + │ + ├─► release.yml + │ │ + │ ▼ + │ python -m build + │ │ + │ ├─► crawl4ai-1.2.3.tar.gz + │ │ │ + │ │ ▼ + │ │ PyPI Storage + │ │ │ + │ │ ▼ + │ │ pip install crawl4ai + │ │ + │ └─► crawl4ai-1.2.3-py3-none-any.whl + │ │ + │ ▼ + │ PyPI Storage + │ │ + │ ▼ + │ pip install crawl4ai + │ + └─► docker-release.yml + │ + ▼ + docker build + │ + ├─► Image: linux/amd64 + │ │ + │ └─► Docker Hub + │ unclecode/crawl4ai:1.2.3-amd64 + │ + └─► Image: linux/arm64 + │ + └─► Docker Hub + unclecode/crawl4ai:1.2.3-arm64 +``` + +--- + +## State Machines + +### Release Pipeline State Machine + +``` +┌─────────┐ +│ START │ +└────┬────┘ + │ + ▼ +┌──────────────┐ +│ Extract │ +│ Version │ +└──────┬───────┘ + │ + ▼ +┌──────────────┐ ┌─────────┐ +│ Validate │─────►│ FAILED │ +│ Version │ No │ (Exit 1)│ +└──────┬───────┘ └─────────┘ + │ Yes + ▼ +┌──────────────┐ +│ Build │ +│ Package │ +└──────┬───────┘ + │ + ▼ +┌──────────────┐ ┌─────────┐ +│ Upload │─────►│ FAILED │ +│ to PyPI │ Error│ (Exit 1)│ +└──────┬───────┘ └─────────┘ + │ Success + ▼ +┌──────────────┐ +│ Create │ +│ GH Release │ +└──────┬───────┘ + │ + ▼ +┌──────────────┐ +│ SUCCESS │ +│ (Emit Event) │ +└──────────────┘ +``` + +### Docker Pipeline State Machine + +``` +┌─────────┐ +│ START │ +│ (Event) │ +└────┬────┘ + │ + ▼ +┌──────────────┐ +│ Detect │ +│ Version │ +│ Source │ +└──────┬───────┘ + │ + ▼ +┌──────────────┐ +│ Parse │ +│ Semantic │ +│ Versions │ +└──────┬───────┘ + │ + ▼ +┌──────────────┐ ┌─────────┐ +│ Authenticate │─────►│ FAILED │ +│ Docker Hub │ Error│ (Exit 1)│ +└──────┬───────┘ └─────────┘ + │ Success + ▼ +┌──────────────┐ +│ Build │ +│ amd64 │ +└──────┬───────┘ + │ + ▼ +┌──────────────┐ ┌─────────┐ +│ Build │─────►│ FAILED │ +│ arm64 │ Error│ (Exit 1)│ +└──────┬───────┘ └─────────┘ + │ Success + ▼ +┌──────────────┐ +│ Push All │ +│ Tags │ +└──────┬───────┘ + │ + ▼ +┌──────────────┐ +│ SUCCESS │ +└──────────────┘ +``` + +--- + +## Security Architecture + +### Threat Model + +#### Threats Mitigated + +1. **Secret Exposure** + - Mitigation: GitHub Actions secret masking + - Evidence: Secrets never appear in logs + +2. **Unauthorized Package Upload** + - Mitigation: Scoped PyPI tokens + - Evidence: Token limited to `crawl4ai` project + +3. **Man-in-the-Middle** + - Mitigation: HTTPS for all API calls + - Evidence: PyPI, Docker Hub, GitHub all use TLS + +4. **Supply Chain Tampering** + - Mitigation: Immutable artifacts, content checksums + - Evidence: PyPI stores SHA256, Docker uses content-addressable storage + +#### Trust Boundaries + +``` +┌─────────────────────────────────────────┐ +│ Trusted Zone │ +│ ┌────────────────────────────────┐ │ +│ │ GitHub Actions Runner │ │ +│ │ - Ephemeral VM │ │ +│ │ - Isolated environment │ │ +│ │ - Access to secrets │ │ +│ └────────────────────────────────┘ │ +│ │ │ +│ │ HTTPS (TLS 1.2+) │ +│ ▼ │ +└─────────────────────────────────────────┘ + │ + ┌────────────┼────────────┐ + │ │ │ + ▼ ▼ ▼ +┌────────┐ ┌─────────┐ ┌──────────┐ +│ PyPI │ │ Docker │ │ GitHub │ +│ API │ │ Hub │ │ API │ +└────────┘ └─────────┘ └──────────┘ + External External External + Service Service Service +``` + +### Secret Management + +#### Secret Lifecycle + +``` +Creation (Developer) + │ + ├─► PyPI: Create API token (scoped to project) + ├─► Docker Hub: Create access token (read/write) + │ + ▼ +Storage (GitHub) + │ + ├─► Encrypted at rest (AES-256) + ├─► Access controlled (repo-scoped) + │ + ▼ +Usage (Workflow) + │ + ├─► Injected as env vars + ├─► Masked in logs (GitHub redacts on output) + ├─► Never persisted to disk (in-memory only) + │ + ▼ +Transmission (API Call) + │ + ├─► HTTPS only + ├─► TLS 1.2+ with strong ciphers + │ + ▼ +Rotation (Manual) + │ + └─► Regenerate on PyPI/Docker Hub + Update GitHub secret +``` + +--- + +## Performance Characteristics + +### Release Pipeline Performance + +| Metric | Value | Notes | +|--------|-------|-------| +| Cold start | ~2-3 min | First run on new runner | +| Warm start | ~2-3 min | Minimal caching benefit | +| PyPI upload | ~30-60 sec | Network-bound | +| Package build | ~30 sec | CPU-bound | +| Parallelization | None | Sequential by design | + +### Docker Pipeline Performance + +| Metric | Cold Cache | Warm Cache (code) | Warm Cache (deps) | +|--------|-----------|-------------------|-------------------| +| Total time | 10-15 min | 1-2 min | 3-5 min | +| amd64 build | 5-7 min | 30-60 sec | 1-2 min | +| arm64 build | 5-7 min | 30-60 sec | 1-2 min | +| Push time | 1-2 min | 30 sec | 30 sec | +| Cache hit rate | 0% | 85% | 60% | + +### Cache Performance Model + +```python +def estimate_build_time(changes): + base_time = 60 # seconds (setup + push) + + if "Dockerfile" in changes: + return base_time + (10 * 60) # Full rebuild: ~11 min + elif "requirements.txt" in changes: + return base_time + (3 * 60) # Deps rebuild: ~4 min + elif any(f.endswith(".py") for f in changes): + return base_time + 60 # Code only: ~2 min + else: + return base_time # No changes: ~1 min +``` + +--- + +## Scalability Considerations + +### Current Limits + +| Resource | Limit | Impact | +|----------|-------|--------| +| Workflow concurrency | 20 (default) | Max 20 releases in parallel | +| Artifact storage | 500 MB/artifact | PyPI packages small (<10 MB) | +| Cache storage | 10 GB/repo | Docker layers fit comfortably | +| Workflow run time | 6 hours | Plenty of headroom | + +### Scaling Strategies + +#### Horizontal Scaling (Multiple Repos) +``` +crawl4ai (main) + ├─ release.yml + └─ docker-release.yml + +crawl4ai-plugins (separate) + ├─ release.yml + └─ docker-release.yml + +Each repo has independent: + - Secrets + - Cache (10 GB each) + - Concurrency limits (20 each) +``` + +#### Vertical Scaling (Larger Runners) +```yaml +jobs: + docker: + runs-on: ubuntu-latest-8-cores # GitHub-hosted larger runner + # 4x faster builds for CPU-bound layers +``` + +--- + +## Disaster Recovery + +### Failure Scenarios + +#### Scenario 1: Release Pipeline Fails + +**Failure Point**: PyPI upload fails (network error) + +**State**: +- ✓ Version validated +- ✓ Package built +- ✗ PyPI upload +- ✗ GitHub release + +**Recovery**: +```bash +# Manual upload +twine upload dist/* + +# Retry workflow (re-run from GitHub Actions UI) +``` + +**Prevention**: Add retry logic to PyPI upload + +#### Scenario 2: Docker Pipeline Fails + +**Failure Point**: ARM build fails (dependency issue) + +**State**: +- ✓ PyPI published +- ✓ GitHub release created +- ✓ amd64 image built +- ✗ arm64 image build + +**Recovery**: +```bash +# Fix Dockerfile +git commit -am "fix: ARM build dependency" + +# Trigger rebuild +git tag docker-rebuild-v1.2.3 +git push origin docker-rebuild-v1.2.3 +``` + +**Impact**: PyPI package available, only Docker ARM users affected + +#### Scenario 3: Partial Release + +**Failure Point**: GitHub release creation fails + +**State**: +- ✓ PyPI published +- ✗ GitHub release +- ✗ Docker images + +**Recovery**: +```bash +# Create release manually +gh release create v1.2.3 \ + --title "Release v1.2.3" \ + --notes "..." + +# This triggers docker-release.yml automatically +``` + +--- + +## Monitoring and Observability + +### Metrics to Track + +#### Release Pipeline +- Success rate (target: >99%) +- Duration (target: <3 min) +- PyPI upload time (target: <60 sec) + +#### Docker Pipeline +- Success rate (target: >95%) +- Duration (target: <15 min cold, <2 min warm) +- Cache hit rate (target: >80% for code changes) + +### Alerting + +**Critical Alerts**: +- Release pipeline failure (blocks release) +- PyPI authentication failure (expired token) + +**Warning Alerts**: +- Docker build >15 min (performance degradation) +- Cache hit rate <50% (cache issue) + +### Logging + +**GitHub Actions Logs**: +- Retention: 90 days +- Downloadable: Yes +- Searchable: Limited + +**Recommended External Logging**: +```yaml +- name: Send logs to external service + if: failure() + run: | + curl -X POST https://logs.example.com/api/v1/logs \ + -H "Content-Type: application/json" \ + -d "{\"workflow\": \"${{ github.workflow }}\", \"status\": \"failed\"}" +``` + +--- + +## Future Enhancements + +### Planned Improvements + +1. **Automated Changelog Generation** + - Use conventional commits + - Generate CHANGELOG.md automatically + +2. **Pre-release Testing** + - Test builds on `test-v*` tags + - Upload to TestPyPI + +3. **Notification System** + - Slack/Discord notifications on release + - Email on failure + +4. **Performance Optimization** + - Parallel Docker builds (amd64 + arm64 simultaneously) + - Persistent runners for better caching + +5. **Enhanced Validation** + - Smoke tests after PyPI upload + - Container security scanning + +--- + +## References + +- [GitHub Actions Architecture](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions) +- [Docker Build Cache](https://docs.docker.com/build/cache/) +- [PyPI API Documentation](https://warehouse.pypa.io/api-reference/) + +--- + +**Last Updated**: 2025-01-21 +**Version**: 2.0 diff --git a/.github/workflows/docs/README.md b/.github/workflows/docs/README.md new file mode 100644 index 0000000..e96a4c5 --- /dev/null +++ b/.github/workflows/docs/README.md @@ -0,0 +1,1029 @@ +# GitHub Actions Workflows Documentation + +## Table of Contents + +1. [Overview](#overview) +2. [Workflow Architecture](#workflow-architecture) +3. [Workflows](#workflows) + - [Release Pipeline](#release-pipeline) + - [Docker Release](#docker-release) +4. [Usage Guide](#usage-guide) +5. [Secrets Configuration](#secrets-configuration) +6. [Troubleshooting](#troubleshooting) +7. [Advanced Topics](#advanced-topics) + +--- + +## Overview + +This repository uses a **split release pipeline** architecture to optimize release times and provide flexibility. The release process is divided into two independent workflows: + +1. **Release Pipeline** (`release.yml`) - Fast PyPI and GitHub release publication +2. **Docker Release** (`docker-release.yml`) - Multi-architecture Docker image builds with caching + +### Why Split Workflows? + +**Problem**: Docker multi-architecture builds take 10-15 minutes, blocking quick package releases. + +**Solution**: Separate Docker builds into an independent workflow that runs in parallel. + +**Benefits**: +- ✅ PyPI package available in ~2-3 minutes +- ✅ GitHub release published immediately +- ✅ Docker images build in parallel (non-blocking) +- ✅ Can rebuild Docker images independently +- ✅ Faster subsequent builds with layer caching + +--- + +## Workflow Architecture + +``` +Tag Push (v1.2.3) + │ + ├─► Release Pipeline (release.yml) + │ ├─ Version validation + │ ├─ Build Python package + │ ├─ Upload to PyPI ✓ + │ └─ Create GitHub Release ✓ + │ │ + │ └─► Triggers Docker Release (docker-release.yml) + │ ├─ Build multi-arch images + │ ├─ Use GitHub Actions cache + │ └─ Push to Docker Hub ✓ + │ + └─► Total Time: + - PyPI/GitHub: 2-3 minutes + - Docker: 1-15 minutes (parallel) +``` + +### Event Flow + +```mermaid +graph TD + A[Push tag v1.2.3] --> B[release.yml triggered] + B --> C{Version Check} + C -->|Match| D[Build Package] + C -->|Mismatch| E[❌ Fail - Update __version__.py] + D --> F[Upload to PyPI] + F --> G[Create GitHub Release] + G --> H[docker-release.yml triggered] + H --> I[Build Docker Images] + I --> J[Push to Docker Hub] + + K[Push tag docker-rebuild-v1.2.3] --> H +``` + +--- + +## Workflows + +### Release Pipeline + +**File**: `.github/workflows/release.yml` + +#### Trigger + +```yaml +on: + push: + tags: + - 'v*' # Matches: v1.2.3, v2.0.0, etc. + - '!test-v*' # Excludes: test-v1.2.3 +``` + +#### Jobs & Steps + +##### 1. Version Extraction +```bash +# Extracts version from tag +v1.2.3 → 1.2.3 +``` + +##### 2. Version Consistency Check +Validates that the git tag matches `crawl4ai/__version__.py`: + +```python +# crawl4ai/__version__.py must contain: +__version__ = "1.2.3" # Must match tag v1.2.3 +``` + +**Failure Example**: +``` +Tag version: 1.2.3 +Package version: 1.2.2 +❌ Version mismatch! Please update crawl4ai/__version__.py +``` + +##### 3. Package Build +- Installs build dependencies (`build`, `twine`) +- Builds source distribution and wheel: `python -m build` +- Validates package: `twine check dist/*` + +##### 4. PyPI Upload +```bash +twine upload dist/* +# Uploads to: https://pypi.org/project/crawl4ai/ +``` + +**Environment Variables**: +- `TWINE_USERNAME`: `__token__` (PyPI API token authentication) +- `TWINE_PASSWORD`: `${{ secrets.PYPI_TOKEN }}` + +##### 5. GitHub Release Creation +Creates a release with: +- Tag: `v1.2.3` +- Title: `Release v1.2.3` +- Body: Installation instructions + changelog link +- Status: Published (not draft) + +**Note**: The release body includes a link to the Docker workflow status, informing users that Docker images are building. + +##### 6. Summary Report +Generates a GitHub Actions summary with: +- PyPI package URL and version +- GitHub release URL +- Link to Docker workflow status + +#### Output Artifacts + +| Artifact | Location | Time | +|----------|----------|------| +| PyPI Package | https://pypi.org/project/crawl4ai/ | ~2-3 min | +| GitHub Release | Repository releases page | ~2-3 min | + +--- + +### Docker Release + +**File**: `.github/workflows/docker-release.yml` + +#### Triggers + +This workflow has **two independent triggers**: + +##### 1. Automatic Trigger (Release Event) +```yaml +on: + release: + types: [published] +``` + +Triggers when `release.yml` publishes a GitHub release. + +##### 2. Manual Trigger (Docker Rebuild Tag) +```yaml +on: + push: + tags: + - 'docker-rebuild-v*' +``` + +Allows rebuilding Docker images without creating a new release. + +**Use case**: Fix Dockerfile, rebuild images for existing version. + +#### Jobs & Steps + +##### 1. Version Detection +Intelligently detects version from either trigger: + +```bash +# From release event: +github.event.release.tag_name → v1.2.3 → 1.2.3 + +# From docker-rebuild tag: +docker-rebuild-v1.2.3 → 1.2.3 +``` + +##### 2. Semantic Version Extraction +```bash +VERSION=1.2.3 +MAJOR=1 # First component +MINOR=1.2 # First two components +``` + +Used for Docker tag variations. + +##### 3. Docker Buildx Setup +Configures multi-architecture build support: +- Platform: linux/amd64, linux/arm64 +- Builder: Buildx with QEMU emulation + +##### 4. Docker Hub Authentication +```yaml +username: ${{ secrets.DOCKER_USERNAME }} +password: ${{ secrets.DOCKER_TOKEN }} +``` + +##### 5. Multi-Architecture Build & Push + +**Docker Tags Created**: +``` +unclecode/crawl4ai:1.2.3 # Exact version +unclecode/crawl4ai:1.2 # Minor version +unclecode/crawl4ai:1 # Major version +unclecode/crawl4ai:latest # Latest stable +``` + +**Platforms**: +- `linux/amd64` (x86_64 - Intel/AMD processors) +- `linux/arm64` (ARM processors - Apple Silicon, AWS Graviton) + +**Caching Configuration**: +```yaml +cache-from: type=gha # Read from GitHub Actions cache +cache-to: type=gha,mode=max # Write all layers to cache +``` + +##### 6. Summary Report +Generates a summary with: +- Published image tags +- Supported platforms +- Pull command example + +#### Docker Layer Caching + +**How It Works**: + +Docker builds images in layers: +```dockerfile +FROM python:3.12 # Layer 1 (base image) +RUN apt-get update # Layer 2 (system packages) +COPY requirements.txt . # Layer 3 (dependency file) +RUN pip install -r ... # Layer 4 (Python packages) +COPY . . # Layer 5 (application code) +``` + +**Cache Behavior**: + +| Change Type | Cached Layers | Rebuild Time | +|-------------|---------------|--------------| +| No changes | 1-5 | ~30-60 sec | +| Code only | 1-4 | ~1-2 min | +| Dependencies | 1-3 | ~3-5 min | +| Dockerfile | None | ~10-15 min | + +**Cache Storage**: +- Location: GitHub Actions cache +- Limit: 10GB per repository +- Retention: 7 days for unused cache +- Cleanup: Automatic (LRU eviction) + +**Cache Efficiency Example**: + +```bash +# First build (v1.0.0) +Build time: 12m 34s +Cache: 0% (cold start) + +# Second build (v1.0.1 - code change only) +Build time: 1m 47s +Cache: 85% hit rate +Cached: Base image, system packages, Python dependencies + +# Third build (v1.0.2 - dependency update) +Build time: 4m 12s +Cache: 60% hit rate +Cached: Base image, system packages +``` + +#### Output Artifacts + +| Artifact | Location | Tags | Time | +|----------|----------|------|------| +| Docker Images | Docker Hub | 4 tags | 1-15 min | + +**Docker Hub URL**: https://hub.docker.com/r/unclecode/crawl4ai + +--- + +## Usage Guide + +### Standard Release Process + +#### Step 1: Update Version + +Edit `crawl4ai/__version__.py`: +```python +__version__ = "1.2.3" +``` + +#### Step 2: Commit and Tag + +```bash +git add crawl4ai/__version__.py +git commit -m "chore: bump version to 1.2.3" +git tag v1.2.3 +git push origin main +git push origin v1.2.3 +``` + +#### Step 3: Monitor Workflows + +**Release Pipeline** (~2-3 minutes): +``` +✓ Version check passed +✓ Package built +✓ Uploaded to PyPI +✓ GitHub release created +``` + +**Docker Release** (~1-15 minutes, runs in parallel): +``` +✓ Images built for amd64, arm64 +✓ Pushed 4 tags to Docker Hub +✓ Cache updated +``` + +#### Step 4: Verify Deployment + +```bash +# Check PyPI +pip install crawl4ai==1.2.3 + +# Check Docker +docker pull unclecode/crawl4ai:1.2.3 +docker run unclecode/crawl4ai:1.2.3 --version +``` + +### Manual Docker Rebuild + +**When to Use**: +- Dockerfile fixed after release +- Security patch in base image +- Rebuild needed without new version + +**Process**: + +```bash +# Rebuild Docker images for existing version 1.2.3 +git tag docker-rebuild-v1.2.3 +git push origin docker-rebuild-v1.2.3 +``` + +This triggers **only** `docker-release.yml`, not `release.yml`. + +**Result**: +- Docker images rebuilt with same version tag +- PyPI package unchanged +- GitHub release unchanged + +### Rollback Procedure + +#### Rollback PyPI Package +PyPI does not allow re-uploading the same version. Instead: + +```bash +# Publish a patch version +git tag v1.2.4 +git push origin v1.2.4 +``` + +Then update documentation to recommend the new version. + +#### Rollback Docker Images + +```bash +# Option 1: Rebuild with fixed code +git tag docker-rebuild-v1.2.3 +git push origin docker-rebuild-v1.2.3 + +# Option 2: Manually retag in Docker Hub (advanced) +# Not recommended - use git tags for traceability +``` + +--- + +## Secrets Configuration + +### Required Secrets + +Configure these in: **Repository Settings → Secrets and variables → Actions** + +#### 1. PYPI_TOKEN + +**Purpose**: Authenticate with PyPI for package uploads + +**How to Create**: +1. Go to https://pypi.org/manage/account/token/ +2. Create token with scope: "Entire account" or "Project: crawl4ai" +3. Copy token (starts with `pypi-`) +4. Add to GitHub secrets as `PYPI_TOKEN` + +**Format**: +``` +pypi-AgEIcHlwaS5vcmcCJGQ4M2Y5YTM5LWRjMzUtNGY3MS04ZmMwLWVhNzA5MjkzMjk5YQACKl... +``` + +#### 2. DOCKER_USERNAME + +**Purpose**: Docker Hub username for authentication + +**Value**: Your Docker Hub username (e.g., `unclecode`) + +#### 3. DOCKER_TOKEN + +**Purpose**: Docker Hub access token for authentication + +**How to Create**: +1. Go to https://hub.docker.com/settings/security +2. Click "New Access Token" +3. Name: `github-actions-crawl4ai` +4. Permissions: Read, Write, Delete +5. Copy token +6. Add to GitHub secrets as `DOCKER_TOKEN` + +**Format**: +``` +dckr_pat_1a2b3c4d5e6f7g8h9i0j +``` + +### Built-in Secrets + +#### GITHUB_TOKEN + +**Purpose**: Create GitHub releases + +**Note**: Automatically provided by GitHub Actions. No configuration needed. + +**Permissions**: Configured in workflow file: +```yaml +permissions: + contents: write # Required for creating releases +``` + +--- + +## Troubleshooting + +### Version Mismatch Error + +**Error**: +``` +❌ Version mismatch! Tag: 1.2.3, Package: 1.2.2 +Please update crawl4ai/__version__.py to match the tag version +``` + +**Cause**: Git tag doesn't match `__version__` in `crawl4ai/__version__.py` + +**Fix**: +```bash +# Option 1: Update __version__.py and re-tag +vim crawl4ai/__version__.py # Change to 1.2.3 +git add crawl4ai/__version__.py +git commit -m "fix: update version to 1.2.3" +git tag -d v1.2.3 # Delete local tag +git push --delete origin v1.2.3 # Delete remote tag +git tag v1.2.3 # Create new tag +git push origin main +git push origin v1.2.3 + +# Option 2: Use correct tag +git tag v1.2.2 # Match existing __version__ +git push origin v1.2.2 +``` + +### PyPI Upload Failure + +**Error**: +``` +HTTPError: 403 Forbidden +``` + +**Causes & Fixes**: + +1. **Invalid Token**: + - Verify `PYPI_TOKEN` in GitHub secrets + - Ensure token hasn't expired + - Regenerate token on PyPI + +2. **Version Already Exists**: + ``` + HTTPError: 400 File already exists + ``` + - PyPI doesn't allow re-uploading same version + - Increment version number and retry + +3. **Package Name Conflict**: + - Ensure you own the `crawl4ai` package on PyPI + - Check token scope includes this project + +### Docker Build Failure + +**Error**: +``` +failed to solve: process "/bin/sh -c ..." did not complete successfully +``` + +**Debug Steps**: + +1. **Check Build Logs**: + - Go to Actions tab → Docker Release workflow + - Expand "Build and push Docker images" step + - Look for specific error + +2. **Test Locally**: + ```bash + docker build -t crawl4ai:test . + ``` + +3. **Common Issues**: + + **Dependency installation fails**: + ```dockerfile + # Check requirements.txt is valid + # Ensure all packages are available + ``` + + **Architecture-specific issues**: + ```bash + # Test both platforms locally (if on Mac with Apple Silicon) + docker buildx build --platform linux/amd64,linux/arm64 -t test . + ``` + +4. **Cache Issues**: + ```bash + # Clear cache by pushing a tag with different content + # Or wait 7 days for automatic cache eviction + ``` + +### Docker Authentication Failure + +**Error**: +``` +Error: Cannot perform an interactive login from a non TTY device +``` + +**Cause**: Docker Hub credentials invalid + +**Fix**: +1. Verify `DOCKER_USERNAME` is correct +2. Regenerate `DOCKER_TOKEN` on Docker Hub +3. Update secret in GitHub + +### Docker Release Not Triggering + +**Issue**: Pushed tag `v1.2.3`, but `docker-release.yml` didn't run + +**Causes**: + +1. **Release Not Published**: + - Check if `release.yml` completed successfully + - Verify GitHub release is published (not draft) + +2. **Workflow File Syntax Error**: + ```bash + # Validate YAML syntax + yamllint .github/workflows/docker-release.yml + ``` + +3. **Workflow Not on Default Branch**: + - Workflow files must be on `main` branch + - Check if `.github/workflows/docker-release.yml` exists on `main` + +**Debug**: +```bash +# Check workflow files +git ls-tree main .github/workflows/ + +# Check GitHub Actions tab for workflow runs +``` + +### Cache Not Working + +**Issue**: Every build takes 10-15 minutes despite using cache + +**Causes**: + +1. **Cache Scope**: + - Cache is per-branch and per-workflow + - First build on new branch is always cold + +2. **Dockerfile Changes**: + - Any change invalidates subsequent layers + - Optimize Dockerfile layer order (stable → volatile) + +3. **Base Image Updates**: + - `FROM python:3.12` pulls latest monthly + - Pin to specific digest for stable cache + +**Optimization**: +```dockerfile +# Good: Stable layers first +FROM python:3.12 +RUN apt-get update && apt-get install -y ... +COPY requirements.txt . +RUN pip install -r requirements.txt +COPY . . + +# Bad: Volatile layers first (breaks cache often) +FROM python:3.12 +COPY . . +RUN pip install -r requirements.txt +``` + +--- + +## Advanced Topics + +### Multi-Architecture Build Details + +#### Platform Support + +| Platform | Architecture | Use Cases | +|----------|-------------|-----------| +| linux/amd64 | x86_64 | AWS EC2, GCP, Azure, Traditional servers | +| linux/arm64 | aarch64 | Apple Silicon, AWS Graviton, Raspberry Pi | + +#### Build Process + +```bash +# Buildx uses QEMU to emulate different architectures +docker buildx create --use # Create builder +docker buildx build --platform linux/amd64,linux/arm64 ... +``` + +**Under the Hood**: +1. For each platform: + - Spawn QEMU emulator + - Execute Dockerfile instructions + - Generate platform-specific image +2. Create manifest list (multi-arch index) +3. Push all variants + manifest to registry + +**Pull Behavior**: +```bash +# Docker automatically selects correct platform +docker pull unclecode/crawl4ai:latest + +# On M1 Mac: Pulls arm64 variant +# On Intel Linux: Pulls amd64 variant + +# Force specific platform +docker pull --platform linux/amd64 unclecode/crawl4ai:latest +``` + +### Semantic Versioning Strategy + +#### Tag Scheme + +``` +v1.2.3 + │ │ │ + │ │ └─ Patch: Bug fixes, no API changes + │ └─── Minor: New features, backward compatible + └───── Major: Breaking changes +``` + +#### Docker Tag Mapping + +| Git Tag | Docker Tags Created | Use Case | +|---------|---------------------|----------| +| v1.2.3 | 1.2.3, 1.2, 1, latest | Full version chain | +| v2.0.0 | 2.0.0, 2.0, 2, latest | Major version bump | + +**Example Evolution**: + +```bash +# Release v1.0.0 +Tags: 1.0.0, 1.0, 1, latest + +# Release v1.1.0 +Tags: 1.1.0, 1.1, 1, latest +# Note: 1.0 still exists, but 1 and latest now point to 1.1.0 + +# Release v1.2.0 +Tags: 1.2.0, 1.2, 1, latest +# Note: 1.0 and 1.1 still exist, but 1 and latest now point to 1.2.0 + +# Release v2.0.0 +Tags: 2.0.0, 2.0, 2, latest +# Note: All v1.x tags still exist, but latest now points to 2.0.0 +``` + +**User Pinning Strategies**: + +```bash +# Maximum stability (never updates) +docker pull unclecode/crawl4ai:1.2.3 + +# Get patch updates only +docker pull unclecode/crawl4ai:1.2 + +# Get minor updates (features, bug fixes) +docker pull unclecode/crawl4ai:1 + +# Always get latest (potentially breaking) +docker pull unclecode/crawl4ai:latest +``` + +### Cache Optimization Strategies + +#### 1. Layer Order Optimization + +```dockerfile +# BEFORE (cache breaks often) +FROM python:3.12 +COPY . /app # Changes every commit +RUN pip install -r requirements.txt +RUN apt-get install -y ffmpeg + +# AFTER (cache-optimized) +FROM python:3.12 +RUN apt-get update && apt-get install -y ffmpeg # Rarely changes +COPY requirements.txt /app/requirements.txt # Changes occasionally +RUN pip install -r /app/requirements.txt +COPY . /app # Changes every commit +``` + +#### 2. Multi-Stage Builds + +```dockerfile +# Build stage (cached separately) +FROM python:3.12 as builder +COPY requirements.txt . +RUN pip install --user -r requirements.txt + +# Runtime stage +FROM python:3.12-slim +COPY --from=builder /root/.local /root/.local +COPY . /app +ENV PATH=/root/.local/bin:$PATH +``` + +**Benefits**: +- Builder stage cached independently +- Runtime image smaller +- Faster rebuilds + +#### 3. Dependency Caching + +```dockerfile +# Cache pip packages +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install -r requirements.txt + +# Cache apt packages +RUN --mount=type=cache,target=/var/cache/apt \ + apt-get update && apt-get install -y ... +``` + +**Note**: Requires BuildKit (enabled by default in GitHub Actions) + +#### 4. Base Image Pinning + +```dockerfile +# VOLATILE (updates monthly, breaks cache) +FROM python:3.12 + +# STABLE (fixed digest, cache preserved) +FROM python:3.12@sha256:8c5e5c77e7b9e44a6f0e3b9e8f5e5c77e7b9e44a6f0e3b9e8f5e5c77e7b9e44a +``` + +Find digest: +```bash +docker pull python:3.12 +docker inspect python:3.12 | grep -A 2 RepoDigests +``` + +### Workflow Security Best Practices + +#### 1. Secret Handling + +**Never**: +```yaml +# DON'T: Hardcode secrets +run: echo "my-secret-token" | docker login + +# DON'T: Log secrets +run: echo "Token is ${{ secrets.PYPI_TOKEN }}" +``` + +**Always**: +```yaml +# DO: Use environment variables +env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} +run: twine upload dist/* + +# DO: Use action inputs (masked automatically) +uses: docker/login-action@v3 +with: + password: ${{ secrets.DOCKER_TOKEN }} +``` + +#### 2. Permission Minimization + +```yaml +# Specific permissions only +permissions: + contents: write # Only what's needed + # NOT: permissions: write-all +``` + +#### 3. Dependency Pinning + +```yaml +# DON'T: Use floating versions +uses: actions/checkout@v4 + +# DO: Pin to SHA (immutable) +uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 +``` + +#### 4. Token Scoping + +**PyPI Token**: +- Scope: Project-specific (`crawl4ai` only) +- Not: Account-wide access + +**Docker Token**: +- Permissions: Read, Write (not Delete unless needed) +- Expiration: Set to 1 year, rotate regularly + +### Monitoring and Observability + +#### GitHub Actions Metrics + +**Available in Actions tab**: +- Workflow run duration +- Success/failure rates +- Cache hit rates +- Artifact sizes + +#### Custom Metrics + +Add to workflow summary: +```yaml +- name: Build Metrics + run: | + echo "## Build Metrics" >> $GITHUB_STEP_SUMMARY + echo "- Duration: $(date -u -d @$SECONDS +%T)" >> $GITHUB_STEP_SUMMARY + echo "- Cache hit rate: 85%" >> $GITHUB_STEP_SUMMARY +``` + +#### External Monitoring + +**Webhooks**: Configure in Settings → Webhooks +```json +{ + "events": ["workflow_run"], + "url": "https://your-monitoring-service.com/webhook" +} +``` + +**Status Badges**: +```markdown +[![Release](https://github.com/user/repo/actions/workflows/release.yml/badge.svg)](https://github.com/user/repo/actions/workflows/release.yml) + +[![Docker](https://github.com/user/repo/actions/workflows/docker-release.yml/badge.svg)](https://github.com/user/repo/actions/workflows/docker-release.yml) +``` + +### Disaster Recovery + +#### Backup Workflow Files + +**Current Backup**: +- `.github/workflows/release.yml.backup` + +**Recommended**: +```bash +# Automatic backup before modifications +cp .github/workflows/release.yml .github/workflows/release.yml.backup-$(date +%Y%m%d) +git add .github/workflows/*.backup* +git commit -m "backup: workflow before modification" +``` + +#### Recovery from Failed Release + +**Scenario**: v1.2.3 release failed mid-way + +**Steps**: +1. **Identify what succeeded**: + - Check PyPI: `pip search crawl4ai` + - Check Docker Hub: https://hub.docker.com/r/unclecode/crawl4ai/tags + - Check GitHub Releases + +2. **Clean up partial release**: + ```bash + # Delete tag + git tag -d v1.2.3 + git push --delete origin v1.2.3 + + # Delete GitHub release (if created) + gh release delete v1.2.3 + ``` + +3. **Fix issue and retry**: + ```bash + # Fix the issue + # Re-tag and push + git tag v1.2.3 + git push origin v1.2.3 + ``` + +**Note**: Cannot delete PyPI uploads. If PyPI succeeded, increment to v1.2.4. + +### CI/CD Best Practices + +#### 1. Version Validation + +Add pre-commit hook: +```bash +# .git/hooks/pre-commit +#!/bin/bash +VERSION_FILE="crawl4ai/__version__.py" +VERSION=$(python -c "exec(open('$VERSION_FILE').read()); print(__version__)") +echo "Current version: $VERSION" +``` + +#### 2. Changelog Automation + +Use conventional commits: +```bash +git commit -m "feat: add new scraping mode" +git commit -m "fix: handle timeout errors" +git commit -m "docs: update API reference" +``` + +Generate changelog: +```bash +# Use git-cliff or similar +git cliff --tag v1.2.3 > CHANGELOG.md +``` + +#### 3. Pre-Release Testing + +Add test workflow: +```yaml +# .github/workflows/test.yml +on: + push: + tags: + - 'test-v*' + +jobs: + test-release: + runs-on: ubuntu-latest + steps: + - name: Build package + run: python -m build + - name: Upload to TestPyPI + run: twine upload --repository testpypi dist/* +``` + +#### 4. Release Checklist + +Create issue template: +```markdown +## Release Checklist + +- [ ] Update version in `crawl4ai/__version__.py` +- [ ] Update CHANGELOG.md +- [ ] Run tests locally: `pytest` +- [ ] Build package locally: `python -m build` +- [ ] Create and push tag: `git tag v1.2.3 && git push origin v1.2.3` +- [ ] Monitor Release Pipeline workflow +- [ ] Monitor Docker Release workflow +- [ ] Verify PyPI: `pip install crawl4ai==1.2.3` +- [ ] Verify Docker: `docker pull unclecode/crawl4ai:1.2.3` +- [ ] Announce release +``` + +--- + +## References + +### Official Documentation + +- [GitHub Actions Documentation](https://docs.github.com/en/actions) +- [Docker Build Push Action](https://github.com/docker/build-push-action) +- [PyPI Publishing Guide](https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/) + +### Related Files + +- [`release.yml`](../release.yml) - Main release workflow +- [`docker-release.yml`](../docker-release.yml) - Docker build workflow +- [`release.yml.backup`](../release.yml.backup) - Original combined workflow + +### Changelog + +| Date | Version | Changes | +|------|---------|---------| +| 2025-01-XX | 2.0 | Split workflows, added Docker caching | +| 2024-XX-XX | 1.0 | Initial combined workflow | + +--- + +## Support + +For issues or questions: +1. Check [Troubleshooting](#troubleshooting) section +2. Review [GitHub Actions logs](../../actions) +3. Create issue in repository + +--- + +**Last Updated**: 2025-01-21 +**Maintainer**: Crawl4AI Team diff --git a/.github/workflows/docs/WORKFLOW_REFERENCE.md b/.github/workflows/docs/WORKFLOW_REFERENCE.md new file mode 100644 index 0000000..208b4d6 --- /dev/null +++ b/.github/workflows/docs/WORKFLOW_REFERENCE.md @@ -0,0 +1,287 @@ +# Workflow Quick Reference + +## Quick Commands + +### Standard Release +```bash +# 1. Update version +vim crawl4ai/__version__.py # Set to "1.2.3" + +# 2. Commit and tag +git add crawl4ai/__version__.py +git commit -m "chore: bump version to 1.2.3" +git tag v1.2.3 +git push origin main +git push origin v1.2.3 + +# 3. Monitor +# - PyPI: ~2-3 minutes +# - Docker: ~1-15 minutes +``` + +### Docker Rebuild Only +```bash +git tag docker-rebuild-v1.2.3 +git push origin docker-rebuild-v1.2.3 +``` + +### Delete Tag (Undo Release) +```bash +# Local +git tag -d v1.2.3 + +# Remote +git push --delete origin v1.2.3 + +# GitHub Release +gh release delete v1.2.3 +``` + +--- + +## Workflow Triggers + +### release.yml +| Event | Pattern | Example | +|-------|---------|---------| +| Tag push | `v*` | `v1.2.3` | +| Excludes | `test-v*` | `test-v1.2.3` | + +### docker-release.yml +| Event | Pattern | Example | +|-------|---------|---------| +| Release published | `release.published` | Automatic | +| Tag push | `docker-rebuild-v*` | `docker-rebuild-v1.2.3` | + +--- + +## Environment Variables + +### release.yml +| Variable | Source | Example | +|----------|--------|---------| +| `VERSION` | Git tag | `1.2.3` | +| `TWINE_USERNAME` | Static | `__token__` | +| `TWINE_PASSWORD` | Secret | `pypi-Ag...` | +| `GITHUB_TOKEN` | Auto | `ghp_...` | + +### docker-release.yml +| Variable | Source | Example | +|----------|--------|---------| +| `VERSION` | Release/Tag | `1.2.3` | +| `MAJOR` | Computed | `1` | +| `MINOR` | Computed | `1.2` | +| `DOCKER_USERNAME` | Secret | `unclecode` | +| `DOCKER_TOKEN` | Secret | `dckr_pat_...` | + +--- + +## Docker Tags Generated + +| Version | Tags Created | +|---------|-------------| +| v1.0.0 | `1.0.0`, `1.0`, `1`, `latest` | +| v1.1.0 | `1.1.0`, `1.1`, `1`, `latest` | +| v1.2.3 | `1.2.3`, `1.2`, `1`, `latest` | +| v2.0.0 | `2.0.0`, `2.0`, `2`, `latest` | + +--- + +## Workflow Outputs + +### release.yml +| Output | Location | Time | +|--------|----------|------| +| PyPI Package | https://pypi.org/project/crawl4ai/ | ~2-3 min | +| GitHub Release | Repository → Releases | ~2-3 min | +| Workflow Summary | Actions → Run → Summary | Immediate | + +### docker-release.yml +| Output | Location | Time | +|--------|----------|------| +| Docker Images | https://hub.docker.com/r/unclecode/crawl4ai | ~1-15 min | +| Workflow Summary | Actions → Run → Summary | Immediate | + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| Version mismatch | Update `crawl4ai/__version__.py` to match tag | +| PyPI 403 Forbidden | Check `PYPI_TOKEN` secret | +| PyPI 400 File exists | Version already published, increment version | +| Docker auth failed | Regenerate `DOCKER_TOKEN` | +| Docker build timeout | Check Dockerfile, review build logs | +| Cache not working | First build on branch always cold | + +--- + +## Secrets Checklist + +- [ ] `PYPI_TOKEN` - PyPI API token (project or account scope) +- [ ] `DOCKER_USERNAME` - Docker Hub username +- [ ] `DOCKER_TOKEN` - Docker Hub access token (read/write) +- [ ] `GITHUB_TOKEN` - Auto-provided (no action needed) + +--- + +## Workflow Dependencies + +### release.yml Dependencies +```yaml +Python: 3.12 +Actions: + - actions/checkout@v4 + - actions/setup-python@v5 + - softprops/action-gh-release@v2 +PyPI Packages: + - build + - twine +``` + +### docker-release.yml Dependencies +```yaml +Actions: + - actions/checkout@v4 + - docker/setup-buildx-action@v3 + - docker/login-action@v3 + - docker/build-push-action@v5 +Docker: + - Buildx + - QEMU (for multi-arch) +``` + +--- + +## Cache Information + +### Type +- GitHub Actions Cache (`type=gha`) + +### Storage +- **Limit**: 10GB per repository +- **Retention**: 7 days for unused entries +- **Cleanup**: Automatic LRU eviction + +### Performance +| Scenario | Cache Hit | Build Time | +|----------|-----------|------------| +| First build | 0% | 10-15 min | +| Code change only | 85% | 1-2 min | +| Dependency update | 60% | 3-5 min | +| No changes | 100% | 30-60 sec | + +--- + +## Build Platforms + +| Platform | Architecture | Devices | +|----------|--------------|---------| +| linux/amd64 | x86_64 | Intel/AMD servers, AWS EC2, GCP | +| linux/arm64 | aarch64 | Apple Silicon, AWS Graviton, Raspberry Pi | + +--- + +## Version Validation + +### Pre-Tag Checklist +```bash +# Check current version +python -c "from crawl4ai.__version__ import __version__; print(__version__)" + +# Verify it matches intended tag +# If tag is v1.2.3, version should be "1.2.3" +``` + +### Post-Release Verification +```bash +# PyPI +pip install crawl4ai==1.2.3 +python -c "import crawl4ai; print(crawl4ai.__version__)" + +# Docker +docker pull unclecode/crawl4ai:1.2.3 +docker run unclecode/crawl4ai:1.2.3 python -c "import crawl4ai; print(crawl4ai.__version__)" +``` + +--- + +## Monitoring URLs + +| Service | URL | +|---------|-----| +| GitHub Actions | `https://github.com/{owner}/{repo}/actions` | +| PyPI Project | `https://pypi.org/project/crawl4ai/` | +| Docker Hub | `https://hub.docker.com/r/unclecode/crawl4ai` | +| GitHub Releases | `https://github.com/{owner}/{repo}/releases` | + +--- + +## Rollback Strategy + +### PyPI (Cannot Delete) +```bash +# Increment patch version +git tag v1.2.4 +git push origin v1.2.4 +``` + +### Docker (Can Overwrite) +```bash +# Rebuild with fix +git tag docker-rebuild-v1.2.3 +git push origin docker-rebuild-v1.2.3 +``` + +### GitHub Release +```bash +# Delete release +gh release delete v1.2.3 + +# Delete tag +git push --delete origin v1.2.3 +``` + +--- + +## Status Badge Markdown + +```markdown +[![Release Pipeline](https://github.com/{owner}/{repo}/actions/workflows/release.yml/badge.svg)](https://github.com/{owner}/{repo}/actions/workflows/release.yml) + +[![Docker Release](https://github.com/{owner}/{repo}/actions/workflows/docker-release.yml/badge.svg)](https://github.com/{owner}/{repo}/actions/workflows/docker-release.yml) +``` + +--- + +## Timeline Example + +``` +0:00 - Push tag v1.2.3 +0:01 - release.yml starts +0:02 - Version validation passes +0:03 - Package built +0:04 - PyPI upload starts +0:06 - PyPI upload complete ✓ +0:07 - GitHub release created ✓ +0:08 - release.yml complete +0:08 - docker-release.yml triggered +0:10 - Docker build starts +0:12 - amd64 image built (cache hit) +0:14 - arm64 image built (cache hit) +0:15 - Images pushed to Docker Hub ✓ +0:16 - docker-release.yml complete + +Total: ~16 minutes +Critical path (PyPI + GitHub): ~8 minutes +``` + +--- + +## Contact + +For workflow issues: +1. Check Actions tab for logs +2. Review this reference +3. See [README.md](./README.md) for detailed docs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..b53eb7b --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,46 @@ +name: Discord GitHub Notifications + +on: + issues: + types: [opened] + issue_comment: + types: [created] + pull_request: + types: [opened] + discussion: + types: [created] + watch: + types: [started] + +jobs: + notify-discord: + runs-on: ubuntu-latest + steps: + - name: Send to Google Apps Script (Stars only) + if: github.event_name == 'watch' + run: | + curl -fSs -X POST "${{ secrets.GOOGLE_SCRIPT_ENDPOINT }}" \ + -H 'Content-Type: application/json' \ + -d '{"url":"${{ github.event.sender.html_url }}"}' + - name: Set webhook based on event type + id: set-webhook + run: | + if [ "${{ github.event_name }}" == "discussion" ]; then + echo "webhook=${{ secrets.DISCORD_DISCUSSIONS_WEBHOOK }}" >> $GITHUB_OUTPUT + elif [ "${{ github.event_name }}" == "watch" ]; then + echo "webhook=${{ secrets.DISCORD_STAR_GAZERS }}" >> $GITHUB_OUTPUT + else + echo "webhook=${{ secrets.DISCORD_WEBHOOK }}" >> $GITHUB_OUTPUT + fi + + - name: Discord Notification + uses: Ilshidur/action-discord@master + env: + DISCORD_WEBHOOK: ${{ steps.set-webhook.outputs.webhook }} + with: + args: | + ${{ github.event_name == 'issues' && format('📣 New issue created: **{0}** by {1} - {2}', github.event.issue.title, github.event.issue.user.login, github.event.issue.html_url) || + github.event_name == 'issue_comment' && format('💬 New comment on issue **{0}** by {1} - {2}', github.event.issue.title, github.event.comment.user.login, github.event.comment.html_url) || + github.event_name == 'pull_request' && format('🔄 New PR opened: **{0}** by {1} - {2}', github.event.pull_request.title, github.event.pull_request.user.login, github.event.pull_request.html_url) || + github.event_name == 'watch' && format('⭐ {0} starred Crawl4AI 🥳! Check out their profile: {1}', github.event.sender.login, github.event.sender.html_url) || + format('💬 New discussion started: **{0}** by {1} - {2}', github.event.discussion.title, github.event.discussion.user.login, github.event.discussion.html_url) }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..91e84ff --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,113 @@ +name: Release Pipeline +on: + push: + tags: + - 'v*' + - '!test-v*' # Exclude test tags + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write # Required for creating releases + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Extract version from tag + id: get_version + run: | + TAG_VERSION=${GITHUB_REF#refs/tags/v} + echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT + echo "Releasing version: $TAG_VERSION" + + - name: Install package dependencies + run: | + pip install -e . + + - name: Check version consistency + run: | + TAG_VERSION=${{ steps.get_version.outputs.VERSION }} + PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)") + + echo "Tag version: $TAG_VERSION" + echo "Package version: $PACKAGE_VERSION" + + if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then + echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION" + echo "Please update crawl4ai/__version__.py to match the tag version" + exit 1 + fi + echo "✅ Version check passed: $TAG_VERSION" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Check package + run: twine check dist/* + + - name: Upload to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + echo "📦 Uploading to PyPI..." + twine upload dist/* + echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.get_version.outputs.VERSION }} + name: Release v${{ steps.get_version.outputs.VERSION }} + body: | + ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released! + + ### 📦 Installation + + **PyPI:** + ```bash + pip install crawl4ai==${{ steps.get_version.outputs.VERSION }} + ``` + + **Docker:** + ```bash + docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }} + docker pull unclecode/crawl4ai:latest + ``` + + **Note:** Docker images are being built and will be available shortly. + Check the [Docker Release workflow](https://github.com/${{ github.repository }}/actions/workflows/docker-release.yml) for build status. + + ### 📝 What's Changed + See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details. + draft: false + prerelease: false + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Summary + run: | + echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 📦 PyPI Package" >> $GITHUB_STEP_SUMMARY + echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY + echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY + echo "- https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY + echo "Docker images are being built in a separate workflow." >> $GITHUB_STEP_SUMMARY + echo "Check: https://github.com/${{ github.repository }}/actions/workflows/docker-release.yml" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/release.yml.backup b/.github/workflows/release.yml.backup new file mode 100644 index 0000000..3ee9042 --- /dev/null +++ b/.github/workflows/release.yml.backup @@ -0,0 +1,142 @@ +name: Release Pipeline +on: + push: + tags: + - 'v*' + - '!test-v*' # Exclude test tags + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write # Required for creating releases + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Extract version from tag + id: get_version + run: | + TAG_VERSION=${GITHUB_REF#refs/tags/v} + echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT + echo "Releasing version: $TAG_VERSION" + + - name: Install package dependencies + run: | + pip install -e . + + - name: Check version consistency + run: | + TAG_VERSION=${{ steps.get_version.outputs.VERSION }} + PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)") + + echo "Tag version: $TAG_VERSION" + echo "Package version: $PACKAGE_VERSION" + + if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then + echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION" + echo "Please update crawl4ai/__version__.py to match the tag version" + exit 1 + fi + echo "✅ Version check passed: $TAG_VERSION" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Check package + run: twine check dist/* + + - name: Upload to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + echo "📦 Uploading to PyPI..." + twine upload dist/* + echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Extract major and minor versions + id: versions + run: | + VERSION=${{ steps.get_version.outputs.VERSION }} + MAJOR=$(echo $VERSION | cut -d. -f1) + MINOR=$(echo $VERSION | cut -d. -f1-2) + echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT + echo "MINOR=$MINOR" >> $GITHUB_OUTPUT + + - name: Build and push Docker images + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: | + unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }} + unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }} + unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }} + unclecode/crawl4ai:latest + platforms: linux/amd64,linux/arm64 + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.get_version.outputs.VERSION }} + name: Release v${{ steps.get_version.outputs.VERSION }} + body: | + ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released! + + ### 📦 Installation + + **PyPI:** + ```bash + pip install crawl4ai==${{ steps.get_version.outputs.VERSION }} + ``` + + **Docker:** + ```bash + docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }} + docker pull unclecode/crawl4ai:latest + ``` + + ### 📝 What's Changed + See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details. + draft: false + prerelease: false + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Summary + run: | + echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 📦 PyPI Package" >> $GITHUB_STEP_SUMMARY + echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY + echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY + echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..14c80a1 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,65 @@ +name: Security + +# Runs the offline Docker-server security suite: behavioral tests for the +# secure-by-default posture (R1-R7). No network, browser, Redis or Docker needed +# - it boots the app via TestClient and monkeypatches DNS, so it is fast +# (~seconds) and deterministic. + +on: + push: + branches: [main, develop, "security/**"] + paths: + - "deploy/docker/**" + - "crawl4ai/async_configs.py" + - ".github/workflows/security.yml" + pull_request: + paths: + - "deploy/docker/**" + - "crawl4ai/async_configs.py" + - ".github/workflows/security.yml" + +jobs: + security-offline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install -r deploy/docker/requirements.txt + pip install pytest pytest-asyncio + + - name: Run security suite + run: | + pytest deploy/docker/tests/test_security_*.py -q + + posture-gate: + # The headline secure-by-default acceptance gate, isolated so it can be a + # required status check. xfail-marked items (e.g. the build-gated + # --no-sandbox removal) do not fail this job. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install -r deploy/docker/requirements.txt + pip install pytest pytest-asyncio + + - name: Default-posture gate + run: | + pytest deploy/docker/tests/test_security_default_posture.py -q diff --git a/.github/workflows/test-release.yml.disabled b/.github/workflows/test-release.yml.disabled new file mode 100644 index 0000000..ce4fb67 --- /dev/null +++ b/.github/workflows/test-release.yml.disabled @@ -0,0 +1,116 @@ +name: Test Release Pipeline +on: + push: + tags: + - 'test-v*' + +jobs: + test-release: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Extract version from tag + id: get_version + run: | + TAG_VERSION=${GITHUB_REF#refs/tags/test-v} + echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT + echo "Testing with version: $TAG_VERSION" + + - name: Install package dependencies + run: | + pip install -e . + + - name: Check version consistency + run: | + TAG_VERSION=${{ steps.get_version.outputs.VERSION }} + PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)") + + echo "Tag version: $TAG_VERSION" + echo "Package version: $PACKAGE_VERSION" + + if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then + echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION" + echo "Please update crawl4ai/__version__.py to match the tag version" + exit 1 + fi + echo "✅ Version check passed: $TAG_VERSION" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Check package + run: twine check dist/* + + - name: Upload to Test PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} + run: | + echo "📦 Uploading to Test PyPI..." + twine upload --repository testpypi dist/* || { + if [ $? -eq 1 ]; then + echo "⚠️ Upload failed - likely version already exists on Test PyPI" + echo "Continuing anyway for test purposes..." + else + exit 1 + fi + } + echo "✅ Test PyPI step complete" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push Docker test images + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: | + unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }} + unclecode/crawl4ai:test-latest + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Summary + run: | + echo "## 🎉 Test Release Complete!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 📦 Test PyPI Package" >> $GITHUB_STEP_SUMMARY + echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- URL: https://test.pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY + echo "- Install: \`pip install -i https://test.pypi.org/simple/ crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 🐳 Docker Test Images" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`unclecode/crawl4ai:test-latest\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 🧹 Cleanup Commands" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "# Remove test tag" >> $GITHUB_STEP_SUMMARY + echo "git tag -d test-v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "git push origin :test-v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "# Remove Docker test images" >> $GITHUB_STEP_SUMMARY + echo "docker rmi unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "docker rmi unclecode/crawl4ai:test-latest" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9238148 --- /dev/null +++ b/.gitignore @@ -0,0 +1,313 @@ +# Scripts folder (private tools) +.scripts/ + +# Database files +*.db + +# Environment files +.env +.env.local + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +Crawl4AI.egg-info/ +Crawl4AI.egg-info/* +crawler_data.db +.vscode/ +.tests/ +.test_pads/ +test_pad.py +test_pad*.py +.data/ +Crawl4AI.egg-info/ + +requirements0.txt +a.txt + +*.sh +.idea +docs/examples/.chainlit/ +docs/examples/.chainlit/* +.chainlit/config.toml +.chainlit/translations/en-US.json + +local/ +.files/ + +a.txt +.lambda_function.py +ec2* + +update_changelog.sh + +.DS_Store +docs/.DS_Store +tmp/ +test_env/ +**/.DS_Store +**/.DS_Store + +todo.md +todo_executor.md +git_changes.py +git_changes.md +pypi_build.sh +git_issues.py +git_issues.md + +.next/ +.tests/ +# .issues/ +.docs/ +.issues/ +.gitboss/ +todo_executor.md +protect-all-except-feature.sh +manage-collab.sh +publish.sh +combine.sh +combined_output.txt +.local +.scripts +tree.md +tree.md +.scripts +.local +.do +/plans +plans/ + +# Codeium +.codeiumignore +todo/ + +# Continue development files +.continue/ +.continuerc.json +continue.lock +continue_core.log +contextProviders/ +continue_workspace/ +.continue-cache/ +continue_config.json + +# Continue temporary files +.continue-temp/ +.continue-logs/ +.continue-downloads/ + +# Continue VS Code specific +.vscode-continue/ +.vscode-continue-cache/ + +.prompts/ + +.llm.env +.private/ + +.claude/ +.context/ + +CLAUDE_MONITOR.md +CLAUDE.md + +.claude/ + +tests/**/test_site +tests/**/reports +tests/**/benchmark_reports +test_scripts/ +docs/**/data +.codecat/ + +docs/apps/linkdin/debug*/ +docs/apps/linkdin/samples/insights/* + +scripts/ +!scripts/gen-sbom.sh +!scripts/update_stats.py + + +# Databse files +*.sqlite3 +*.sqlite3-journal +*.db-journal +*.db-wal +*.db-shm +*.db +*.rdb +*.ldb +MEMORY.md + +# Handoff files +HANDOFF-*.md + +# Security advisory payloads - kept private, published via GitHub Security Advisories only +.security/ + +# Local pipeline/run output +out/ + +# entrypoint.sh is a required deployment artifact (Dockerfile COPYs it) +!deploy/docker/entrypoint.sh diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c09b7d2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,1777 @@ +# Changelog + +All notable changes to Crawl4AI will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.9.0] - 2026-06-18 + +0.9.0 is a major, secure-by-default release of the Crawl4AI Docker API server. The out-of-the-box deployment is now hardened with defense in depth: authentication is on by default, the server binds loopback unless you give it a token, and the network request body is treated as an untrusted trust boundary. This release contains breaking changes for the self-hosted HTTP server only. The core pip library (SDK / in-process use) is unchanged. + +**What changed:** the Docker server moved from an open, trust-the-caller posture to a closed, secure-by-default one. Defaults that used to be permissive (open bind, no auth, request-supplied browser internals, TLS verification off, Redis with no password) are now safe by default and gated behind explicit configuration. + +**What you must do:** set `CRAWL4AI_API_TOKEN` and re-issue any tokens, then review whether you relied on any of the request fields or features that are now configured server-side. Most plain "crawl these URLs" users only need the two steps in the "Everyone" section of the migration guide. The full guide is at `deploy/docker/MIGRATION.md`. + +### Security + +This release completes the secure-by-default hardening of the Docker API server begun in 0.8.7 and 0.8.8. It moves the worst remaining issues from mitigation to architecture: unauthenticated access and request-supplied code/config are eliminated by design rather than patched in place. Every change is hardening; users self-hosting the Docker server should upgrade and follow the migration guide. + +- **Authentication on by default, loopback bind**: the server no longer serves an unauthenticated API on `0.0.0.0`. With no token it binds `127.0.0.1` and prints a one-off local token; exposing it requires `CRAWL4AI_API_TOKEN` and `Authorization: Bearer ` on every request except `GET /health`. +- **Request trust boundary**: a crawl request body now carries declarative, scalar options only. Fields that previously let a caller drive browser internals or arbitrary code are rejected at the network boundary. +- **Declarative hooks replace hook code**: arbitrary Python hook strings are replaced by a fixed set of declarative actions, removing request-supplied code from the server entirely. +- **Strengthened JWT, admin-scoped monitor actions, deny-by-default CORS, strict security headers, TLS verification on, password-protected loopback-only Redis, bounded job queue, generic error responses with correlation ids, and validated webhook headers** round out the defense-in-depth posture. See the migration guide for the full list. +- **Download path confinement (CWE-22)**: both download sinks now confine writes with basename plus realpath plus `O_NOFOLLOW`, closing a path-traversal-to-file-write class. Credit: Y4tacker. +- **SSRF destination validation on the streaming crawl path (CWE-918)**: `/crawl/stream` and `/crawl` with `stream=true` now validate the destination and return HTTP 400 for disallowed targets, matching the non-streaming handlers. Credit: KOH Jun Sheng. +- **Request-supplied `browser_config.extra_args` rejected (CWE-94)**: launch arguments can no longer be supplied over the network, closing a Chromium launch-arg injection class. Credit: Y4tacker, UDU_RisePho ([hoanggxyuuki](https://github.com/hoanggxyuuki)). + +All reporters are credited in `SECURITY-CREDITS.md`. GitHub Security Advisories accompany this release. + +### Breaking Changes + +These apply to the self-hosted Docker API server only. The pip library is unaffected. See `deploy/docker/MIGRATION.md` for the step-by-step migration and `deploy/docker/SECURITY-VERIFY.md` for the deployment checklist. + +- **Auth is on by default**: set `CRAWL4AI_API_TOKEN` and send `Authorization: Bearer `. With no token the server binds loopback only. +- **Loopback bind by default**: the server no longer binds `0.0.0.0` without a token; put a TLS-terminating reverse proxy in front when you expose it. +- **Tokens must be re-issued**: the JWT implementation changed and tokens from older versions are no longer valid. Re-mint via `POST /token`. +- **Request trust boundary**: `js_code`, `js_code_before_wait`, `c4a_script`, `proxy` / `proxy_config`, `extra_args`, `user_data_dir`, `cdp_url`, `cookies`, `headers`, `init_scripts`, `base_url`, `deep_crawl_strategy`, `simulate_user`, `magic`, `process_in_browser`, and nested LLM config objects are rejected with HTTP 400 when sent over the network. Configure them server-side or use the in-process SDK. Unknown fields are dropped; timeouts, viewport, and scroll counts are clamped. +- **Hooks are declarative**: `hooks.code` is replaced by a fixed action set (`block_resources`, `add_cookies`, `set_headers`, `scroll_to_bottom`, `wait_for_timeout`). See `GET /hooks/info`. +- **`output_path` removed, replaced by an artifact id**: `/screenshot` and `/pdf` store the result and return `artifact_id` + URL; fetch via authenticated `GET /artifacts/{artifact_id}` (TTL and quota apply). +- **LLM `base_url` removed**: `/md`, `/llm`, and `/llm/job` select a provider by name only; endpoint and key are configured server-side and constrained by `config.llm.allowed_providers`. +- **Monitor actions require an admin token**: `POST /monitor/actions/*` and `/monitor/stats/reset` need an admin-scope principal. +- **CORS deny-by-default**: cross-origin browser requests are denied unless listed in `security.cors_allow_origins`. +- **TLS verification on**: self-signed / internal TLS targets fail by default. Escape hatches for trusted internal testing: `CRAWL4AI_ALLOW_INSECURE_TLS=true`, `CRAWL4AI_ALLOW_INTERNAL_URLS=true`. +- **Webhook headers validated**: malformed or hop-by-hop / sensitive headers are rejected with HTTP 422. +- **Redis requires a password**: in-container Redis is loopback-only, password-protected, and its port is no longer published. For external Redis set `REDIS_PASSWORD`. +- **Bounded background job queue**: request body size, per-crawl wall clock, queue size, and per-principal concurrency are now capped (configurable; `0` = unbounded). +- **Generic 5xx responses**: server errors return `{"error": "Internal server error", "correlation_id": "…"}`; match the id in the logs for detail. + +### Security Credits + +Y4tacker, KOH Jun Sheng, and UDU_RisePho ([hoanggxyuuki](https://github.com/hoanggxyuuki)). See `SECURITY-CREDITS.md`. + +## [0.8.9] - 2026-06-04 + +0.8.9 is a follow-up, backward-compatible security patch for the self-hosted Docker API server, closing an SSRF path that 0.8.8 did not cover. Upgrade in place; no configuration changes required. + +### Security + +A security advisory accompanies this release. + +- **SSRF via proxy settings (CWE-918)**: the SSRF destination check was applied only to the crawl target URL, not to the proxy address. An unauthenticated `/crawl`, `/crawl/stream`, or `/crawl/job` request could set `browser_config.proxy_config.server` (or the deprecated `browser_config.proxy`, or `crawler_config.proxy_config`, or a `--proxy-server` / `--host-resolver-rules` flag in `extra_args`) to an internal address and route the browser through it, reaching internal services and cloud-metadata endpoints. All proxy destinations are now validated with the same global-routability check before the browser is built, and proxy/DNS-redirecting flags are stripped from `extra_args`. A legitimate public proxy still works. Credit: Geo ([geo-chen](https://github.com/geo-chen)). + +Backward compatible. Note: raw `--proxy-server` / `--host-resolver-rules` / `--proxy-bypass-list` / `--proxy-pac-url` flags passed via `extra_args` are now ignored; configure proxies through `proxy_config` (which is validated). + +## [0.8.8] - 2026-06-04 + +0.8.8 is a focused, backward-compatible security patch for the self-hosted Docker API server. Upgrade in place; no configuration changes are required. If you run the Docker server, upgrade. If it is exposed to a network, also set `CRAWL4AI_API_TOKEN`. + +### Security + +Security advisories accompany this release. + +- **SSRF filter gaps closed (CWE-918)**: the Docker server's SSRF protection now rejects any resolved address that is not globally routable, evaluated on IPv6 transition forms too (NAT64 `64:ff9b::/96`, 6to4 `2002::/16`, IPv4-mapped, and the unspecified `::`), which previously bypassed the explicit blocklist and could reach internal services and cloud-metadata endpoints. SSRF errors no longer echo the resolved address. Credit: internal security audit. +- **Arbitrary file write via `output_path` hardened (CWE-59/22)**: `/screenshot` and `/pdf` now resolve symlinks and re-check containment before writing, and write with `O_NOFOLLOW`, closing a symlink/TOCTOU bypass of the directory restriction. `output_path` behavior is unchanged for normal use. Credit: internal security audit. +- **LLM credential exfiltration closed (CWE-522/200)**: the LLM endpoints (`/md`, `/llm`, `/llm/job`) ignore a request-supplied `base_url`, so the configured provider key can no longer be redirected to an attacker endpoint. `LLMConfig` additionally refuses to resolve protected environment variables via the `env:` token form. The `base_url` field is still accepted but no longer honored. Credit: Geo ([geo-chen](https://github.com/geo-chen)); the `env:` hardening from internal security audit. +- **CRLF-safe logging (CWE-117)** and **webhook request-header validation (CWE-93)**: log records are stripped of CR/LF/control characters, and user-supplied webhook headers are validated (name pattern, no control characters, hop-by-hop/sensitive headers denied). + +All changes are backward compatible. + +### Coming next: secure-by-default Docker server (~1-2 weeks) + +The next release is a larger, secure-by-default update for the self-hosted Docker API server, with intentional breaking changes. We are giving advance notice so you can prepare. If you run the Docker server, start planning now and test in staging before upgrading: + +- Authentication will be on by default. The server binds loopback unless a credential (`CRAWL4AI_API_TOKEN`) is configured. +- Request bodies are validated more strictly and safer defaults apply (TLS verification on, stricter outbound egress controls, declarative hook actions instead of inline code). +- A few request options move server-side: `/screenshot` and `/pdf` return an artifact id instead of a file path, and the LLM endpoint is selected by provider name. +- Hardened container defaults (least-privilege compose, Redis authentication, loopback bind). + +A full migration guide will accompany the pre-announcement on Discord and X. + +## [0.8.7] - 2026-06-01 + +0.8.7 is a security-hardening release. It bundles every responsibly-disclosed vulnerability patched since 0.8.6, plus the new DomainMapper feature and a batch of scraping, deep-crawl, and LLM fixes. + +### Security + +This release fixes multiple critical vulnerabilities in the Docker API server. If you self-host the Docker API, upgrade immediately. Two GitHub Security Advisories accompany this release. + +- **CRITICAL: AST Sandbox Escape leading to Pre-Auth RCE (CVSS 9.8, CWE-94/913)**: a `gi_frame.f_back` frame-chain escape in the computed-field `eval()` path. Removed `eval()` from computed fields entirely and deleted `_safe_eval_expression`. Credit: Song Binglin ([q1uf3ng](https://github.com/q1uf3ng)). +- **CRITICAL: Hook Sandbox Escape RCE (CVSS 9.8, CWE-94)**: injected module objects (`asyncio`, `json`, `re`) carried a full `__builtins__`, bypassing the `__import__` block. Stripped injected builtins and removed dangerous allowlist entries. Credit: by111 ([August829](https://github.com/August829)). +- **CRITICAL: Hardcoded JWT Secret (CVSS 9.8, CWE-798)**: the default signing key `"mysecret"` allowed token forgery. Removed the default, reject weak/short secrets, and auto-generate an ephemeral key when JWT is enabled with no key set. Credit: by111 ([August829](https://github.com/August829)). +- **HIGH: Arbitrary File Write via `output_path` (CVSS 9.1, CWE-22)**: `/screenshot` and `/pdf` wrote to any path. Restricted writes to `CRAWL4AI_OUTPUT_DIR` and reject `..` traversal. Credit: Jeongbean Jeon, wulonchia. +- **HIGH: SSRF via Webhook URL (CVSS 8.6, CWE-918)**: webhook URLs on `/crawl/job` and `/llm/job` could reach internal and cloud-metadata IPs. Added a blocklist and `follow_redirects=False`. Credit: Jeongbean Jeon. +- **HIGH: SSRF via Direct Crawl Endpoints (CVSS 8.6, CWE-918)**: `/crawl`, `/md`, and `/llm` fetched arbitrary URLs, and IPv6-mapped IPv4 addresses (`[::ffff:169.254.169.254]`) bypassed naive checks. Added destination validation on all entry points and normalize IPv6-mapped IPv4 before the blocklist check. Credit: secsys_codex, Velayutham Selvaraj, IcySun. +- **HIGH: Arbitrary JavaScript Execution via `/execute_js` (CVSS 8.1, CWE-94)**: disabled by default via `CRAWL4AI_EXECUTE_JS_ENABLED`, removed `--disable-web-security` from default browser args, and added an SSRF blocklist on the destination. Credit: by111 ([August829](https://github.com/August829)). +- **MEDIUM: Monitor Endpoint Auth Bypass (CVSS 6.5, CWE-306)**: `/monitor/*` routes, including destructive actions, were unauthenticated. Added `token_dep` to the router and an explicit token check on the WebSocket endpoint. Credit: Jeongbean Jeon. +- **MEDIUM: Stored XSS in Monitor Dashboard (CVSS 6.1, CWE-79)**: URLs and errors were rendered via `innerHTML` without escaping. Added server-side `html.escape()` and a client-side `escapeHtml()` wrapper. Credit: Jeongbean Jeon. +- **eval() removed from `/config/dump`**: replaced with JSON input validated by Pydantic. +- **Config hardening**: validate the `markdown_generator` type in `CrawlerRunConfig` to reject malformed JSON (#1880). + +### Added + +- **DomainMapper**: comprehensive domain URL discovery, with an `include_subdomains` flag and a per-source timeout. +- **arun_many config-list support in the Docker API**: per-URL configs (#1837). +- **Docker server can listen on all addresses.** + +### Fixed + +- **Markdown and scraping fidelity**: + - Preserve mermaid diagram text from SVGs and prevent nested fences (#1043) + - Preserve table `rowspan`/`colspan` in cleaned HTML (#1920) + - Preserve `.tail` text when removing empty elements (#1938) + - Keep sentence order in `NlpSentenceChunking` (#1909) +- **Deep crawl and dispatcher**: + - Fix the deep-crawl streaming ContextVar bug, using `set(False)` instead of `reset(token)` (#1917) + - Wire `semaphore_count` into the auto-created `MemoryAdaptiveDispatcher` and default it to 10 (#1927) +- **LLM and providers**: + - Add Bedrock to the provider prefixes so AWS credential auth works + - Default `LLMExtractionStrategy.extraction_type` to schema + - Add `LLMTableExtraction` to the Docker deserialization allowlist +- **Crawler and downloads**: + - Return `success=True` for binary downloads and skip the block check when `downloaded_files` is set + - Honor `` in prefetch `quick_extract_links` (#752) +- **Logging and MCP**: + - Route `AsyncLogger` output to stderr by default (#1968) and use `Console(width=200)` for non-TTY contexts + - Use `ensure_ascii=False` in the MCP bridge to preserve CJK characters (#1967) +- **Browser and misc**: + - `browser_adapter` now uses the `Stealth` import, fixing a stealth import mismatch (#1960) + - Correct the `arun()` return type to `CrawlResultContainer` (#1898) + - Log the real failure reason before COMPLETE, fixing a misleading "SCRAPE ok" line (#1949) + - Assistant toolbar scroll fix and issue-1973 fix + +### Docs + +- Added Privacy Policy, Terms of Service, and Support pages. + +### Security Credits + +Song Binglin (q1uf3ng), by111 (August829), Jeongbean Jeon, wulonchia, secsys_codex, Velayutham Selvaraj, and IcySun. See `SECURITY-CREDITS.md`. + +## [0.8.0] - 2026-01-12 + +### Security +- **🔒 CRITICAL: Remote Code Execution Fix**: Removed `__import__` from hook allowed builtins + - Prevents arbitrary module imports in user-provided hook code + - Hooks now disabled by default via `CRAWL4AI_HOOKS_ENABLED` environment variable + - Credit: Neo by ProjectDiscovery +- **🔒 HIGH: Local File Inclusion Fix**: Added URL scheme validation to Docker API endpoints + - Blocks `file://`, `javascript:`, `data:` URLs on `/execute_js`, `/screenshot`, `/pdf`, `/html` + - Only allows `http://`, `https://`, and `raw:` URLs + - Credit: Neo by ProjectDiscovery + +### Breaking Changes +- **Docker API: Hooks disabled by default**: Set `CRAWL4AI_HOOKS_ENABLED=true` to enable +- **Docker API: file:// URLs blocked**: Use Python library directly for local file processing + +### Added +- **🚀 init_scripts for BrowserConfig**: Pre-page-load JavaScript injection for stealth evasions +- **🔄 CDP Connection Improvements**: WebSocket URL support, proper cleanup, browser reuse +- **💾 Crash Recovery for Deep Crawl**: `resume_state` and `on_state_change` for BFS/DFS/Best-First strategies +- **📄 PDF/MHTML for raw:/file:// URLs**: Generate PDFs and MHTML from cached HTML content +- **📸 Screenshots for raw:/file:// URLs**: Render cached HTML and capture screenshots +- **🔗 base_url Parameter**: Proper URL resolution for raw: HTML processing +- **⚡ Prefetch Mode**: Two-phase deep crawling with fast link extraction +- **🔀 Enhanced Proxy Support**: Improved proxy rotation and sticky sessions +- **🌐 HTTP Strategy Proxy Support**: Non-browser crawler now supports proxies +- **🖥️ Browser Pipeline for raw:/file://**: New `process_in_browser` parameter +- **📋 Smart TTL Cache for Sitemap Seeder**: `cache_ttl_hours` and `validate_sitemap_lastmod` parameters +- **📚 Security Documentation**: Added SECURITY.md with vulnerability reporting guidelines + +### Fixed +- **raw: URL Parsing**: Fixed truncation at `#` character (CSS color codes like `#eee`) +- **Caching System**: Various improvements to cache validation and persistence + +### Documentation +- Multi-sample schema generation section +- URL seeder smart TTL cache parameters +- v0.8.0 migration guide +- Security policy and disclosure process + +## [Unreleased] + +### Added +- **🔒 HTTPS Preservation for Internal Links**: New `preserve_https_for_internal_links` configuration flag + - Maintains HTTPS scheme for internal links even when servers redirect to HTTP + - Prevents security downgrades during deep crawling + - Useful for security-conscious crawling and sites supporting both protocols + - Fully backward compatible with opt-in flag (default: `False`) + - Fixes issue #1410 where HTTPS URLs were being downgraded to HTTP + +## [0.7.3] - 2025-08-09 + +### Added +- **🕵️ Undetected Browser Support**: New browser adapter pattern with stealth capabilities + - `browser_adapter.py` with undetected Chrome integration + - Bypass sophisticated bot detection systems (Cloudflare, Akamai, custom solutions) + - Support for headless stealth mode with anti-detection techniques + - Human-like behavior simulation with random mouse movements and scrolling + - Comprehensive examples for anti-bot strategies and stealth crawling + - Full documentation guide for undetected browser usage + +- **🎨 Multi-URL Configuration System**: URL-specific crawler configurations for batch processing + - Different crawling strategies for different URL patterns in a single batch + - Support for string patterns with wildcards (`"*.pdf"`, `"*/blog/*"`) + - Lambda function matchers for complex URL logic + - Mixed matchers combining strings and functions with AND/OR logic + - Fallback configuration support when no patterns match + - First-match-wins configuration selection with optional fallback + +- **🧠 Memory Monitoring & Optimization**: Comprehensive memory usage tracking + - New `memory_utils.py` module for memory monitoring and optimization + - Real-time memory usage tracking during crawl sessions + - Memory leak detection and reporting + - Performance optimization recommendations + - Peak memory usage analysis and efficiency metrics + - Automatic cleanup suggestions for memory-intensive operations + +- **📊 Enhanced Table Extraction**: Improved table access and DataFrame conversion + - Direct `result.tables` interface replacing generic `result.media` approach + - Instant pandas DataFrame conversion with `pd.DataFrame(table['data'])` + - Enhanced table detection algorithms for better accuracy + - Table metadata including source XPath and headers + - Improved table structure preservation during extraction + +- **💰 GitHub Sponsors Integration**: 4-tier sponsorship system + - Supporter ($5/month): Community support + early feature previews + - Professional ($25/month): Priority support + beta access + - Business ($100/month): Direct consultation + custom integrations + - Enterprise ($500/month): Dedicated support + feature development + - Custom arrangement options for larger organizations + +- **🐳 Docker LLM Provider Flexibility**: Environment-based LLM configuration + - `LLM_PROVIDER` environment variable support for dynamic provider switching + - `.llm.env` file support for secure configuration management + - Per-request provider override capabilities in API endpoints + - Support for OpenAI, Groq, and other providers without rebuilding images + - Enhanced Docker documentation with deployment examples + +### Fixed +- **URL Matcher Fallback**: Resolved edge cases in URL pattern matching logic +- **Memory Management**: Fixed memory leaks in long-running crawl sessions +- **Sitemap Processing**: Improved redirect handling in sitemap fetching +- **Table Extraction**: Enhanced table detection and extraction accuracy +- **Error Handling**: Better error messages and recovery from network failures + +### Changed +- **Architecture Refactoring**: Major cleanup and optimization + - Moved 2,450+ lines from main `async_crawler_strategy.py` to backup + - Cleaner separation of concerns in crawler architecture + - Better maintainability and code organization + - Preserved backward compatibility while improving performance + +### Documentation +- **Comprehensive Examples**: Added real-world URLs and practical use cases +- **API Documentation**: Complete CrawlResult field documentation with all available fields +- **Migration Guides**: Updated table extraction patterns from `result.media` to `result.tables` +- **Undetected Browser Guide**: Full documentation for stealth mode and anti-bot strategies +- **Multi-Config Examples**: Detailed examples for URL-specific configurations +- **Docker Deployment**: Enhanced Docker documentation with LLM provider configuration + +## [0.7.x] - 2025-06-29 + +### Added +- **Virtual Scroll Support**: New `VirtualScrollConfig` for handling virtualized scrolling on modern websites + - Automatically detects and handles three scrolling scenarios: + - Content unchanged (continue scrolling) + - Content appended (traditional infinite scroll) + - Content replaced (true virtual scroll - Twitter/Instagram style) + - Captures ALL content from pages that replace DOM elements during scroll + - Intelligent deduplication based on normalized text content + - Configurable scroll amount, count, and wait times + - Seamless integration with existing extraction strategies + - Comprehensive examples including Twitter timeline, Instagram grid, and mixed content scenarios + +## [Unreleased] + +### Added +- **Flexible LLM Provider Configuration** (Docker): + - Support for `LLM_PROVIDER` environment variable to override default provider + - Per-request provider override via optional `provider` parameter in API endpoints + - Automatic provider validation with clear error messages + - Updated Docker documentation and examples + +### Changed +- **WebScrapingStrategy Refactoring**: Simplified content scraping architecture + - `WebScrapingStrategy` is now an alias for `LXMLWebScrapingStrategy` for backward compatibility + - Removed redundant BeautifulSoup-based implementation (~1000 lines of code) + - `LXMLWebScrapingStrategy` now inherits directly from `ContentScrapingStrategy` + - All existing code using `WebScrapingStrategy` continues to work without modification + - Default scraping strategy remains `LXMLWebScrapingStrategy` for optimal performance + +### Added +- **AsyncUrlSeeder**: High-performance URL discovery system for intelligent crawling at scale + - Discover URLs from sitemaps and Common Crawl index + - Extract and analyze page metadata without full crawling + - BM25 relevance scoring for query-based URL filtering + - Multi-domain parallel discovery with `many_urls()` method + - Automatic caching with TTL for discovered URLs + - Rate limiting and concurrent request management + - Live URL validation with HEAD requests + - JSON-LD and Open Graph metadata extraction +- **SeedingConfig**: Configuration class for URL seeding operations + - Support for multiple discovery sources (`sitemap`, `cc`, `sitemap+cc`) + - Pattern-based URL filtering with wildcards + - Configurable concurrency and rate limiting + - Query-based relevance scoring with BM25 + - Score threshold filtering for quality control +- Comprehensive documentation for URL seeding feature + - Detailed comparison with deep crawling approaches + - Complete API reference with examples + - Integration guide with AsyncWebCrawler + - Performance benchmarks and best practices +- Example scripts demonstrating URL seeding: + - `url_seeder_demo.py`: Interactive Rich-based demonstration + - `url_seeder_quick_demo.py`: Screenshot-friendly examples +- Test suite for URL seeding with BM25 scoring + +### Changed +- Updated `__init__.py` to export AsyncUrlSeeder and SeedingConfig +- Enhanced documentation with URL seeding integration examples + +### Fixed +- Corrected examples to properly extract URLs from seeder results before passing to `arun_many()` +- Fixed logger color compatibility issue (changed `lightblack` to `bright_black`) + +## [0.6.2] - 2025-05-02 + +### Added +- New `RegexExtractionStrategy` for fast pattern-based extraction without requiring LLM + - Built-in patterns for emails, URLs, phone numbers, dates, and more + - Support for custom regex patterns + - `generate_pattern` utility for LLM-assisted pattern creation (one-time use) +- Added `fit_html` as a top-level field in `CrawlResult` for optimized HTML extraction +- Added support for network response body capture in network request tracking + +### Changed +- Updated documentation for no-LLM extraction strategies +- Enhanced API reference to include RegexExtractionStrategy examples and usage +- Improved HTML preprocessing with optimized performance for extraction strategies + +## [0.6.1] - 2025-04-24 + +### Added +- New dedicated `tables` field in `CrawlResult` model for better table extraction handling +- Updated crypto_analysis_example.py to use the new tables field with backward compatibility + +### Changed +- Improved playground UI in Docker deployment with better endpoint handling and UI feedback + +## [0.6.0] ‑ 2025‑04‑22 + +### Added +- Browser pooling with page pre‑warming and fine‑grained **geolocation, locale, and timezone** controls +- Crawler pool manager (SDK + Docker API) for smarter resource allocation +- Network & console log capture plus MHTML snapshot export +- **Table extractor**: turn HTML ``s into DataFrames or CSV with one flag +- High‑volume stress‑test framework in `tests/memory` and API load scripts +- MCP protocol endpoints with socket & SSE support; playground UI scaffold +- Docs v2 revamp: TOC, GitHub badge, copy‑code buttons, Docker API demo +- “Ask AI” helper button *(work‑in‑progress, shipping soon)* +- New examples: geo‑location usage, network/console capture, Docker API, markdown source selection, crypto analysis +- Expanded automated test suites for browser, Docker, MCP and memory benchmarks + +### Changed +- Consolidated and renamed browser strategies; legacy docker strategy modules removed +- `ProxyConfig` moved to `async_configs` +- Server migrated to pool‑based crawler management +- FastAPI validators replace custom query validation +- Docker build now uses Chromium base image +- Large‑scale repo tidy‑up (≈36 k insertions, ≈5 k deletions) + +### Fixed +- Async crawler session leak, duplicate‑visit handling, URL normalisation +- Target‑element regressions in scraping strategies +- Logged‑URL readability, encoded‑URL decoding, middle truncation for long URLs +- Closed issues: #701, #733, #756, #774, #804, #822, #839, #841, #842, #843, #867, #902, #911 + +### Removed +- Obsolete modules under `crawl4ai/browser/*` superseded by the new pooled browser layer + +### Deprecated +- Old markdown generator names now alias `DefaultMarkdownGenerator` and emit warnings + +--- + +#### Upgrade notes +1. Update any direct imports from `crawl4ai/browser/*` to the new pooled browser modules +2. If you override `AsyncPlaywrightCrawlerStrategy.get_page`, adopt the new signature +3. Rebuild Docker images to pull the new Chromium layer +4. Switch to `DefaultMarkdownGenerator` (or silence the deprecation warning) + +--- + +`121 files changed, ≈36 223 insertions, ≈4 975 deletions` :contentReference[oaicite:0]{index=0}​:contentReference[oaicite:1]{index=1} + + +### [Feature] 2025-04-21 +- Implemented MCP protocol for machine-to-machine communication + - Added WebSocket and SSE transport for MCP server + - Exposed server endpoints via MCP protocol + - Created tests for MCP socket and SSE communication +- Enhanced Docker server with file handling and intelligent search + - Added PDF and screenshot endpoints with file saving capability + - Added JavaScript execution endpoint for page interaction + - Implemented advanced context search with BM25 and code chunking + - Added file path output support for generated assets +- Improved server endpoints and API surface + - Added intelligent context search with query filtering + - Added syntax-aware code function chunking + - Implemented efficient HTML processing pipeline +- Added support for controlling browser geolocation via new GeolocationConfig class + - Added locale and timezone configuration options to CrawlerRunConfig + - Added example script demonstrating geolocation and locale usage + - Added documentation for location-based identity features + +### [Refactor] 2025-04-20 +- Replaced crawler_manager.py with simpler crawler_pool.py implementation +- Added global page semaphore for hard concurrency cap +- Implemented browser pool with idle cleanup +- Added playground UI for testing and stress testing +- Updated API handlers to use pooled crawlers +- Enhanced logging levels and symbols +- Added memory tests and stress test utilities + +### [Added] 2025-04-17 +- Added content source selection feature for markdown generation + - New `content_source` parameter allows choosing between `cleaned_html`, `raw_html`, and `fit_html` + - Provides flexibility in how HTML content is processed before markdown conversion + - Added examples and documentation for the new feature + - Includes backward compatibility with default `cleaned_html` behavior + +## Version 0.5.0.post5 (2025-03-14) + +### Added + +- *(crawler)* Add experimental parameters dictionary to CrawlerRunConfig to support beta features +- *(tables)* Add comprehensive table detection and extraction functionality with scoring system +- *(monitor)* Add real-time crawler monitoring system with memory management +- *(content)* Add target_elements parameter for selective content extraction +- *(browser)* Add standalone CDP browser launch capability +- *(schema)* Add preprocess_html_for_schema utility for better HTML cleaning +- *(api)* Add special handling for single URL requests in Docker API + +### Changed + +- *(filters)* Add reverse option to URLPatternFilter for inverting filter logic +- *(browser)* Make CSP nonce headers optional via experimental config +- *(browser)* Remove default cookie injection from page initialization +- *(crawler)* Optimize response handling for single-URL processing +- *(api)* Refactor crawl request handling to streamline processing +- *(config)* Update default provider to gpt-4o +- *(cache)* Change default cache_mode from aggressive to bypass in examples + +### Fixed + +- *(browser)* Clean up browser context creation code +- *(api)* Improve code formatting in API handler + +### Breaking Changes + +- WebScrapingStrategy no longer returns 'scraped_html' in its output dictionary +- Table extraction logic has been modified to better handle thead/tbody structures +- Default cookie injection has been removed from page initialization + +## Version 0.5.0 (2025-03-02) + +### Added + +- *(profiles)* Add BrowserProfiler class for dedicated browser profile management +- *(cli)* Add interactive profile management to CLI with rich UI +- *(profiles)* Add ability to crawl directly from profile management interface +- *(browser)* Support identity-based browsing with persistent profiles +- *(deep-crawling)* Add max_pages parameter to limit the number of pages crawled in all deep crawling strategies +- *(deep-crawling)* Add score_threshold parameter to BFS and DFS strategies to filter URLs by score + +### Changed + +- *(browser)* Refactor profile management from ManagedBrowser to BrowserProfiler class +- *(cli)* Enhance CLI with profile selection and status display for crawling +- *(examples)* Update identity-based browsing example to use BrowserProfiler class +- *(docs)* Update identity-based crawling documentation +- *(docs)* Update deep crawling documentation with max_pages and score_threshold parameters +- *(examples)* Add example demonstrating the use of max_pages and score_threshold parameters + +### Fixed + +- *(browser)* Fix profile detection and management on different platforms +- *(cli)* Fix CLI command structure for better user experience +- *(deep-crawling)* Improve BFS and DFS strategies to handle page count limits more efficiently + + +## Version 0.5.0 (2025-02-21) + +### Added + +- *(crawler)* [**breaking**] Add memory-adaptive dispatcher with rate limiting +- *(scraping)* [**breaking**] Add LXML-based scraping mode for improved performance +- *(content-filter)* Add LLMContentFilter for intelligent markdown generation +- *(dispatcher)* [**breaking**] Add streaming support for URL processing +- *(browser)* [**breaking**] Improve browser context management and add shared data support +- *(config)* [**breaking**] Add streaming support and config cloning +- *(crawler)* Add URL redirection tracking +- *(extraction)* Add LLM-powered schema generation utility +- *(proxy)* Add proxy configuration support to CrawlerRunConfig +- *(robots)* Add robots.txt compliance support +- *(release)* [**breaking**] Prepare v0.4.3 beta release +- *(proxy)* Add proxy rotation support and documentation +- *(browser)* Add CDP URL configuration support +- *(demo)* Uncomment feature demos and add fake-useragent dependency +- *(pdf)* Add PDF processing capabilities +- *(crawler)* [**breaking**] Enhance JavaScript execution and PDF processing +- *(docker)* Add Docker deployment configuration and API server +- *(docker)* Add Docker service integration and config serialization +- *(docker)* [**breaking**] Enhance Docker deployment setup and configuration +- *(api)* Improve cache handling and add API tests +- *(crawler)* [**breaking**] Add deep crawling capabilities with BFS strategy +- *(proxy)* [**breaking**] Add proxy rotation strategy +- *(deep-crawling)* Add DFS strategy and update exports; refactor CLI entry point +- *(cli)* Add command line interface with comprehensive features +- *(config)* Enhance serialization and add deep crawling exports +- *(crawler)* Add HTTP crawler strategy for lightweight web scraping +- *(docker)* [**breaking**] Implement supervisor and secure API endpoints +- *(docker)* [**breaking**] Add JWT authentication and improve server architecture + +### Changed + +- *(browser)* Update browser channel default to 'chromium' in BrowserConfig.from_args method +- *(crawler)* Optimize response handling and default settings +- *(crawler)* - Update hello_world example with proper content filtering +- - Update hello_world.py example +- *(docs)* [**breaking**] Reorganize documentation structure and update styles +- *(dispatcher)* [**breaking**] Migrate to modular dispatcher system with enhanced monitoring +- *(scraping)* [**breaking**] Replace ScrapingMode enum with strategy pattern +- *(browser)* Improve browser path management +- *(models)* Rename final_url to redirected_url for consistency +- *(core)* [**breaking**] Improve type hints and remove unused file +- *(docs)* Improve code formatting in features demo +- *(user-agent)* Improve user agent generation system +- *(core)* [**breaking**] Reorganize project structure and remove legacy code +- *(docker)* Clean up import statements in server.py +- *(docker)* Remove unused models and utilities for cleaner codebase +- *(docker)* [**breaking**] Improve server architecture and configuration +- *(deep-crawl)* [**breaking**] Reorganize deep crawling functionality into dedicated module +- *(deep-crawling)* [**breaking**] Reorganize deep crawling strategies and add new implementations +- *(crawling)* [**breaking**] Improve type hints and code cleanup +- *(crawler)* [**breaking**] Improve HTML handling and cleanup codebase +- *(crawler)* [**breaking**] Remove content filter functionality +- *(examples)* Update API usage in features demo +- *(config)* [**breaking**] Enhance serialization and config handling + +### Docs + +- Add Code of Conduct for the project (#410) + +### Documentation + +- *(extraction)* Add clarifying comments for CSS selector behavior +- *(readme)* Update personal story and project vision +- *(urls)* [**breaking**] Update documentation URLs to new domain +- *(api)* Add streaming mode documentation and examples +- *(readme)* Update version and feature announcements for v0.4.3b1 +- *(examples)* Update demo scripts and fix output formats +- *(examples)* Update v0.4.3 features demo to v0.4.3b2 +- *(readme)* Update version references and fix links +- *(multi-url)* [**breaking**] Improve documentation clarity and update examples +- *(examples)* Update proxy rotation demo and disable other demos +- *(api)* Improve formatting and readability of API documentation +- *(examples)* Add SERP API project example +- *(urls)* Update documentation URLs to new domain +- *(readme)* Resolve merge conflict and update version info + +### Fixed + +- *(browser)* Update default browser channel to chromium and simplify channel selection logic +- *(browser)* [**breaking**] Default to Chromium channel for new headless mode (#387) +- *(browser)* Resolve merge conflicts in browser channel configuration +- Prevent memory leaks by ensuring proper closure of Playwright pages +- Not working long page screenshot (#403) +- *(extraction)* JsonCss selector and crawler improvements +- *(models)* [**breaking**] Make model fields optional with default values +- *(dispatcher)* Adjust memory threshold and fix dispatcher initialization +- *(install)* Ensure proper exit after running doctor command + +### Miscellaneous Tasks + +- *(cleanup)* Remove unused files and improve type hints +- Add .gitattributes file + +## License Update + +Crawl4AI v0.5.0 updates the license to Apache 2.0 *with a required attribution clause*. This means you are free to use, modify, and distribute Crawl4AI (even commercially), but you *must* clearly attribute the project in any public use or distribution. See the updated `LICENSE` file for the full legal text and specific requirements. + +--- + +## Version 0.4.3b2 (2025-01-21) + +This release introduces several powerful new features, including robots.txt compliance, dynamic proxy support, LLM-powered schema generation, and improved documentation. + +### Features + +- **Robots.txt Compliance:** + - Added robots.txt compliance support with efficient SQLite-based caching. + - New `check_robots_txt` parameter in `CrawlerRunConfig` to enable robots.txt checking before crawling a URL. + - Automated robots.txt checking is now integrated into `AsyncWebCrawler` with 403 status codes for blocked URLs. + +- **Proxy Configuration:** + - Added proxy configuration support to `CrawlerRunConfig`, allowing dynamic proxy settings per crawl request. + - Updated documentation with examples for using proxy configuration in crawl operations. + +- **LLM-Powered Schema Generation:** + - Introduced a new utility for automatic CSS and XPath schema generation using OpenAI or Ollama models. + - Added comprehensive documentation and examples for schema generation. + - New prompt templates optimized for HTML schema analysis. + +- **URL Redirection Tracking:** + - Added URL redirection tracking to capture the final URL after any redirects. + - The final URL is now available in the `redirected_url` field of the `AsyncCrawlResponse` object. + +- **Enhanced Streamlined Documentation:** + - Refactored and improved the documentation structure for clarity and ease of use. + - Added detailed explanations of new features and updated examples. + +- **Improved Browser Context Management:** + - Enhanced the management of browser contexts and added shared data support. + - Introduced the `shared_data` parameter in `CrawlerRunConfig` to pass data between hooks. + +- **Memory Dispatcher System:** + - Migrated to a memory dispatcher system with enhanced monitoring capabilities. + - Introduced `MemoryAdaptiveDispatcher` and `SemaphoreDispatcher` for improved resource management. + - Added `RateLimiter` for rate limiting support. + - New `CrawlerMonitor` for real-time monitoring of crawler operations. + +- **Streaming Support:** + - Added streaming support for processing crawled URLs as they are processed. + - Enabled streaming mode with the `stream` parameter in `CrawlerRunConfig`. + +- **Content Scraping Strategy:** + - Introduced a new `LXMLWebScrapingStrategy` for faster content scraping. + - Added support for selecting the scraping strategy via the `scraping_strategy` parameter in `CrawlerRunConfig`. + +### Bug Fixes + +- **Browser Path Management:** + - Improved browser path management for consistent behavior across different environments. + +- **Memory Threshold:** + - Adjusted the default memory threshold to improve resource utilization. + +- **Pydantic Model Fields:** + - Made several model fields optional with default values to improve flexibility. + +### Refactor + +- **Documentation Structure:** + - Reorganized documentation structure to improve navigation and readability. + - Updated styles and added new sections for advanced features. + +- **Scraping Mode:** + - Replaced the `ScrapingMode` enum with a strategy pattern for more flexible content scraping. + +- **Version Update:** + - Updated the version to `0.4.248`. + +- **Code Cleanup:** + - Removed unused files and improved type hints. + - Applied Ruff corrections for code quality. + +- **Updated dependencies:** + - Updated dependencies to their latest versions to ensure compatibility and security. + +- **Ignored certain patterns and directories:** + - Updated `.gitignore` and `.codeiumignore` to ignore additional patterns and directories, streamlining the development environment. + +- **Simplified Personal Story in README:** + - Streamlined the personal story and project vision in the `README.md` for clarity. + +- **Removed Deprecated Files:** + - Deleted several deprecated files and examples that are no longer relevant. + +--- +**Previous Releases:** + +### 0.4.24x (2024-12-31) +- **Enhanced SSL & Security**: New SSL certificate handling with custom paths and validation options for secure crawling. +- **Smart Content Filtering**: Advanced filtering system with regex support and efficient chunking strategies. +- **Improved JSON Extraction**: Support for complex JSONPath, JSON-CSS, and Microdata extraction. +- **New Field Types**: Added `computed`, `conditional`, `aggregate`, and `template` field types. +- **Performance Boost**: Optimized caching, parallel processing, and memory management. +- **Better Error Handling**: Enhanced debugging capabilities with detailed error tracking. +- **Security Features**: Improved input validation and safe expression evaluation. + +### 0.4.247 (2025-01-06) + +#### Added +- **Windows Event Loop Configuration**: Introduced a utility function `configure_windows_event_loop` to resolve `NotImplementedError` for asyncio subprocesses on Windows. ([#utils.py](crawl4ai/utils.py), [#tutorials/async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md)) +- **`page_need_scroll` Method**: Added a method to determine if a page requires scrolling before taking actions in `AsyncPlaywrightCrawlerStrategy`. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py)) + +#### Changed +- **Version Bump**: Updated the version from `0.4.246` to `0.4.247`. ([#__version__.py](crawl4ai/__version__.py)) +- **Improved Scrolling Logic**: Enhanced scrolling methods in `AsyncPlaywrightCrawlerStrategy` by adding a `scroll_delay` parameter for better control. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py)) +- **Markdown Generation Example**: Updated the `hello_world.py` example to reflect the latest API changes and better illustrate features. ([#examples/hello_world.py](docs/examples/hello_world.py)) +- **Documentation Update**: + - Added Windows-specific instructions for handling asyncio event loops. ([#async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md)) + +#### Removed +- **Legacy Markdown Generation Code**: Removed outdated and unused code for markdown generation in `content_scraping_strategy.py`. ([#content_scraping_strategy.py](crawl4ai/content_scraping_strategy.py)) + +#### Fixed +- **Page Closing to Prevent Memory Leaks**: + - **Description**: Added a `finally` block to ensure pages are closed when no `session_id` is provided. + - **Impact**: Prevents memory leaks caused by lingering pages after a crawl. + - **File**: [`async_crawler_strategy.py`](crawl4ai/async_crawler_strategy.py) + - **Code**: + ```python + finally: + # If no session_id is given we should close the page + if not config.session_id: + await page.close() + ``` +- **Multiple Element Selection**: Modified `_get_elements` in `JsonCssExtractionStrategy` to return all matching elements instead of just the first one, ensuring comprehensive extraction. ([#extraction_strategy.py](crawl4ai/extraction_strategy.py)) +- **Error Handling in Scrolling**: Added robust error handling to ensure scrolling proceeds safely even if a configuration is missing. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py)) + +## [0.4.267] - 2025 - 01 - 06 + +### Added +- **Windows Event Loop Configuration**: Introduced a utility function `configure_windows_event_loop` to resolve `NotImplementedError` for asyncio subprocesses on Windows. ([#utils.py](crawl4ai/utils.py), [#tutorials/async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md)) +- **`page_need_scroll` Method**: Added a method to determine if a page requires scrolling before taking actions in `AsyncPlaywrightCrawlerStrategy`. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py)) + +## [0.4.24] - 2024-12-31 + +### Added +- **Browser and SSL Handling** + - SSL certificate validation options in extraction strategies + - Custom certificate paths support + - Configurable certificate validation skipping + - Enhanced response status code handling with retry logic + +- **Content Processing** + - New content filtering system with regex support + - Advanced chunking strategies for large content + - Memory-efficient parallel processing + - Configurable chunk size optimization + +- **JSON Extraction** + - Complex JSONPath expression support + - JSON-CSS and Microdata extraction + - RDFa parsing capabilities + - Advanced data transformation pipeline + +- **Field Types** + - New field types: `computed`, `conditional`, `aggregate`, `template` + - Field inheritance system + - Reusable field definitions + - Custom validation rules + +### Changed +- **Performance** + - Optimized selector compilation with caching + - Improved HTML parsing efficiency + - Enhanced memory management for large documents + - Batch processing optimizations + +- **Error Handling** + - More detailed error messages and categorization + - Enhanced debugging capabilities + - Improved performance metrics tracking + - Better error recovery mechanisms + +### Deprecated +- Old field computation method using `eval` +- Direct browser manipulation without proper SSL handling +- Simple text-based content filtering + +### Removed +- Legacy extraction patterns without proper error handling +- Unsafe eval-based field computation +- Direct DOM manipulation without sanitization + +### Fixed +- Memory leaks in large document processing +- SSL certificate validation issues +- Incorrect handling of nested JSON structures +- Performance bottlenecks in parallel processing + +### Security +- Improved input validation and sanitization +- Safe expression evaluation system +- Enhanced resource protection +- Rate limiting implementation + +## [0.4.1] - 2024-12-08 + +### **File: `crawl4ai/async_crawler_strategy.py`** + +#### **New Parameters and Attributes Added** +- **`text_mode` (boolean)**: Enables text-only mode, disables images, JavaScript, and GPU-related features for faster, minimal rendering. +- **`light_mode` (boolean)**: Optimizes the browser by disabling unnecessary background processes and features for efficiency. +- **`viewport_width` and `viewport_height`**: Dynamically adjusts based on `text_mode` mode (default values: 800x600 for `text_mode`, 1920x1080 otherwise). +- **`extra_args`**: Adds browser-specific flags for `text_mode` mode. +- **`adjust_viewport_to_content`**: Dynamically adjusts the viewport to the content size for accurate rendering. + +#### **Browser Context Adjustments** +- Added **`viewport` adjustments**: Dynamically computed based on `text_mode` or custom configuration. +- Enhanced support for `light_mode` and `text_mode` by adding specific browser arguments to reduce resource consumption. + +#### **Dynamic Content Handling** +- **Full Page Scan Feature**: + - Scrolls through the entire page while dynamically detecting content changes. + - Ensures scrolling stops when no new dynamic content is loaded. + +#### **Session Management** +- Added **`create_session`** method: + - Creates a new browser session and assigns a unique ID. + - Supports persistent and non-persistent contexts with full compatibility for cookies, headers, and proxies. + +#### **Improved Content Loading and Adjustment** +- **`adjust_viewport_to_content`**: + - Automatically adjusts viewport to match content dimensions. + - Includes scaling via Chrome DevTools Protocol (CDP). +- Enhanced content loading: + - Waits for images to load and ensures network activity is idle before proceeding. + +#### **Error Handling and Logging** +- Improved error handling and detailed logging for: + - Viewport adjustment (`adjust_viewport_to_content`). + - Full page scanning (`scan_full_page`). + - Dynamic content loading. + +#### **Refactoring and Cleanup** +- Removed hardcoded viewport dimensions in multiple places, replaced with dynamic values (`self.viewport_width`, `self.viewport_height`). +- Removed commented-out and unused code for better readability. +- Added default value for `delay_before_return_html` parameter. + +#### **Optimizations** +- Reduced resource usage in `light_mode` by disabling unnecessary browser features such as extensions, background timers, and sync. +- Improved compatibility for different browser types (`chrome`, `firefox`, `webkit`). + +--- + +### **File: `docs/examples/quickstart_async.py`** + +#### **Schema Adjustment** +- Changed schema reference for `LLMExtractionStrategy`: + - **Old**: `OpenAIModelFee.schema()` + - **New**: `OpenAIModelFee.model_json_schema()` + - This likely ensures better compatibility with the `OpenAIModelFee` class and its JSON schema. + +#### **Documentation Comments Updated** +- Improved extraction instruction for schema-based LLM strategies. + +--- + +### **New Features Added** +1. **Text-Only Mode**: + - Focuses on minimal resource usage by disabling non-essential browser features. +2. **Light Mode**: + - Optimizes browser for performance by disabling background tasks and unnecessary services. +3. **Full Page Scanning**: + - Ensures the entire content of a page is crawled, including dynamic elements loaded during scrolling. +4. **Dynamic Viewport Adjustment**: + - Automatically resizes the viewport to match content dimensions, improving compatibility and rendering accuracy. +5. **Session Management**: + - Simplifies session handling with better support for persistent and non-persistent contexts. + +--- + +### **Bug Fixes** +- Fixed potential viewport mismatches by ensuring consistent use of `self.viewport_width` and `self.viewport_height` throughout the code. +- Improved robustness of dynamic content loading to avoid timeouts and failed evaluations. + +## [0.3.75] December 1, 2024 + +### PruningContentFilter + +#### 1. Introduced PruningContentFilter (Dec 01, 2024) (Dec 01, 2024) +A new content filtering strategy that removes less relevant nodes based on metrics like text and link density. + +**Affected Files:** +- `crawl4ai/content_filter_strategy.py`: Enhancement of content filtering capabilities. +```diff +Implemented effective pruning algorithm with comprehensive scoring. +``` +- `README.md`: Improved documentation regarding new features. +```diff +Updated to include usage and explanation for the PruningContentFilter. +``` +- `docs/md_v2/basic/content_filtering.md`: Expanded documentation for users. +```diff +Added detailed section explaining the PruningContentFilter. +``` + +#### 2. Added Unit Tests for PruningContentFilter (Dec 01, 2024) (Dec 01, 2024) +Comprehensive tests added to ensure correct functionality of PruningContentFilter + +**Affected Files:** +- `tests/async/test_content_filter_prune.py`: Increased test coverage for content filtering strategies. +```diff +Created test cases for various scenarios using the PruningContentFilter. +``` + +### Development Updates + +#### 3. Enhanced BM25ContentFilter tests (Dec 01, 2024) (Dec 01, 2024) +Extended testing to cover additional edge cases and performance metrics. + +**Affected Files:** +- `tests/async/test_content_filter_bm25.py`: Improved reliability and performance assurance. +```diff +Added tests for new extraction scenarios including malformed HTML. +``` + +### Infrastructure & Documentation + +#### 4. Updated Examples (Dec 01, 2024) (Dec 01, 2024) +Altered examples in documentation to promote the use of PruningContentFilter alongside existing strategies. + +**Affected Files:** +- `docs/examples/quickstart_async.py`: Enhanced usability and clarity for new users. +- Revised example to illustrate usage of PruningContentFilter. + +## [0.3.746] November 29, 2024 + +### Major Features +1. Enhanced Docker Support (Nov 29, 2024) + - Improved GPU support in Docker images. + - Dockerfile refactored for better platform-specific installations. + - Introduced new Docker commands for different platforms: + - `basic-amd64`, `all-amd64`, `gpu-amd64` for AMD64. + - `basic-arm64`, `all-arm64`, `gpu-arm64` for ARM64. + +### Infrastructure & Documentation +- Enhanced README.md to improve user guidance and installation instructions. +- Added installation instructions for Playwright setup in README. +- Created and updated examples in `docs/examples/quickstart_async.py` to be more useful and user-friendly. +- Updated `requirements.txt` with a new `pydantic` dependency. +- Bumped version number in `crawl4ai/__version__.py` to 0.3.746. + +### Breaking Changes +- Streamlined application structure: + - Removed static pages and related code from `main.py` which might affect existing deployments relying on static content. + +### Development Updates +- Developed `post_install` method in `crawl4ai/install.py` to streamline post-installation setup tasks. +- Refined migration processes in `crawl4ai/migrations.py` with enhanced logging for better error visibility. +- Updated `docker-compose.yml` to support local and hub services for different architectures, enhancing build and deploy capabilities. +- Refactored example test cases in `docs/examples/docker_example.py` to facilitate comprehensive testing. + +### README.md +Updated README with new docker commands and setup instructions. +Enhanced installation instructions and guidance. + +### crawl4ai/install.py +Added post-install script functionality. +Introduced `post_install` method for automation of post-installation tasks. + +### crawl4ai/migrations.py +Improved migration logging. +Refined migration processes and added better logging. + +### docker-compose.yml +Refactored docker-compose for better service management. +Updated to define services for different platforms and versions. + +### requirements.txt +Updated dependencies. +Added `pydantic` to requirements file. + +### crawler/__version__.py +Updated version number. +Bumped version number to 0.3.746. + +### docs/examples/quickstart_async.py +Enhanced example scripts. +Uncommented example usage in async guide for user functionality. + +### main.py +Refactored code to improve maintainability. +Streamlined app structure by removing static pages code. + +## [0.3.743] November 27, 2024 + +Enhance features and documentation +- Updated version to 0.3.743 +- Improved ManagedBrowser configuration with dynamic host/port +- Implemented fast HTML formatting in web crawler +- Enhanced markdown generation with a new generator class +- Improved sanitization and utility functions +- Added contributor details and pull request acknowledgments +- Updated documentation for clearer usage scenarios +- Adjusted tests to reflect class name changes + +### CONTRIBUTORS.md +Added new contributors and pull request details. +Updated community contributions and acknowledged pull requests. + +### crawl4ai/__version__.py +Version update. +Bumped version to 0.3.743. + +### crawl4ai/async_crawler_strategy.py +Improved ManagedBrowser configuration. +Enhanced browser initialization with configurable host and debugging port; improved hook execution. + +### crawl4ai/async_webcrawler.py +Optimized HTML processing. +Implemented 'fast_format_html' for optimized HTML formatting; applied it when 'prettiify' is enabled. + +### crawl4ai/content_scraping_strategy.py +Enhanced markdown generation strategy. +Updated to use DefaultMarkdownGenerator and improved markdown generation with filters option. + +### crawl4ai/markdown_generation_strategy.py +Refactored markdown generation class. +Renamed DefaultMarkdownGenerationStrategy to DefaultMarkdownGenerator; added content filter handling. + +### crawl4ai/utils.py +Enhanced utility functions. +Improved input sanitization and enhanced HTML formatting method. + +### docs/md_v2/advanced/hooks-auth.md +Improved documentation for hooks. +Updated code examples to include cookies in crawler strategy initialization. + +### tests/async/test_markdown_genertor.py +Refactored tests to match class renaming. +Updated tests to use renamed DefaultMarkdownGenerator class. + +## [0.3.74] November 17, 2024 + +This changelog details the updates and changes introduced in Crawl4AI version 0.3.74. It's designed to inform developers about new features, modifications to existing components, removals, and other important information. + +### 1. File Download Processing + +- Users can now specify download folders using the `downloads_path` parameter in the `AsyncWebCrawler` constructor or the `arun` method. If not specified, downloads are saved to a "downloads" folder within the `.crawl4ai` directory. +- File download tracking is integrated into the `CrawlResult` object. Successfully downloaded files are listed in the `downloaded_files` attribute, providing their paths. +- Added `accept_downloads` parameter to the crawler strategies (defaults to `False`). If set to True you can add JS code and `wait_for` parameter for file download. + +**Example:** + +```python +import asyncio +import os +from pathlib import Path +from crawl4ai import AsyncWebCrawler + +async def download_example(): + downloads_path = os.path.join(Path.home(), ".crawl4ai", "downloads") + os.makedirs(downloads_path, exist_ok=True) + + async with AsyncWebCrawler( + accept_downloads=True, + downloads_path=downloads_path, + verbose=True + ) as crawler: + result = await crawler.arun( + url="https://www.python.org/downloads/", + js_code=""" + const downloadLink = document.querySelector('a[href$=".exe"]'); + if (downloadLink) { downloadLink.click(); } + """, + wait_for=5 # To ensure download has started + ) + + if result.downloaded_files: + print("Downloaded files:") + for file in result.downloaded_files: + print(f"- {file}") + +asyncio.run(download_example()) + +``` + +### 2. Refined Content Filtering + +- Introduced the `RelevanceContentFilter` strategy (and its implementation `BM25ContentFilter`) for extracting relevant content from web pages, replacing Fit Markdown and other content cleaning strategy. This new strategy leverages the BM25 algorithm to identify chunks of text relevant to the page's title, description, keywords, or a user-provided query. +- The `fit_markdown` flag in the content scraper is used to filter content based on title, meta description, and keywords. + +**Example:** + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.content_filter_strategy import BM25ContentFilter + +async def filter_content(url, query): + async with AsyncWebCrawler() as crawler: + content_filter = BM25ContentFilter(user_query=query) + result = await crawler.arun(url=url, extraction_strategy=content_filter, fit_markdown=True) + print(result.extracted_content) # Or result.fit_markdown for the markdown version + print(result.fit_html) # Or result.fit_html to show HTML with only the filtered content + +asyncio.run(filter_content("https://en.wikipedia.org/wiki/Apple", "fruit nutrition health")) +``` + +### 3. Raw HTML and Local File Support + +- Added support for crawling local files and raw HTML content directly. +- Use the `file://` prefix for local file paths. +- Use the `raw:` prefix for raw HTML strings. + +**Example:** + +```python +async def crawl_local_or_raw(crawler, content, content_type): + prefix = "file://" if content_type == "local" else "raw:" + url = f"{prefix}{content}" + result = await crawler.arun(url=url) + if result.success: + print(f"Markdown Content from {content_type.title()} Source:") + print(result.markdown) + +# Example usage with local file and raw HTML +async def main(): + async with AsyncWebCrawler() as crawler: + # Local File + await crawl_local_or_raw( + crawler, os.path.abspath('tests/async/sample_wikipedia.html'), "local" + ) + # Raw HTML + await crawl_raw_html(crawler, "

Raw Test

This is raw HTML.

") + + +asyncio.run(main()) +``` + +### 4. Browser Management + +- New asynchronous crawler strategy implemented using Playwright. +- `ManagedBrowser` class introduced for improved browser session handling, offering features like persistent browser sessions between requests (using `session_id` parameter) and browser process monitoring. +- Updated to tf-playwright-stealth for enhanced stealth capabilities. +- Added `use_managed_browser`, `use_persistent_context`, and `chrome_channel` parameters to AsyncPlaywrightCrawlerStrategy. + + +**Example:** +```python +async def browser_management_demo(): + user_data_dir = os.path.join(Path.home(), ".crawl4ai", "user-data-dir") + os.makedirs(user_data_dir, exist_ok=True) # Ensure directory exists + async with AsyncWebCrawler( + use_managed_browser=True, + user_data_dir=user_data_dir, + use_persistent_context=True, + verbose=True + ) as crawler: + result1 = await crawler.arun( + url="https://example.com", session_id="my_session" + ) + result2 = await crawler.arun( + url="https://example.com/anotherpage", session_id="my_session" + ) + +asyncio.run(browser_management_demo()) +``` + + +### 5. API Server & Cache Improvements + +- Added CORS support to API server. +- Implemented static file serving. +- Enhanced root redirect functionality. +- Cache database updated to store response headers and downloaded files information. It utilizes a file system approach to manage large content efficiently. +- New, more efficient caching database built using xxhash and file system approach. +- Introduced `CacheMode` enum (`ENABLED`, `DISABLED`, `READ_ONLY`, `WRITE_ONLY`, `BYPASS`) and `always_bypass_cache` parameter in AsyncWebCrawler for fine-grained cache control. This replaces `bypass_cache`, `no_cache_read`, `no_cache_write`, and `always_by_pass_cache`. + + +### 🗑️ Removals + +- Removed deprecated: `crawl4ai/content_cleaning_strategy.py`. +- Removed internal class ContentCleaningStrategy +- Removed legacy cache control flags: `bypass_cache`, `disable_cache`, `no_cache_read`, `no_cache_write`, and `always_by_pass_cache`. These have been superseded by `cache_mode`. + + +### ⚙️ Other Changes + +- Moved version file to `crawl4ai/__version__.py`. +- Added `crawl4ai/cache_context.py`. +- Added `crawl4ai/version_manager.py`. +- Added `crawl4ai/migrations.py`. +- Added `crawl4ai-migrate` entry point. +- Added config `NEED_MIGRATION` and `SHOW_DEPRECATION_WARNINGS`. +- API server now requires an API token for authentication, configurable with the `CRAWL4AI_API_TOKEN` environment variable. This enhances API security. +- Added synchronous crawl endpoint `/crawl_sync` for immediate result retrieval, and direct crawl endpoint `/crawl_direct` bypassing the task queue. + + +### ⚠️ Deprecation Notices + +- The synchronous version of `WebCrawler` is being phased out. While still available via `crawl4ai[sync]`, it will eventually be removed. Transition to `AsyncWebCrawler` is strongly recommended. Boolean cache control flags in `arun` are also deprecated, migrate to using the `cache_mode` parameter. See examples in the "New Features" section above for correct usage. + + +### 🐛 Bug Fixes + +- Resolved issue with browser context closing unexpectedly in Docker. This significantly improves stability, particularly within containerized environments. +- Fixed memory leaks associated with incorrect asynchronous cleanup by removing the `__del__` method and ensuring the browser context is closed explicitly using context managers. +- Improved error handling in `WebScrapingStrategy`. More detailed error messages and suggestions for debugging will minimize frustration when running into unexpected issues. +- Fixed issue with incorrect text parsing in specific HTML structures. + + +### Example of migrating to the new CacheMode: + +**Old way:** + +```python +crawler = AsyncWebCrawler(always_by_pass_cache=True) +result = await crawler.arun(url="https://example.com", bypass_cache=True) +``` + +**New way:** + +```python +from crawl4ai import CacheMode + +crawler = AsyncWebCrawler(always_bypass_cache=True) +result = await crawler.arun(url="https://example.com", cache_mode=CacheMode.BYPASS) +``` + + +## [0.3.74] - November 13, 2024 + +1. **File Download Processing** (Nov 14, 2024) + - Added capability for users to specify download folders + - Implemented file download tracking in crowd result object + - Created new file: `tests/async/test_async_doanloader.py` + +2. **Content Filtering Improvements** (Nov 14, 2024) + - Introduced Relevance Content Filter as an improvement over Fit Markdown + - Implemented BM25 algorithm for content relevance matching + - Added new file: `crawl4ai/content_filter_strategy.py` + - Removed deprecated: `crawl4ai/content_cleaning_strategy.py` + +3. **Local File and Raw HTML Support** (Nov 13, 2024) + - Added support for processing local files + - Implemented raw HTML input handling in AsyncWebCrawler + - Enhanced `crawl4ai/async_webcrawler.py` with significant performance improvements + +4. **Browser Management Enhancements** (Nov 12, 2024) + - Implemented new async crawler strategy using Playwright + - Introduced ManagedBrowser for better browser session handling + - Added support for persistent browser sessions + - Updated from playwright_stealth to tf-playwright-stealth + +5. **API Server Component** + - Added CORS support + - Implemented static file serving + - Enhanced root redirect functionality + + + +## [0.3.731] - November 13, 2024 + +### Added +- Support for raw HTML and local file crawling via URL prefixes ('raw:', 'file://') +- Browser process monitoring for managed browser instances +- Screenshot capability for raw HTML and local file content +- Response headers storage in cache database +- New `fit_markdown` flag for optional markdown generation + +### Changed +- Switched HTML parser from 'html.parser' to 'lxml' for ~4x performance improvement +- Optimized BeautifulSoup text conversion and element selection +- Pre-compiled regular expressions for better performance +- Improved metadata extraction efficiency +- Response headers now stored alongside HTML in cache + +### Removed +- `__del__` method from AsyncPlaywrightCrawlerStrategy to prevent async cleanup issues + +### Fixed +- Issue #256: Added support for crawling raw HTML content +- Issue #253: Implemented file:// protocol handling +- Missing response headers in cached results +- Memory leaks from improper async cleanup + +## [v0.3.731] - 2024-11-13 Changelog for Issue 256 Fix +- Fixed: Browser context unexpectedly closing in Docker environment during crawl operations. +- Removed: __del__ method from AsyncPlaywrightCrawlerStrategy to prevent unreliable asynchronous cleanup, ensuring - browser context is closed explicitly within context managers. +- Added: Monitoring for ManagedBrowser subprocess to detect and log unexpected terminations. +- Updated: Dockerfile configurations to expose debugging port (9222) and allocate additional shared memory for improved browser stability. +- Improved: Error handling and resource cleanup processes for browser lifecycle management within the Docker environment. + +## [v0.3.73] - 2024-11-05 + +### Major Features +- **New Doctor Feature** + - Added comprehensive system diagnostics tool + - Available through package hub and CLI + - Provides automated troubleshooting and system health checks + - Includes detailed reporting of configuration issues + +- **Dockerized API Server** + - Released complete Docker implementation for API server + - Added comprehensive documentation for Docker deployment + - Implemented container communication protocols + - Added environment configuration guides + +- **Managed Browser Integration** + - Added support for user-controlled browser instances + - Implemented `ManagedBrowser` class for better browser lifecycle management + - Added ability to connect to existing Chrome DevTools Protocol (CDP) endpoints + - Introduced user data directory support for persistent browser profiles + +- **Enhanced HTML Processing** + - Added HTML tag preservation feature during markdown conversion + - Introduced configurable tag preservation system + - Improved pre-tag and code block handling + - Added support for nested preserved tags with attribute retention + +### Improvements +- **Browser Handling** + - Added flag to ignore body visibility for problematic pages + - Improved browser process cleanup and management + - Enhanced temporary directory handling for browser profiles + - Added configurable browser launch arguments + +- **Database Management** + - Implemented connection pooling for better performance + - Added retry logic for database operations + - Improved error handling and logging + - Enhanced cleanup procedures for database connections + +- **Resource Management** + - Added memory and CPU monitoring + - Implemented dynamic task slot allocation based on system resources + - Added configurable cleanup intervals + +### Technical Improvements +- **Code Structure** + - Moved version management to dedicated _version.py file + - Improved error handling throughout the codebase + - Enhanced logging system with better error reporting + - Reorganized core components for better maintainability + +### Bug Fixes +- Fixed issues with browser process termination +- Improved handling of connection timeouts +- Enhanced error recovery in database operations +- Fixed memory leaks in long-running processes + +### Dependencies +- Updated Playwright to v1.47 +- Updated core dependencies with more flexible version constraints +- Added new development dependencies for testing + +### Breaking Changes +- Changed default browser handling behavior +- Modified database connection management approach +- Updated API response structure for better consistency + +### Migration Guide +When upgrading to v0.3.73, be aware of the following changes: + +1. Docker Deployment: + - Review Docker documentation for new deployment options + - Update environment configurations as needed + - Check container communication settings + +2. If using custom browser management: + - Update browser initialization code to use new ManagedBrowser class + - Review browser cleanup procedures + +3. For database operations: + - Check custom database queries for compatibility with new connection pooling + - Update error handling to work with new retry logic + +4. Using the Doctor: + - Run doctor command for system diagnostics: `crawl4ai doctor` + - Review generated reports for potential issues + - Follow recommended fixes for any identified problems + + +## [v0.3.73] - 2024-11-04 +This commit introduces several key enhancements, including improved error handling and robust database operations in `async_database.py`, which now features a connection pool and retry logic for better reliability. Updates to the README.md provide clearer instructions and a better user experience with links to documentation sections. The `.gitignore` file has been refined to include additional directories, while the async web crawler now utilizes a managed browser for more efficient crawling. Furthermore, multiple dependency updates and introduction of the `CustomHTML2Text` class enhance text extraction capabilities. + +## [v0.3.73] - 2024-10-24 + +### Added +- preserve_tags: Added support for preserving specific HTML tags during markdown conversion. +- Smart overlay removal system in AsyncPlaywrightCrawlerStrategy: + - Automatic removal of popups, modals, and cookie notices + - Detection and removal of fixed/sticky position elements + - Cleaning of empty block elements + - Configurable via `remove_overlay_elements` parameter +- Enhanced screenshot capabilities: + - Added `screenshot_wait_for` parameter to control timing + - Improved screenshot handling with existing page context + - Better error handling with fallback error images +- New URL normalization utilities: + - `normalize_url` function for consistent URL formatting + - `is_external_url` function for better link classification +- Custom base directory support for cache storage: + - New `base_directory` parameter in AsyncWebCrawler + - Allows specifying alternative locations for `.crawl4ai` folder + +### Enhanced +- Link handling improvements: + - Better duplicate link detection + - Enhanced internal/external link classification + - Improved handling of special URL protocols + - Support for anchor links and protocol-relative URLs +- Configuration refinements: + - Streamlined social media domain list + - More focused external content filtering +- LLM extraction strategy: + - Added support for separate API base URL via `api_base` parameter + - Better handling of base URLs in configuration + +### Fixed +- Screenshot functionality: + - Resolved issues with screenshot timing and context + - Improved error handling and recovery +- Link processing: + - Fixed URL normalization edge cases + - Better handling of invalid URLs + - Improved error messages for link processing failures + +### Developer Notes +- The overlay removal system uses advanced JavaScript injection for better compatibility +- URL normalization handles special cases like mailto:, tel:, and protocol-relative URLs +- Screenshot system now reuses existing page context for better performance +- Link processing maintains separate dictionaries for internal and external links to ensure uniqueness + +## [v0.3.72] - 2024-10-22 + +### Added +- New `ContentCleaningStrategy` class: + - Smart content extraction based on text density and element scoring + - Automatic removal of boilerplate content + - DOM tree analysis for better content identification + - Configurable thresholds for content detection +- Advanced proxy support: + - Added `proxy_config` option for authenticated proxy connections + - Support for username/password in proxy configuration +- New content output formats: + - `fit_markdown`: Optimized markdown output with main content focus + - `fit_html`: Clean HTML with only essential content + +### Enhanced +- Image source detection: + - Support for multiple image source attributes (`src`, `data-src`, `srcset`, etc.) + - Automatic fallback through potential source attributes + - Smart handling of srcset attribute +- External content handling: + - Made external link exclusion optional (disabled by default) + - Improved detection and handling of social media links + - Better control over external image filtering + +### Fixed +- Image extraction reliability with multiple source attribute checks +- External link and image handling logic for better accuracy + +### Developer Notes +- The new `ContentCleaningStrategy` uses configurable thresholds for customization +- Proxy configuration now supports more complex authentication scenarios +- Content extraction process now provides both regular and optimized outputs + +## [v0.3.72] - 2024-10-20 + +### Fixed +- Added support for parsing Base64 encoded images in WebScrapingStrategy + +### Added +- Forked and integrated a customized version of the html2text library for more control over Markdown generation +- New configuration options for controlling external content: + - Ability to exclude all external links + - Option to specify domains to exclude (default includes major social media platforms) + - Control over excluding external images + +### Changed +- Improved Markdown generation process: + - Added fine-grained control over character escaping in Markdown output + - Enhanced handling of code blocks and pre-formatted text +- Updated `AsyncPlaywrightCrawlerStrategy.close()` method to use a shorter sleep time (0.5 seconds instead of 500) +- Enhanced flexibility in `CosineStrategy` with a more generic `load_HF_embedding_model` function + +### Improved +- Optimized content scraping and processing for better efficiency +- Enhanced error handling and logging in various components + +### Developer Notes +- The customized html2text library is now located within the crawl4ai package +- New configuration options are available in the `config.py` file for external content handling +- The `WebScrapingStrategy` class has been updated to accommodate new external content exclusion options + +## [v0.3.71] - 2024-10-19 + +### Added +- New chunking strategies: + - `OverlappingWindowChunking`: Allows for overlapping chunks of text, useful for maintaining context between chunks. + - Enhanced `SlidingWindowChunking`: Improved to handle edge cases and last chunks more effectively. + +### Changed +- Updated `CHUNK_TOKEN_THRESHOLD` in config to 2048 tokens (2^11) for better compatibility with most LLM models. +- Improved `AsyncPlaywrightCrawlerStrategy.close()` method to use a shorter sleep time (0.5 seconds instead of 500), significantly reducing wait time when closing the crawler. +- Enhanced flexibility in `CosineStrategy`: + - Now uses a more generic `load_HF_embedding_model` function, allowing for easier swapping of embedding models. +- Updated `JsonCssExtractionStrategy` and `JsonXPathExtractionStrategy` for better JSON-based extraction. + +### Fixed +- Addressed potential issues with the sliding window chunking strategy to ensure all text is properly chunked. + +### Developer Notes +- Added more comprehensive docstrings to chunking strategies for better code documentation. +- Removed hardcoded device setting in `CosineStrategy`, now using the automatically detected device. +- Added a new example in `quickstart_async.py` for generating a knowledge graph from crawled content. + +These updates aim to provide more flexibility in text processing, improve performance, and enhance the overall capabilities of the crawl4ai library. The new chunking strategies, in particular, offer more options for handling large texts in various scenarios. + +## [v0.3.71] - 2024-10-18 + +### Changes +1. **Version Update**: + - Updated version number from 0.3.7 to 0.3.71. + +2. **Crawler Enhancements**: + - Added `sleep_on_close` option to AsyncPlaywrightCrawlerStrategy for delayed browser closure. + - Improved context creation with additional options: + - Enabled `accept_downloads` and `java_script_enabled`. + - Added a cookie to enable cookies by default. + +3. **Error Handling Improvements**: + - Enhanced error messages in AsyncWebCrawler's `arun` method. + - Updated error reporting format for better visibility and consistency. + +4. **Performance Optimization**: + - Commented out automatic page and context closure in `crawl` method to potentially improve performance in certain scenarios. + +### Documentation +- Updated quickstart notebook: + - Changed installation command to use the released package instead of GitHub repository. + - Updated kernel display name. + +### Developer Notes +- Minor code refactoring and cleanup. + +## [v0.3.7] - 2024-10-17 + +### New Features +1. **Enhanced Browser Stealth**: + - Implemented `playwright_stealth` for improved bot detection avoidance. + - Added `StealthConfig` for fine-tuned control over stealth parameters. + +2. **User Simulation**: + - New `simulate_user` option to mimic human-like interactions (mouse movements, clicks, keyboard presses). + +3. **Navigator Override**: + - Added `override_navigator` option to modify navigator properties, further improving bot detection evasion. + +4. **Improved iframe Handling**: + - New `process_iframes` parameter to extract and integrate iframe content into the main page. + +5. **Flexible Browser Selection**: + - Support for choosing between Chromium, Firefox, and WebKit browsers. + +6. **Include Links in Markdown**: + - Added support for including links in Markdown content, by definin g a new flag `include_links_on_markdown` in `crawl` method. + +### Improvements +1. **Better Error Handling**: + - Enhanced error reporting in WebScrapingStrategy with detailed error messages and suggestions. + - Added console message and error logging for better debugging. + +2. **Image Processing Enhancements**: + - Improved image dimension updating and filtering logic. + +3. **Crawling Flexibility**: + - Added support for custom viewport sizes. + - Implemented delayed content retrieval with `delay_before_return_html` parameter. + +4. **Performance Optimization**: + - Adjusted default semaphore count for parallel crawling. + +### Bug Fixes +- Fixed an issue where the HTML content could be empty after processing. + +### Examples +- Added new example `crawl_with_user_simulation()` demonstrating the use of user simulation and navigator override features. + +### Developer Notes +- Refactored code for better maintainability and readability. +- Updated browser launch arguments for improved compatibility and performance. + +## [v0.3.6] - 2024-10-12 + +### 1. Improved Crawling Control +- **New Hook**: Added `before_retrieve_html` hook in `AsyncPlaywrightCrawlerStrategy`. +- **Delayed HTML Retrieval**: Introduced `delay_before_return_html` parameter to allow waiting before retrieving HTML content. + - Useful for pages with delayed content loading. +- **Flexible Timeout**: `smart_wait` function now uses `page_timeout` (default 60 seconds) instead of a fixed 30-second timeout. + - Provides better handling for slow-loading pages. +- **How to use**: Set `page_timeout=your_desired_timeout` (in milliseconds) when calling `crawler.arun()`. + +### 2. Browser Type Selection +- Added support for different browser types (Chromium, Firefox, WebKit). +- Users can now specify the browser type when initializing AsyncWebCrawler. +- **How to use**: Set `browser_type="firefox"` or `browser_type="webkit"` when initializing AsyncWebCrawler. + +### 3. Screenshot Capture +- Added ability to capture screenshots during crawling. +- Useful for debugging and content verification. +- **How to use**: Set `screenshot=True` when calling `crawler.arun()`. + +### 4. Enhanced LLM Extraction Strategy +- Added support for multiple LLM providers (OpenAI, Hugging Face, Ollama). +- **Custom Arguments**: Added support for passing extra arguments to LLM providers via `extra_args` parameter. +- **Custom Headers**: Users can now pass custom headers to the extraction strategy. +- **How to use**: Specify the desired provider and custom arguments when using `LLMExtractionStrategy`. + +### 5. iframe Content Extraction +- New feature to process and extract content from iframes. +- **How to use**: Set `process_iframes=True` in the crawl method. + +### 6. Delayed Content Retrieval +- Introduced `get_delayed_content` method in `AsyncCrawlResponse`. +- Allows retrieval of content after a specified delay, useful for dynamically loaded content. +- **How to use**: Access `result.get_delayed_content(delay_in_seconds)` after crawling. + +### Improvements and Optimizations + +#### 1. AsyncWebCrawler Enhancements +- **Flexible Initialization**: Now accepts arbitrary keyword arguments, passed directly to the crawler strategy. +- Allows for more customized setups. + +#### 2. Image Processing Optimization +- Enhanced image handling in WebScrapingStrategy. +- Added filtering for small, invisible, or irrelevant images. +- Improved image scoring system for better content relevance. +- Implemented JavaScript-based image dimension updating for more accurate representation. + +#### 3. Database Schema Auto-updates +- Automatic database schema updates ensure compatibility with the latest version. + +#### 4. Enhanced Error Handling and Logging +- Improved error messages and logging for easier debugging. + +#### 5. Content Extraction Refinements +- Refined HTML sanitization process. +- Improved handling of base64 encoded images. +- Enhanced Markdown conversion process. +- Optimized content extraction algorithms. + +#### 6. Utility Function Enhancements +- `perform_completion_with_backoff` function now supports additional arguments for more customized API calls to LLM providers. + +### Bug Fixes +- Fixed an issue where image tags were being prematurely removed during content extraction. + +### Examples and Documentation +- Updated `quickstart_async.py` with examples of: + - Using custom headers in LLM extraction. + - Different LLM provider usage (OpenAI, Hugging Face, Ollama). + - Custom browser type usage. + +### Developer Notes +- Refactored code for better maintainability, flexibility, and performance. +- Enhanced type hinting throughout the codebase for improved development experience. +- Expanded error handling for more robust operation. + +These updates significantly enhance the flexibility, accuracy, and robustness of crawl4ai, providing users with more control and options for their web crawling and content extraction tasks. + +## [v0.3.5] - 2024-09-02 + +Enhance AsyncWebCrawler with smart waiting and screenshot capabilities + +- Implement smart_wait function in AsyncPlaywrightCrawlerStrategy +- Add screenshot support to AsyncCrawlResponse and AsyncWebCrawler +- Improve error handling and timeout management in crawling process +- Fix typo in CrawlResult model (responser_headers -> response_headers) + +## [v0.2.77] - 2024-08-04 + +Significant improvements in text processing and performance: + +- 🚀 **Dependency reduction**: Removed dependency on spaCy model for text chunk labeling in cosine extraction strategy. +- 🤖 **Transformer upgrade**: Implemented text sequence classification using a transformer model for labeling text chunks. +- ⚡ **Performance enhancement**: Improved model loading speed due to removal of spaCy dependency. +- 🔧 **Future-proofing**: Laid groundwork for potential complete removal of spaCy dependency in future versions. + +These changes address issue #68 and provide a foundation for faster, more efficient text processing in Crawl4AI. + +## [v0.2.76] - 2024-08-02 + +Major improvements in functionality, performance, and cross-platform compatibility! 🚀 + +- 🐳 **Docker enhancements**: Significantly improved Dockerfile for easy installation on Linux, Mac, and Windows. +- 🌐 **Official Docker Hub image**: Launched our first official image on Docker Hub for streamlined deployment. +- 🔧 **Selenium upgrade**: Removed dependency on ChromeDriver, now using Selenium's built-in capabilities for better compatibility. +- 🖼️ **Image description**: Implemented ability to generate textual descriptions for extracted images from web pages. +- ⚡ **Performance boost**: Various improvements to enhance overall speed and performance. + +A big shoutout to our amazing community contributors: +- [@aravindkarnam](https://github.com/aravindkarnam) for developing the textual description extraction feature. +- [@FractalMind](https://github.com/FractalMind) for creating the first official Docker Hub image and fixing Dockerfile errors. +- [@ketonkss4](https://github.com/ketonkss4) for identifying Selenium's new capabilities, helping us reduce dependencies. + +Your contributions are driving Crawl4AI forward! 🙌 + +## [v0.2.75] - 2024-07-19 + +Minor improvements for a more maintainable codebase: + +- 🔄 Fixed typos in `chunking_strategy.py` and `crawler_strategy.py` to improve code readability +- 🔄 Removed `.test_pads/` directory from `.gitignore` to keep our repository clean and organized + +These changes may seem small, but they contribute to a more stable and sustainable codebase. By fixing typos and updating our `.gitignore` settings, we're ensuring that our code is easier to maintain and scale in the long run. + +## [v0.2.74] - 2024-07-08 +A slew of exciting updates to improve the crawler's stability and robustness! 🎉 + +- 💻 **UTF encoding fix**: Resolved the Windows \"charmap\" error by adding UTF encoding. +- 🛡️ **Error handling**: Implemented MaxRetryError exception handling in LocalSeleniumCrawlerStrategy. +- 🧹 **Input sanitization**: Improved input sanitization and handled encoding issues in LLMExtractionStrategy. +- 🚮 **Database cleanup**: Removed existing database file and initialized a new one. + + +## [v0.2.73] - 2024-07-03 + +💡 In this release, we've bumped the version to v0.2.73 and refreshed our documentation to ensure you have the best experience with our project. + +* Supporting website need "with-head" mode to crawl the website with head. +* Fixing the installation issues for setup.py and dockerfile. +* Resolve multiple issues. + +## [v0.2.72] - 2024-06-30 + +This release brings exciting updates and improvements to our project! 🎉 + +* 📚 **Documentation Updates**: Our documentation has been revamped to reflect the latest changes and additions. +* 🚀 **New Modes in setup.py**: We've added support for three new modes in setup.py: default, torch, and transformers. This enhances the project's flexibility and usability. +* 🐳 **Docker File Updates**: The Docker file has been updated to ensure seamless compatibility with the new modes and improvements. +* 🕷️ **Temporary Solution for Headless Crawling**: We've implemented a temporary solution to overcome issues with crawling websites in headless mode. + +These changes aim to improve the overall user experience, provide more flexibility, and enhance the project's performance. We're thrilled to share these updates with you and look forward to continuing to evolve and improve our project! + +## [0.2.71] - 2024-06-26 + +**Improved Error Handling and Performance** 🚧 + +* 🚫 Refactored `crawler_strategy.py` to handle exceptions and provide better error messages, making it more robust and reliable. +* 💻 Optimized the `get_content_of_website_optimized` function in `utils.py` for improved performance, reducing potential bottlenecks. +* 💻 Updated `utils.py` with the latest changes, ensuring consistency and accuracy. +* 🚫 Migrated to `ChromeDriverManager` to resolve Chrome driver download issues, providing a smoother user experience. + +These changes focus on refining the existing codebase, resulting in a more stable, efficient, and user-friendly experience. With these improvements, you can expect fewer errors and better performance in the crawler strategy and utility functions. + +## [0.2.71] - 2024-06-25 +### Fixed +- Speed up twice the extraction function. + + +## [0.2.6] - 2024-06-22 +### Fixed +- Fix issue #19: Update Dockerfile to ensure compatibility across multiple platforms. + +## [0.2.5] - 2024-06-18 +### Added +- Added five important hooks to the crawler: + - on_driver_created: Called when the driver is ready for initializations. + - before_get_url: Called right before Selenium fetches the URL. + - after_get_url: Called after Selenium fetches the URL. + - before_return_html: Called when the data is parsed and ready. + - on_user_agent_updated: Called when the user changes the user_agent, causing the driver to reinitialize. +- Added an example in `quickstart.py` in the example folder under the docs. +- Enhancement issue #24: Replaced inline HTML tags (e.g., DEL, INS, SUB, ABBR) with textual format for better context handling in LLM. +- Maintaining the semantic context of inline tags (e.g., abbreviation, DEL, INS) for improved LLM-friendliness. +- Updated Dockerfile to ensure compatibility across multiple platforms (Hopefully!). + +## [v0.2.4] - 2024-06-17 +### Fixed +- Fix issue #22: Use MD5 hash for caching HTML files to handle long URLs diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..31dad7b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,131 @@ +# Crawl4AI Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +unclecode@crawl4ai.com. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..38d22c2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,102 @@ +# Contributing to Crawl4AI + +Welcome to the Crawl4AI project! As an open-source library for web crawling and AI integration, we value contributions from the community. This guide explains our branching strategy, how to contribute effectively, and the overall release process. Our goal is to maintain a stable, collaborative environment where bug fixes, features, and improvements can be integrated smoothly while allowing for experimental development. + +We follow a GitFlow-inspired workflow to ensure predictability and quality. Releases occur approximately every two weeks, with a focus on semantic versioning, comprehensive documentation, and user-friendly updates. + +## Core Branches + +- **main**: The stable branch containing production-ready code. It's always identical to the latest released version and is tagged for releases. Do not submit PRs directly here. +- **develop**: The primary integration branch for ongoing development. This is where all contributions (bug fixes, minor features, documentation updates) are merged. Submit your pull requests targeting this branch. +- **next**: Reserved for the lead maintainer (Unclecode) to experiment with major features, refactors, or cutting-edge changes. These are merged into `develop` when ready. +- **release/vX.Y.Z**: Temporary branches created from `develop` for final release preparations (e.g., version bumps, demos, release notes). These are short-lived and deleted after the release. + +## Contributor Workflow + +We encourage contributions of all kinds: bug fixes, new features, documentation improvements, tests, or even Docker enhancements. Follow these steps to contribute: + +1. **Fork the Repository**: Create your own fork on GitHub. +2. **Create a Branch**: Base your work on the `develop` branch. + + ``` + git checkout develop + git checkout -b feature/your-feature-name # Or bugfix/your-bugfix-name + + ``` + +3. **Make Changes**: + - Implement your feature or fix. + - If updating documentation (e.g., README.md, mkdocs.yml, or docs/blog/), ensure version references are consistent (e.g., update site_name in mkdocs.yml to reflect the upcoming version if relevant). + - For Docker-related changes (e.g., Dockerfile, docker-compose.yml, or docs/md_v2/core/docker-deployment.md), test locally and include build instructions in your PR description. + - Add tests if applicable (run `pytest` to verify). + - Follow code style guidelines (use black for formatting). +4. **Commit and Push**: + - Use descriptive commit messages (e.g., "Fix: Resolve issue with async crawling"). + - Push to your fork: `git push origin feature/your-feature-name`. +5. **Submit a Pull Request**: + - Target the `develop` branch. + - Provide a clear description: What does it do? Link to any issues. Include screenshots or code examples if helpful. + - If your change affects documentation or Docker, mention how it aligns with the version (e.g., "Updates Docker docs for v0.7.0 compatibility"). + - We'll review and merge approved PRs into `develop`. +6. **Discuss Large Changes**: For major features or experimental ideas, open an issue first to align with the project's direction. + +If your PR involves breaking changes, include a migration guide in the description. + +## Lead Maintainer's Workflow (For Reference) + +- The lead maintainer (Unclecode) uses the `next` branch for isolated experimental work. +- Features from `next` are periodically merged into `develop` (via rebase and merge) to keep everything in sync. +- This isolation ensures your contributions aren't disrupted by ongoing major changes. + +## Release Process (High-Level Overview) + +Releases happen bi-weekly to ship improvements regularly. As a contributor, your merged changes in `develop` will be included in the next release unless specified otherwise. Here's a summary of what happens: + +- **Preparation**: A temporary `release/vX.Y.Z` branch is created from `develop`. Any ready features from `next` are merged here. +- **Final Updates**: + - Version bump in code (e.g., `__version__.py`). + - Creation of a demo script in `examples/` to showcase new features. + - Writing release notes in `docs/blog/` (personal "I" voice from Unclecode, with code examples, impacts, and migration guides if needed). + - Documentation updates: README.md (highlights, version refs), mkdocs.yml (site_name with version), docs/blog/index.md (add new release), and copying notes to `docs/md_v2/blog/releases/`. + - Docker updates: Dockerfile (version arg), docker-compose.yml, deploy/docker/README.md, and docs/md_v2/core/docker-deployment.md. A release candidate image (e.g., `X.Y.Z-r1`) is built and tested. +- **Testing and Merge**: Full tests run; changes committed and merged to `main` with a tag. +- **Publication**: Tagged release on GitHub (with notes), publish to PyPI, and push Docker images (stable and `latest` after testing). +- **Sync**: Back-merge to `develop` and reset `next` for the next cycle. + +Semantic versioning is used: MAJOR for breaking changes, MINOR for features, PATCH for fixes. Pre-releases (e.g., `-rc1`) may be used for testing. + +If your contribution requires Docker testing or affects docs, it may be part of this step—feel free to suggest updates in your PR. + +## Benefits of This Approach + +- **Stability**: `main` is always reliable for users. +- **Collaboration**: Fixed PR target (`develop`) makes contributing straightforward. +- **Isolation**: Experimental work in `next` doesn't block team progress. +- **User-Focused**: Releases include demos, detailed notes, and updated docs/Docker for easy adoption. +- **Predictability**: Bi-weekly cadence keeps the project active. + +## Checklist for Contributors + +Before submitting a PR: + +- [ ] Based on and targeting `develop`. +- [ ] Tests pass (`pytest`). +- [ ] Docs updated if needed (e.g., version refs in mkdocs.yml, Docker files). +- [ ] No breaking changes without a migration guide. +- [ ] Descriptive title and description. + +## Common Issues + +- **Merge Conflicts**: Rebase your branch on latest `develop` before PR. +- **Docker Builds**: Test multi-arch (amd64/arm64) locally if changing Dockerfile. +- **Version Consistency**: Ensure any version mentions match semantic rules. + +## Communication + +- Open issues for discussions or bugs. +- Join our Discord (link in README) for real-time help. +- After releases, announcements go to GitHub, Discord, and social media. + +Thanks for contributing to Crawl4AI — we appreciate your help in making it better! + +*Last Updated: Feb 3, 2026* diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md new file mode 100644 index 0000000..7dae326 --- /dev/null +++ b/CONTRIBUTORS.md @@ -0,0 +1,93 @@ +# Contributors to Crawl4AI + +We would like to thank the following people for their contributions to Crawl4AI: + +## Core Team + +- [Unclecode](https://github.com/unclecode) - Project Creator and Main Developer +- [Nasrin](https://github.com/ntohidi) - Project Manager and Developer +- [Aravind Karnam](https://github.com/aravindkarnam) - Head of Community and Product + +## Community Contributors + +- [aadityakanjolia4](https://github.com/aadityakanjolia4) - Fix for `CustomHTML2Text` is not defined. +- [FractalMind](https://github.com/FractalMind) - Created the first official Docker Hub image and fixed Dockerfile errors +- [ketonkss4](https://github.com/ketonkss4) - Identified Selenium's new capabilities, helping reduce dependencies +- [jonymusky](https://github.com/jonymusky) - Javascript execution documentation, and wait_for +- [datehoer](https://github.com/datehoer) - Add browser prxy support + +## Pull Requests + +- [dvschuyl](https://github.com/dvschuyl) - AsyncPlaywrightCrawlerStrategy page-evaluate context destroyed by navigation [#304](https://github.com/unclecode/crawl4ai/pull/304) +- [nelzomal](https://github.com/nelzomal) - Enhance development installation instructions [#286](https://github.com/unclecode/crawl4ai/pull/286) +- [HamzaFarhan](https://github.com/HamzaFarhan) - Handled the cases where markdown_with_citations, references_markdown, and filtered_html might not be defined [#293](https://github.com/unclecode/crawl4ai/pull/293) +- [NanmiCoder](https://github.com/NanmiCoder) - fix: crawler strategy exception handling and fixes [#271](https://github.com/unclecode/crawl4ai/pull/271) +- [paulokuong](https://github.com/paulokuong) - fix: RAWL4_AI_BASE_DIRECTORY should be Path object instead of string [#298](https://github.com/unclecode/crawl4ai/pull/298) +- [TheRedRad](https://github.com/theredrad) - feat: add force viewport screenshot option [#1694](https://github.com/unclecode/crawl4ai/pull/1694) +- [ChiragBellara](https://github.com/ChiragBellara) - fix: avoid Common Crawl calls for sitemap-only URL seeding [#1746](https://github.com/unclecode/crawl4ai/pull/1746) +- [YuriNachos](https://github.com/YuriNachos) - fix: replace tf-playwright-stealth with playwright-stealth [#1714](https://github.com/unclecode/crawl4ai/pull/1714), fix: respect `` tag for relative link resolution [#1721](https://github.com/unclecode/crawl4ai/pull/1721), fix: include GoogleSearchCrawler script.js in package [#1719](https://github.com/unclecode/crawl4ai/pull/1719), fix: allow local embeddings by removing OpenAI fallback [#1717](https://github.com/unclecode/crawl4ai/pull/1717), docs: add missing CacheMode import [#1715](https://github.com/unclecode/crawl4ai/pull/1715), docs: fix return types to RunManyReturn [#1716](https://github.com/unclecode/crawl4ai/pull/1716) +- [christian-oudard](https://github.com/christian-oudard) - fix: deep-crawl CLI outputting only the first page [#1667](https://github.com/unclecode/crawl4ai/pull/1667) +- [vladmandic](https://github.com/vladmandic) - fix: VersionManager ignoring CRAWL4_AI_BASE_DIRECTORY env var [#1296](https://github.com/unclecode/crawl4ai/pull/1296) +- [nnxiong](https://github.com/nnxiong) - fix: script tag removal losing adjacent text in cleaned_html [#1364](https://github.com/unclecode/crawl4ai/pull/1364) +- [RoyLeviLangware](https://github.com/RoyLeviLangware) - fix: bs4 deprecation warning (text -> string) [#1077](https://github.com/unclecode/crawl4ai/pull/1077) +- [garyluky](https://github.com/garyluky) - fix: proxy auth ERR_INVALID_AUTH_CREDENTIALS [#1281](https://github.com/unclecode/crawl4ai/pull/1281) +- [Martichou](https://github.com/Martichou) - investigation: browser context memory leak under continuous load [#1640](https://github.com/unclecode/crawl4ai/pull/1640), [#943](https://github.com/unclecode/crawl4ai/issues/943) +- [danyQe](https://github.com/danyQe) - identified: temperature typo in async_configs.py [#973](https://github.com/unclecode/crawl4ai/pull/973) +- [saipavanmeruga7797](https://github.com/saipavanmeruga7797) - identified: local HTML file crawling bug with capture_console_messages [#1073](https://github.com/unclecode/crawl4ai/pull/1073) +- [stevenaldinger](https://github.com/stevenaldinger) - identified: duplicate PROMPT_EXTRACT_BLOCKS dead code in prompts.py [#931](https://github.com/unclecode/crawl4ai/pull/931) +- [chrizzly2309](https://github.com/chrizzly2309) - identified: JWT auth bypass when no credentials provided [#1133](https://github.com/unclecode/crawl4ai/pull/1133) +- [complete-dope](https://github.com/complete-dope) - identified: console logging error attribute issue [#729](https://github.com/unclecode/crawl4ai/pull/729) +- [TristanDonze](https://github.com/TristanDonze) - feat: add configurable device_scale_factor for screenshot quality [#1463](https://github.com/unclecode/crawl4ai/pull/1463) +- [charlaie](https://github.com/charlaie) - feat: add redirected_status_code to CrawlResult [#1435](https://github.com/unclecode/crawl4ai/pull/1435) +- [mzyfree](https://github.com/mzyfree) - investigation: Docker concurrency performance and pool resource management [#1689](https://github.com/unclecode/crawl4ai/pull/1689) +- [nightcityblade](https://github.com/nightcityblade) - fix: prevent AdaptiveCrawler from crawling external domains [#1805](https://github.com/unclecode/crawl4ai/pull/1805), fix: stabilize best-first batch ordering [#2003](https://github.com/unclecode/crawl4ai/pull/2003), fix: close browser contexts from snapshot [#2004](https://github.com/unclecode/crawl4ai/pull/2004), fix: make read-only tmpfs writable in Docker [#2028](https://github.com/unclecode/crawl4ai/pull/2028), fix: cap FastAPI below 0.137 [#2034](https://github.com/unclecode/crawl4ai/pull/2034) +- [Otman404](https://github.com/Otman404) - fix: return in finally block silently suppressing exceptions in dispatcher [#1763](https://github.com/unclecode/crawl4ai/pull/1763) +- [SohamKukreti](https://github.com/SohamKukreti) - fix: from_serializable_dict ignoring plain data dicts with "type" key [#1803](https://github.com/unclecode/crawl4ai/pull/1803), fix: deep-crawl streaming mirrors Python library behavior [#1798](https://github.com/unclecode/crawl4ai/pull/1798) +- [Br1an67](https://github.com/Br1an67) - fix: handle nested brackets and parentheses in LINK_PATTERN regex [#1790](https://github.com/unclecode/crawl4ai/pull/1790), identified: strip markdown fences in LLM JSON responses [#1787](https://github.com/unclecode/crawl4ai/pull/1787), fix: preserve class/id in cleaned_html [#1782](https://github.com/unclecode/crawl4ai/pull/1782), fix: guard against None LLM content [#1788](https://github.com/unclecode/crawl4ai/pull/1788), fix: strip port from domain in is_external_url [#1783](https://github.com/unclecode/crawl4ai/pull/1783), fix: UTF-8 encoding for CLI output [#1789](https://github.com/unclecode/crawl4ai/pull/1789), fix: configurable link_preview_timeout [#1793](https://github.com/unclecode/crawl4ai/pull/1793), fix: wait_for_images on screenshot endpoint [#1792](https://github.com/unclecode/crawl4ai/pull/1792), fix: cross-platform terminal input in CrawlerMonitor [#1794](https://github.com/unclecode/crawl4ai/pull/1794), fix: UnicodeEncodeError in URL seeder [#1784](https://github.com/unclecode/crawl4ai/pull/1784), fix: wire mean_delay/max_range into dispatcher [#1786](https://github.com/unclecode/crawl4ai/pull/1786), fix: DOMParser in process_iframes [#1796](https://github.com/unclecode/crawl4ai/pull/1796), fix: require api_token for /token endpoint [#1795](https://github.com/unclecode/crawl4ai/pull/1795) +- [nightcityblade](https://github.com/nightcityblade) - feat: add score_threshold to BestFirstCrawlingStrategy [#1804](https://github.com/unclecode/crawl4ai/pull/1804) +- [phamngocquy](https://github.com/phamngocquy) - identified: raw HTML URL token leak [#1179](https://github.com/unclecode/crawl4ai/pull/1179) +- [AkosLukacs](https://github.com/AkosLukacs) - docs: fix docstring param name crawler_config -> config [#1494](https://github.com/unclecode/crawl4ai/pull/1494) +- [dominicx](https://github.com/dominicx) - docs: fix css_selector type from list to string [#1308](https://github.com/unclecode/crawl4ai/pull/1308) +- [hoi](https://github.com/hoi) - fix: add TTL expiry for Redis task data [#1730](https://github.com/unclecode/crawl4ai/pull/1730) +- [maksimzayats](https://github.com/maksimzayats) - docs: modernize deprecated API usage across 25 files [#1770](https://github.com/unclecode/crawl4ai/pull/1770) +- [jtanningbed](https://github.com/jtanningbed) - fix: add newline before opening code fence in html2text [#462](https://github.com/unclecode/crawl4ai/pull/462) +- [Ahmed-Tawfik94](https://github.com/Ahmed-Tawfik94) - identified: redirect target verification in URL seeder [#1622](https://github.com/unclecode/crawl4ai/pull/1622) +- [hafezparast](https://github.com/hafezparast) - identified: PDFContentScrapingStrategy deserialization fix [#1815](https://github.com/unclecode/crawl4ai/issues/1815); fix: screenshot distortion, deep crawl timeout/arun_many, CLI encoding [#1829](https://github.com/unclecode/crawl4ai/pull/1829), fix: convert page_timeout from ms to seconds for aiohttp ClientTimeout [#1895](https://github.com/unclecode/crawl4ai/pull/1895), feat: add preserve_classes/preserve_tags to PruningContentFilter [#1904](https://github.com/unclecode/crawl4ai/pull/1904) +- [pgoslatara](https://github.com/pgoslatara) - chore: update GitHub Actions to latest versions [#1734](https://github.com/unclecode/crawl4ai/pull/1734) +- [130347665](https://github.com/130347665) - feat: type-list pipeline in JsonCssExtractionStrategy [#1290](https://github.com/unclecode/crawl4ai/pull/1290) +- [microHoffman](https://github.com/microHoffman) - feat: add --json-ensure-ascii CLI flag for Unicode handling [#1668](https://github.com/unclecode/crawl4ai/pull/1668) +- [fstark96](https://github.com/fstark96) - fix: skip channel='chromium' on Windows to prevent Chrome exit code 0 [#2051](https://github.com/unclecode/crawl4ai/pull/2051) +- [TobiasWallura-xitaso](https://github.com/TobiasWallura-xitaso) - fix: use writable directories for supervisord pid and redis data in Docker [#2047](https://github.com/unclecode/crawl4ai/pull/2047) +- [harshmathurx](https://github.com/harshmathurx) - fix: authenticate rate limit Redis storage in Docker [#2043](https://github.com/unclecode/crawl4ai/pull/2043) +- [RajanChavada](https://github.com/RajanChavada) - chore: remove dead normalize_url definitions and accidental adaptive_crawler copy [#2042](https://github.com/unclecode/crawl4ai/pull/2042) +- [bibi-creator](https://github.com/bibi-creator) - docs: split sponsors into Strategic Partners and Enterprise Sponsors [#2056](https://github.com/unclecode/crawl4ai/pull/2056), fix: update sponsor logos [#2052](https://github.com/unclecode/crawl4ai/pull/2052) + +#### Feb-Alpha-1 +- [sufianuddin](https://github.com/sufianuddin) - fix: [Documentation for JsonCssExtractionStrategy](https://github.com/unclecode/crawl4ai/issues/651) +- [tautikAg](https://github.com/tautikAg) - fix: [Markdown output has incorect spacing](https://github.com/unclecode/crawl4ai/issues/599) +- [cardit1](https://github.com/cardit1) - fix: ['AsyncPlaywrightCrawlerStrategy' object has no attribute 'downloads_path'](https://github.com/unclecode/crawl4ai/issues/585) +- [dmurat](https://github.com/dmurat) - fix: [ Incorrect rendering of inline code inside of links ](https://github.com/unclecode/crawl4ai/issues/583) +- [Sparshsing](https://github.com/Sparshsing) - fix: [Relative Urls in the webpage not extracted properly ](https://github.com/unclecode/crawl4ai/issues/570) + + + +## Other Contributors + +- [Gokhan](https://github.com/gkhngyk) +- [Shiv Kumar](https://github.com/shivkumar0757) +- [QIN2DIM](https://github.com/QIN2DIM) + +#### Typo fixes +- [ssoydan](https://github.com/ssoydan) +- [Darshan](https://github.com/Darshan2104) +- [tuhinmallick](https://github.com/tuhinmallick) + +## Acknowledgements + +We also want to thank all the users who have reported bugs, suggested features, or helped in any other way to make Crawl4AI better. + +--- + +If you've contributed to Crawl4AI and your name isn't on this list, please [open a pull request](https://github.com/unclecode/crawl4ai/pulls) with your name, link, and contribution, and we'll review it promptly. + +Thank you all for your contributions! \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..51bc171 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,229 @@ +FROM python:3.12-slim-bookworm AS build + +# C4ai version +ARG C4AI_VER=0.9.1 +ENV C4AI_VERSION=$C4AI_VER +LABEL c4ai.version=$C4AI_VER + +# Set build arguments +ARG APP_HOME=/app +ARG GITHUB_REPO=https://github.com/unclecode/crawl4ai.git +ARG GITHUB_BRANCH=main +ARG USE_LOCAL=true + +ENV PYTHONFAULTHANDLER=1 \ + PYTHONHASHSEED=random \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=100 \ + DEBIAN_FRONTEND=noninteractive \ + REDIS_HOST=localhost \ + REDIS_PORT=6379 + +ARG PYTHON_VERSION=3.12 +ARG INSTALL_TYPE=default +ARG ENABLE_GPU=false +ARG TARGETARCH + +# Redis version — pinned to a CVE-patched release by default. +# Override with --build-arg REDIS_VERSION="" for latest, or +# --build-arg REDIS_VERSION="6:7.2.7-1rl1~bookworm1" for a specific version. +ARG REDIS_VERSION="6:7.2.7-1rl1~bookworm1" + +LABEL maintainer="unclecode" +LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper" +LABEL version="1.0" + +# Install curl and gnupg first (needed to add Redis repo) +RUN apt-get update && apt-get install -y --no-install-recommends curl gnupg \ + && rm -rf /var/lib/apt/lists/* + +# Add official Redis repository for security-patched versions +RUN curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb bookworm main" \ + > /etc/apt/sources.list.d/redis.list + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + wget \ + gnupg \ + git \ + cmake \ + pkg-config \ + python3-dev \ + libjpeg-dev \ + redis-tools${REDIS_VERSION:+=$REDIS_VERSION} \ + redis-server${REDIS_VERSION:+=$REDIS_VERSION} \ + supervisor \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libglib2.0-0 \ + libnss3 \ + libnspr4 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcups2 \ + libdrm2 \ + libdbus-1-3 \ + libxcb1 \ + libxkbcommon0 \ + libx11-6 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxrandr2 \ + libgbm1 \ + libpango-1.0-0 \ + libcairo2 \ + libasound2 \ + libatspi2.0-0 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update && apt-get dist-upgrade -y \ + && rm -rf /var/lib/apt/lists/* + +RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \ + apt-get update && apt-get install -y --no-install-recommends \ + nvidia-cuda-toolkit \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ; \ +else \ + echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \ +fi + +RUN if [ "$TARGETARCH" = "arm64" ]; then \ + echo "🦾 Installing ARM-specific optimizations"; \ + apt-get update && apt-get install -y --no-install-recommends \ + libopenblas-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*; \ +elif [ "$TARGETARCH" = "amd64" ]; then \ + echo "🖥️ Installing AMD64-specific optimizations"; \ + apt-get update && apt-get install -y --no-install-recommends \ + libomp-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*; \ +else \ + echo "Skipping platform-specific optimizations (unsupported platform)"; \ +fi + +# Create a non-root user and group +RUN groupadd -r appuser && useradd --no-log-init -r -g appuser appuser + +# Create and set permissions for appuser home directory +RUN mkdir -p /home/appuser && chown -R appuser:appuser /home/appuser + +WORKDIR ${APP_HOME} + +RUN echo '#!/bin/bash\n\ +if [ "$USE_LOCAL" = "true" ]; then\n\ + echo "📦 Installing from local source..."\n\ + pip install --no-cache-dir /tmp/project/\n\ +else\n\ + echo "🌐 Installing from GitHub..."\n\ + for i in {1..3}; do \n\ + git clone --branch ${GITHUB_BRANCH} ${GITHUB_REPO} /tmp/crawl4ai && break || \n\ + { echo "Attempt $i/3 failed! Taking a short break... ☕"; sleep 5; }; \n\ + done\n\ + pip install --no-cache-dir /tmp/crawl4ai\n\ +fi' > /tmp/install.sh && chmod +x /tmp/install.sh + +COPY . /tmp/project/ + +# Copy supervisor config first (might need root later, but okay for now) +COPY deploy/docker/supervisord.conf . + +COPY deploy/docker/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +RUN if [ "$INSTALL_TYPE" = "all" ] ; then \ + pip install --no-cache-dir \ + torch \ + torchvision \ + torchaudio \ + scikit-learn \ + nltk \ + transformers \ + tokenizers && \ + python -m nltk.downloader punkt stopwords ; \ + fi + +RUN if [ "$INSTALL_TYPE" = "all" ] ; then \ + pip install "/tmp/project/[all]" && \ + python -m crawl4ai.model_loader ; \ + elif [ "$INSTALL_TYPE" = "torch" ] ; then \ + pip install "/tmp/project/[torch]" ; \ + elif [ "$INSTALL_TYPE" = "transformer" ] ; then \ + pip install "/tmp/project/[transformer]" && \ + python -m crawl4ai.model_loader ; \ + else \ + pip install "/tmp/project" ; \ + fi + +RUN pip install --no-cache-dir --upgrade pip && \ + /tmp/install.sh && \ + python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \ + python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')" + +RUN crawl4ai-setup + +RUN playwright install --with-deps + +RUN mkdir -p /home/appuser/.cache/ms-playwright \ + && cp -r /root/.cache/ms-playwright/chromium-* /home/appuser/.cache/ms-playwright/ \ + && chown -R appuser:appuser /home/appuser/.cache/ms-playwright + +RUN crawl4ai-doctor + +# Ensure all cache directories belong to appuser +# This fixes permission issues with .cache/url_seeder and other runtime cache dirs +RUN mkdir -p /home/appuser/.cache \ + && chown -R appuser:appuser /home/appuser/.cache + +# Copy application code +COPY deploy/docker/* ${APP_HOME}/ + +# copy the playground + any future static assets +COPY deploy/docker/static ${APP_HOME}/static + +# /app is root-owned and read-only to the runtime user: a write bug can no +# longer plant a persistent self-RCE in the application directory. +RUN chown -R root:root ${APP_HOME} && chmod -R a-w ${APP_HOME} + +# give permissions to redis persistence dirs if used +RUN mkdir -p /var/lib/redis /var/log/redis && chown -R appuser:appuser /var/lib/redis /var/log/redis + +# Sandboxed artifact store (server-owned screenshot/PDF outputs), 0700. +RUN mkdir -p /var/lib/crawl4ai/outputs \ + && chown -R appuser:appuser /var/lib/crawl4ai \ + && chmod 700 /var/lib/crawl4ai/outputs + +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD bash -c '\ + MEM=$(free -m | awk "/^Mem:/{print \$2}"); \ + if [ $MEM -lt 2048 ]; then \ + echo "⚠️ Warning: Less than 2GB RAM available! Your container might need a memory boost! 🚀"; \ + exit 1; \ + fi && \ + redis-cli ping > /dev/null && \ + curl -f http://localhost:11235/health || exit 1' + +# Redis is in-container only (loopback + requirepass); never expose its port. +# (was: EXPOSE 6379) +# Switch to the non-root user before starting the application +USER appuser + +# Set environment variables to ptoduction +ENV PYTHON_ENV=production + +# Start via entrypoint.sh, which resolves the socket-level auth/egress posture +# (loopback unless a credential is present) and the redis password, then execs +# supervisord. +CMD ["bash", "entrypoint.sh"] \ No newline at end of file diff --git a/JOURNAL.md b/JOURNAL.md new file mode 100644 index 0000000..c2d21e3 --- /dev/null +++ b/JOURNAL.md @@ -0,0 +1,339 @@ +# Development Journal + +This journal tracks significant feature additions, bug fixes, and architectural decisions in the crawl4ai project. It serves as both documentation and a historical record of the project's evolution. + +## [2025-04-17] Added Content Source Selection for Markdown Generation + +**Feature:** Configurable content source for markdown generation + +**Changes Made:** +1. Added `content_source: str = "cleaned_html"` parameter to `MarkdownGenerationStrategy` class +2. Updated `DefaultMarkdownGenerator` to accept and pass the content source parameter +3. Renamed the `cleaned_html` parameter to `input_html` in the `generate_markdown` method +4. Modified `AsyncWebCrawler.aprocess_html` to select the appropriate HTML source based on the generator's config +5. Added `preprocess_html_for_schema` import in `async_webcrawler.py` + +**Implementation Details:** +- Added a new `content_source` parameter to specify which HTML input to use for markdown generation +- Options include: "cleaned_html" (default), "raw_html", and "fit_html" +- Used a dictionary dispatch pattern in `aprocess_html` to select the appropriate HTML source +- Added proper error handling with fallback to cleaned_html if content source selection fails +- Ensured backward compatibility by defaulting to "cleaned_html" option + +**Files Modified:** +- `crawl4ai/markdown_generation_strategy.py`: Added content_source parameter and updated the method signature +- `crawl4ai/async_webcrawler.py`: Added HTML source selection logic and updated imports + +**Examples:** +- Created `docs/examples/content_source_example.py` demonstrating how to use the new parameter + +**Challenges:** +- Maintaining backward compatibility while reorganizing the parameter flow +- Ensuring proper error handling for all content source options +- Making the change with minimal code modifications + +**Why This Feature:** +The content source selection feature allows users to choose which HTML content to use as input for markdown generation: +1. "cleaned_html" - Uses the post-processed HTML after scraping strategy (original behavior) +2. "raw_html" - Uses the original raw HTML directly from the web page +3. "fit_html" - Uses the preprocessed HTML optimized for schema extraction + +This feature provides greater flexibility in how users generate markdown, enabling them to: +- Capture more detailed content from the original HTML when needed +- Use schema-optimized HTML when working with structured data +- Choose the approach that best suits their specific use case +## [2025-04-17] Implemented High Volume Stress Testing Solution for SDK + +**Feature:** Comprehensive stress testing framework using `arun_many` and the dispatcher system to evaluate performance, concurrency handling, and identify potential issues under high-volume crawling scenarios. + +**Changes Made:** +1. Created a dedicated stress testing framework in the `benchmarking/` (or similar) directory. +2. Implemented local test site generation (`SiteGenerator`) with configurable heavy HTML pages. +3. Added basic memory usage tracking (`SimpleMemoryTracker`) using platform-specific commands (avoiding `psutil` dependency for this specific test). +4. Utilized `CrawlerMonitor` from `crawl4ai` for rich terminal UI and real-time monitoring of test progress and dispatcher activity. +5. Implemented detailed result summary saving (JSON) and memory sample logging (CSV). +6. Developed `run_benchmark.py` to orchestrate tests with predefined configurations. +7. Created `run_all.sh` as a simple wrapper for `run_benchmark.py`. + +**Implementation Details:** +- Generates a local test site with configurable pages containing heavy text and image content. +- Uses Python's built-in `http.server` for local serving, minimizing network variance. +- Leverages `crawl4ai`'s `arun_many` method for processing URLs. +- Utilizes `MemoryAdaptiveDispatcher` to manage concurrency via the `max_sessions` parameter (note: memory adaptation features require `psutil`, not used by `SimpleMemoryTracker`). +- Tracks memory usage via `SimpleMemoryTracker`, recording samples throughout test execution to a CSV file. +- Uses `CrawlerMonitor` (which uses the `rich` library) for clear terminal visualization and progress reporting directly from the dispatcher. +- Stores detailed final metrics in a JSON summary file. + +**Files Created/Updated:** +- `stress_test_sdk.py`: Main stress testing implementation using `arun_many`. +- `benchmark_report.py`: (Assumed) Report generator for comparing test results. +- `run_benchmark.py`: Test runner script with predefined configurations. +- `run_all.sh`: Simple bash script wrapper for `run_benchmark.py`. +- `USAGE.md`: Comprehensive documentation on usage and interpretation (updated). + +**Testing Approach:** +- Creates a controlled, reproducible test environment with a local HTTP server. +- Processes URLs using `arun_many`, allowing the dispatcher to manage concurrency up to `max_sessions`. +- Optionally logs per-batch summaries (when not in streaming mode) after processing chunks. +- Supports different test sizes via `run_benchmark.py` configurations. +- Records memory samples via platform commands for basic trend analysis. +- Includes cleanup functionality for the test environment. + +**Challenges:** +- Ensuring proper cleanup of HTTP server processes. +- Getting reliable memory tracking across platforms without adding heavy dependencies (`psutil`) to this specific test script. +- Designing `run_benchmark.py` to correctly pass arguments to `stress_test_sdk.py`. + +**Why This Feature:** +The high volume stress testing solution addresses critical needs for ensuring Crawl4AI's `arun_many` reliability: +1. Provides a reproducible way to evaluate performance under concurrent load. +2. Allows testing the dispatcher's concurrency control (`max_session_permit`) and queue management. +3. Enables performance tuning by observing throughput (`URLs/sec`) under different `max_sessions` settings. +4. Creates a controlled environment for testing `arun_many` behavior. +5. Supports continuous integration by providing deterministic test conditions for `arun_many`. + +**Design Decisions:** +- Chose local site generation for reproducibility and isolation from network issues. +- Utilized the built-in `CrawlerMonitor` for real-time feedback, leveraging its `rich` integration. +- Implemented optional per-batch logging in `stress_test_sdk.py` (when not streaming) to provide chunk-level summaries alongside the continuous monitor. +- Adopted `arun_many` with a `MemoryAdaptiveDispatcher` as the core mechanism for parallel execution, reflecting the intended SDK usage. +- Created `run_benchmark.py` to simplify running standard test configurations. +- Used `SimpleMemoryTracker` to provide basic memory insights without requiring `psutil` for this particular test runner. + +**Future Enhancements to Consider:** +- Create a separate test variant that *does* use `psutil` to specifically stress the memory-adaptive features of the dispatcher. +- Add support for generated JavaScript content. +- Add support for Docker-based testing with explicit memory limits. +- Enhance `benchmark_report.py` to provide more sophisticated analysis of performance and memory trends from the generated JSON/CSV files. + +--- + +## [2025-04-17] Refined Stress Testing System Parameters and Execution + +**Changes Made:** +1. Corrected `run_benchmark.py` and `stress_test_sdk.py` to use `--max-sessions` instead of the incorrect `--workers` parameter, accurately reflecting dispatcher configuration. +2. Updated `run_benchmark.py` argument handling to correctly pass all relevant custom parameters (including `--stream`, `--monitor-mode`, etc.) to `stress_test_sdk.py`. +3. (Assuming changes in `benchmark_report.py`) Applied dark theme to benchmark reports for better readability. +4. (Assuming changes in `benchmark_report.py`) Improved visualization code to eliminate matplotlib warnings. +5. Updated `run_benchmark.py` to provide clickable `file://` links to generated reports in the terminal output. +6. Updated `USAGE.md` with comprehensive parameter descriptions reflecting the final script arguments. +7. Updated `run_all.sh` wrapper to correctly invoke `run_benchmark.py` with flexible arguments. + +**Details of Changes:** + +1. **Parameter Correction (`--max-sessions`)**: + * Identified the fundamental misunderstanding where `--workers` was used incorrectly. + * Refactored `stress_test_sdk.py` to accept `--max-sessions` and configure the `MemoryAdaptiveDispatcher`'s `max_session_permit` accordingly. + * Updated `run_benchmark.py` argument parsing and command construction to use `--max-sessions`. + * Updated `TEST_CONFIGS` in `run_benchmark.py` to use `max_sessions`. + +2. **Argument Handling (`run_benchmark.py`)**: + * Improved logic to collect all command-line arguments provided to `run_benchmark.py`. + * Ensured all relevant arguments (like `--stream`, `--monitor-mode`, `--port`, `--use-rate-limiter`, etc.) are correctly forwarded when calling `stress_test_sdk.py` as a subprocess. + +3. **Dark Theme & Visualization Fixes (Assumed in `benchmark_report.py`)**: + * (Describes changes assumed to be made in the separate reporting script). + +4. **Clickable Links (`run_benchmark.py`)**: + * Added logic to find the latest HTML report and PNG chart in the `benchmark_reports` directory after `benchmark_report.py` runs. + * Used `pathlib` to generate correct `file://` URLs for terminal output. + +5. **Documentation Improvements (`USAGE.md`)**: + * Rewrote sections to explain `arun_many`, dispatchers, and `--max-sessions`. + * Updated parameter tables for all scripts (`stress_test_sdk.py`, `run_benchmark.py`). + * Clarified the difference between batch and streaming modes and their effect on logging. + * Updated examples to use correct arguments. + +**Files Modified:** +- `stress_test_sdk.py`: Changed `--workers` to `--max-sessions`, added new arguments, used `arun_many`. +- `run_benchmark.py`: Changed argument handling, updated configs, calls `stress_test_sdk.py`. +- `run_all.sh`: Updated to call `run_benchmark.py` correctly. +- `USAGE.md`: Updated documentation extensively. +- `benchmark_report.py`: (Assumed modifications for dark theme and viz fixes). + +**Testing:** +- Verified that `--max-sessions` correctly limits concurrency via the `CrawlerMonitor` output. +- Confirmed that custom arguments passed to `run_benchmark.py` are forwarded to `stress_test_sdk.py`. +- Validated clickable links work in supporting terminals. +- Ensured documentation matches the final script parameters and behavior. + +**Why These Changes:** +These refinements correct the fundamental approach of the stress test to align with `crawl4ai`'s actual architecture and intended usage: +1. Ensures the test evaluates the correct components (`arun_many`, `MemoryAdaptiveDispatcher`). +2. Makes test configurations more accurate and flexible. +3. Improves the usability of the testing framework through better argument handling and documentation. + + +**Future Enhancements to Consider:** +- Add support for generated JavaScript content to test JS rendering performance +- Implement more sophisticated memory analysis like generational garbage collection tracking +- Add support for Docker-based testing with memory limits to force OOM conditions +- Create visualization tools for analyzing memory usage patterns across test runs +- Add benchmark comparisons between different crawler versions or configurations + +## [2025-04-17] Fixed Issues in Stress Testing System + +**Changes Made:** +1. Fixed custom parameter handling in run_benchmark.py +2. Applied dark theme to benchmark reports for better readability +3. Improved visualization code to eliminate matplotlib warnings +4. Added clickable links to generated reports in terminal output +5. Enhanced documentation with comprehensive parameter descriptions + +**Details of Changes:** + +1. **Custom Parameter Handling Fix** + - Identified bug where custom URL count was being ignored in run_benchmark.py + - Rewrote argument handling to use a custom args dictionary + - Properly passed parameters to the test_simple_stress.py command + - Added better UI indication of custom parameters in use + +2. **Dark Theme Implementation** + - Added complete dark theme to HTML benchmark reports + - Applied dark styling to all visualization components + - Used Nord-inspired color palette for charts and graphs + - Improved contrast and readability for data visualization + - Updated text colors and backgrounds for better eye comfort + +3. **Matplotlib Warning Fixes** + - Resolved warnings related to improper use of set_xticklabels() + - Implemented correct x-axis positioning for bar charts + - Ensured proper alignment of bar labels and data points + - Updated plotting code to use modern matplotlib practices + +4. **Documentation Improvements** + - Created comprehensive USAGE.md with detailed instructions + - Added parameter documentation for all scripts + - Included examples for all common use cases + - Provided detailed explanations for interpreting results + - Added troubleshooting guide for common issues + +**Files Modified:** +- `tests/memory/run_benchmark.py`: Fixed custom parameter handling +- `tests/memory/benchmark_report.py`: Added dark theme and fixed visualization warnings +- `tests/memory/run_all.sh`: Added clickable links to reports +- `tests/memory/USAGE.md`: Created comprehensive documentation + +**Testing:** +- Verified that custom URL counts are now correctly used +- Confirmed dark theme is properly applied to all report elements +- Checked that matplotlib warnings are no longer appearing +- Validated clickable links to reports work in terminals that support them + +**Why These Changes:** +These improvements address several usability issues with the stress testing system: +1. Better parameter handling ensures test configurations work as expected +2. Dark theme reduces eye strain during extended test review sessions +3. Fixing visualization warnings improves code quality and output clarity +4. Enhanced documentation makes the system more accessible for future use + +**Future Enhancements:** +- Add additional visualization options for different types of analysis +- Implement theme toggle to support both light and dark preferences +- Add export options for embedding reports in other documentation +- Create dedicated CI/CD integration templates for automated testing + +## [2025-04-09] Added MHTML Capture Feature + +**Feature:** MHTML snapshot capture of crawled pages + +**Changes Made:** +1. Added `capture_mhtml: bool = False` parameter to `CrawlerRunConfig` class +2. Added `mhtml: Optional[str] = None` field to `CrawlResult` model +3. Added `mhtml_data: Optional[str] = None` field to `AsyncCrawlResponse` class +4. Implemented `capture_mhtml()` method in `AsyncPlaywrightCrawlerStrategy` class to capture MHTML via CDP +5. Modified the crawler to capture MHTML when enabled and pass it to the result + +**Implementation Details:** +- MHTML capture uses Chrome DevTools Protocol (CDP) via Playwright's CDP session API +- The implementation waits for page to fully load before capturing MHTML content +- Enhanced waiting for JavaScript content with requestAnimationFrame for better JS content capture +- We ensure all browser resources are properly cleaned up after capture + +**Files Modified:** +- `crawl4ai/models.py`: Added the mhtml field to CrawlResult +- `crawl4ai/async_configs.py`: Added capture_mhtml parameter to CrawlerRunConfig +- `crawl4ai/async_crawler_strategy.py`: Implemented MHTML capture logic +- `crawl4ai/async_webcrawler.py`: Added mapping from AsyncCrawlResponse.mhtml_data to CrawlResult.mhtml + +**Testing:** +- Created comprehensive tests in `tests/20241401/test_mhtml.py` covering: + - Capturing MHTML when enabled + - Ensuring mhtml is None when disabled explicitly + - Ensuring mhtml is None by default + - Capturing MHTML on JavaScript-enabled pages + +**Challenges:** +- Had to improve page loading detection to ensure JavaScript content was fully rendered +- Tests needed to be run independently due to Playwright browser instance management +- Modified test expected content to match actual MHTML output + +**Why This Feature:** +The MHTML capture feature allows users to capture complete web pages including all resources (CSS, images, etc.) in a single file. This is valuable for: +1. Offline viewing of captured pages +2. Creating permanent snapshots of web content for archival +3. Ensuring consistent content for later analysis, even if the original site changes + +**Future Enhancements to Consider:** +- Add option to save MHTML to file +- Support for filtering what resources get included in MHTML +- Add support for specifying MHTML capture options + +## [2025-04-10] Added Network Request and Console Message Capturing + +**Feature:** Comprehensive capturing of network requests/responses and browser console messages during crawling + +**Changes Made:** +1. Added `capture_network_requests: bool = False` and `capture_console_messages: bool = False` parameters to `CrawlerRunConfig` class +2. Added `network_requests: Optional[List[Dict[str, Any]]] = None` and `console_messages: Optional[List[Dict[str, Any]]] = None` fields to both `AsyncCrawlResponse` and `CrawlResult` models +3. Implemented event listeners in `AsyncPlaywrightCrawlerStrategy._crawl_web()` to capture browser network events and console messages +4. Added proper event listener cleanup in the finally block to prevent resource leaks +5. Modified the crawler flow to pass captured data from AsyncCrawlResponse to CrawlResult + +**Implementation Details:** +- Network capture uses Playwright event listeners (`request`, `response`, and `requestfailed`) to record all network activity +- Console capture uses Playwright event listeners (`console` and `pageerror`) to record console messages and errors +- Each network event includes metadata like URL, headers, status, and timing information +- Each console message includes type, text content, and source location when available +- All captured events include timestamps for chronological analysis +- Error handling ensures even failed capture attempts won't crash the main crawling process + +**Files Modified:** +- `crawl4ai/models.py`: Added new fields to AsyncCrawlResponse and CrawlResult +- `crawl4ai/async_configs.py`: Added new configuration parameters to CrawlerRunConfig +- `crawl4ai/async_crawler_strategy.py`: Implemented capture logic using event listeners +- `crawl4ai/async_webcrawler.py`: Added data transfer from AsyncCrawlResponse to CrawlResult + +**Documentation:** +- Created detailed documentation in `docs/md_v2/advanced/network-console-capture.md` +- Added feature to site navigation in `mkdocs.yml` +- Updated CrawlResult documentation in `docs/md_v2/api/crawl-result.md` +- Created comprehensive example in `docs/examples/network_console_capture_example.py` + +**Testing:** +- Created `tests/general/test_network_console_capture.py` with tests for: + - Verifying capture is disabled by default + - Testing network request capturing + - Testing console message capturing + - Ensuring both capture types can be enabled simultaneously + - Checking correct content is captured in expected formats + +**Challenges:** +- Initial implementation had synchronous/asynchronous mismatches in event handlers +- Needed to fix type of property access vs. method calls in handlers +- Required careful cleanup of event listeners to prevent memory leaks + +**Why This Feature:** +The network and console capture feature provides deep visibility into web page activity, enabling: +1. Debugging complex web applications by seeing all network requests and errors +2. Security analysis to detect unexpected third-party requests and data flows +3. Performance profiling to identify slow-loading resources +4. API discovery in single-page applications +5. Comprehensive analysis of web application behavior + +**Future Enhancements to Consider:** +- Option to filter captured events by type, domain, or content +- Support for capturing response bodies (with size limits) +- Aggregate statistics calculation for performance metrics +- Integration with visualization tools for network waterfall analysis +- Exporting captures in HAR format for use with external tools \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ade44a7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,69 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and +You must cause any modified files to carry prominent notices stating that You changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +--- +Attribution Requirement + +All distributions, publications, or public uses of this software, or derivative works based on this software, must include the following attribution: + +"This product includes software developed by UncleCode (https://x.com/unclecode) as part of the Crawl4AI project (https://github.com/unclecode/crawl4ai)." + +This attribution must be displayed in a prominent and easily accessible location, such as: + +- For software distributions: In a NOTICE file, README file, or equivalent documentation. +- For publications (research papers, articles, blog posts): In the acknowledgments section or a footnote. +- For websites/web applications: In an "About" or "Credits" section. +- For command-line tools: In the help/usage output. + +This requirement ensures proper credit is given for the use of Crawl4AI and helps promote the project. + +--- \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..73a0e00 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include requirements.txt +recursive-include crawl4ai/js_snippet *.js \ No newline at end of file diff --git a/MISSION.md b/MISSION.md new file mode 100644 index 0000000..726f250 --- /dev/null +++ b/MISSION.md @@ -0,0 +1,46 @@ +# Mission + +![Mission Diagram](./docs/assets/pitch-dark.svg) + +### 1. The Data Capitalization Opportunity + +We live in an unprecedented era of digital wealth creation. Every day, individuals and enterprises generate massive amounts of valuable digital footprints across various platforms, social media channels, messenger apps, and cloud services. While people can interact with their data within these platforms, there's an immense untapped opportunity to transform this data into true capital assets. Just as physical property became a foundational element of wealth creation, personal and enterprise data has the potential to become a new form of capital on balance sheets. + +For individuals, this represents an opportunity to transform their digital activities into valuable assets. For enterprises, their internal communications, team discussions, and collaborative documents contain rich insights that could be structured and valued as intellectual capital. This wealth of information represents an unprecedented opportunity for value creation in the digital age. + +### 2. The Potential of Authentic Data + +While synthetic data has played a crucial role in AI development, there's an enormous untapped potential in the authentic data generated by individuals and organizations. Every message, document, and interaction contains unique insights and patterns that could enhance AI development. The challenge isn't a lack of data - it's that most authentic human-generated data remains inaccessible for productive use. + +By enabling willing participation in data sharing, we can unlock this vast reservoir of authentic human knowledge. This represents an opportunity to enhance AI development with diverse, real-world data that reflects the full spectrum of human experience and knowledge. + +## Our Pathway to Data Democracy + +### 1. Open-Source Foundation + +Our first step is creating an open-source data extraction engine that empowers developers and innovators to build tools for data structuring and organization. This foundation ensures transparency, security, and community-driven development. By making these tools openly available, we enable the technical infrastructure needed for true data ownership and capitalization. + +### 2. Data Capitalization Platform + +Building on this open-source foundation, we're developing a platform that helps individuals and enterprises transform their digital footprints into structured, valuable assets. This platform will provide the tools and frameworks needed to organize, understand, and value personal and organizational data as true capital assets. + +### 3. Creating a Data Marketplace + +The final piece is establishing a marketplace where individuals and organizations can willingly share their data assets. This creates opportunities for: +- Individuals to earn equity, revenue, or other forms of value from their data +- Enterprises to access diverse, high-quality data for AI development +- Researchers to work with authentic human-generated data +- Startups to build innovative solutions using real-world data + +## Economic Vision: A Shared Data Economy + +We envision a future where data becomes a fundamental asset class in a thriving shared economy. This transformation will democratize AI development by enabling willing participation in data sharing, ensuring that the benefits of AI advancement flow back to data creators. Just as property rights revolutionized economic systems, establishing data as a capital asset will create new opportunities for wealth creation and economic participation. + +This shared data economy will: +- Enable individuals to capitalize on their digital footprints +- Create new revenue streams for data creators +- Provide AI developers with access to diverse, authentic data +- Foster innovation through broader access to real-world data +- Ensure more equitable distribution of AI's economic benefits + +Our vision is to facilitate this transformation from the ground up - starting with open-source tools, progressing to data capitalization platforms, and ultimately creating a thriving marketplace where data becomes a true asset class in a shared economy. This approach ensures that the future of AI is built on a foundation of authentic human knowledge, with benefits flowing back to the individuals and organizations who create and share their valuable data. \ No newline at end of file diff --git a/PROGRESSIVE_CRAWLING.md b/PROGRESSIVE_CRAWLING.md new file mode 100644 index 0000000..1d710bd --- /dev/null +++ b/PROGRESSIVE_CRAWLING.md @@ -0,0 +1,320 @@ +# Progressive Web Crawling with Adaptive Information Foraging + +## Abstract + +This paper presents a novel approach to web crawling that adaptively determines when sufficient information has been gathered to answer a given query. Unlike traditional exhaustive crawling methods, our Progressive Information Sufficiency (PIS) framework uses statistical measures to balance information completeness against crawling efficiency. We introduce a multi-strategy architecture supporting pure statistical, embedding-enhanced, and LLM-assisted approaches, with theoretical guarantees on convergence and practical evaluation methods using synthetic datasets. + +## 1. Introduction + +Traditional web crawling approaches follow predetermined patterns (breadth-first, depth-first) without consideration for information sufficiency. This work addresses the fundamental question: *"When do we have enough information to answer a query and similar queries in its domain?"* + +We formalize this as an optimal stopping problem in information foraging, introducing metrics for coverage, consistency, and saturation that enable crawlers to make intelligent decisions about when to stop crawling and which links to follow. + +## 2. Problem Formulation + +### 2.1 Definitions + +Let: +- **K** = {d₁, d₂, ..., dₙ} be the current knowledge base (crawled documents) +- **Q** be the user query +- **L** = {l₁, l₂, ..., lₘ} be available links with preview metadata +- **θ** be the confidence threshold for information sufficiency + +### 2.2 Objectives + +1. **Minimize** |K| (number of crawled pages) +2. **Maximize** P(answers(Q) | K) (probability of answering Q given K) +3. **Ensure** coverage of Q's domain (similar queries) + +## 3. Mathematical Framework + +### 3.1 Information Sufficiency Metric + +We define Information Sufficiency as: + +``` +IS(K, Q) = min(Coverage(K, Q), Consistency(K, Q), 1 - Redundancy(K)) × DomainCoverage(K, Q) +``` + +### 3.2 Coverage Score + +Coverage measures how well current knowledge covers query terms and related concepts: + +``` +Coverage(K, Q) = Σ(t ∈ Q) log(df(t, K) + 1) × idf(t) / |Q| +``` + +Where: +- df(t, K) = document frequency of term t in knowledge base K +- idf(t) = inverse document frequency weight + +### 3.3 Consistency Score + +Consistency measures information coherence across documents: + +``` +Consistency(K, Q) = 1 - Var(answers from random subsets of K) +``` + +This captures the principle that sufficient knowledge should provide stable answers regardless of document subset. + +### 3.4 Saturation Score + +Saturation detects diminishing returns: + +``` +Saturation(K) = 1 - (ΔInfo(Kₙ) / ΔInfo(K₁)) +``` + +Where ΔInfo represents marginal information gain from the nth crawl. + +### 3.5 Link Value Prediction + +Expected information gain from uncrawled links: + +``` +ExpectedGain(l) = Relevance(l, Q) × Novelty(l, K) × Authority(l) +``` + +Components: +- **Relevance**: BM25(preview_text, Q) +- **Novelty**: 1 - max_similarity(preview, K) +- **Authority**: f(url_structure, domain_metrics) + +## 4. Algorithmic Approach + +### 4.1 Progressive Crawling Algorithm + +``` +Algorithm: ProgressiveCrawl(start_url, query, θ) + K ← ∅ + crawled ← {start_url} + pending ← extract_links(crawl(start_url)) + + while IS(K, Q) < θ and |crawled| < max_pages: + candidates ← rank_by_expected_gain(pending, Q, K) + if max(ExpectedGain(candidates)) < min_gain: + break // Diminishing returns + + to_crawl ← top_k(candidates) + new_docs ← parallel_crawl(to_crawl) + K ← K ∪ new_docs + crawled ← crawled ∪ to_crawl + pending ← extract_new_links(new_docs) - crawled + + return K +``` + +### 4.2 Stopping Criteria + +Crawling terminates when: +1. IS(K, Q) ≥ θ (sufficient information) +2. d(IS)/d(crawls) < ε (plateau reached) +3. |crawled| ≥ max_pages (resource limit) +4. max(ExpectedGain) < min_gain (no promising links) + +## 5. Multi-Strategy Architecture + +### 5.1 Strategy Pattern Design + +``` +AbstractStrategy + ├── StatisticalStrategy (no LLM, no embeddings) + ├── EmbeddingStrategy (with semantic similarity) + └── LLMStrategy (with language model assistance) +``` + +### 5.2 Statistical Strategy + +Pure statistical approach using: +- BM25 for relevance scoring +- Term frequency analysis for coverage +- Graph structure for authority +- No external models required + +**Advantages**: Fast, no API costs, works offline +**Best for**: Technical documentation, specific terminology + +### 5.3 Embedding Strategy (Implemented) + +Semantic understanding through embeddings: +- Query expansion into semantic variations +- Coverage mapping in embedding space +- Gap-driven link selection +- Validation-based stopping criteria + +**Mathematical Framework**: +``` +Coverage(K, Q) = mean(max_similarity(q, K) for q in Q_expanded) +Gap(q) = 1 - max_similarity(q, K) +LinkScore(l) = Σ(Gap(q) × relevance(l, q)) × (1 - redundancy(l, K)) +``` + +**Key Parameters**: +- `embedding_k_exp`: Exponential decay factor for distance-to-score mapping +- `embedding_coverage_radius`: Distance threshold for query coverage +- `embedding_min_confidence_threshold`: Minimum relevance threshold + +**Advantages**: Semantic understanding, handles ambiguity, detects irrelevance +**Best for**: Research queries, conceptual topics, diverse content + +### 5.4 Progressive Enhancement Path + +1. **Level 0**: Statistical only (implemented) +2. **Level 1**: + Embeddings for semantic similarity (implemented) +3. **Level 2**: + LLM for query understanding (future) + +## 6. Evaluation Methodology + +### 6.1 Synthetic Dataset Generation + +Using LLM to create evaluation data: + +```python +def generate_synthetic_dataset(domain_url): + # 1. Fully crawl domain + full_knowledge = exhaustive_crawl(domain_url) + + # 2. Generate answerable queries + queries = llm_generate_queries(full_knowledge) + + # 3. Create query variations + for q in queries: + variations = generate_variations(q) # synonyms, sub/super queries + + return queries, variations, full_knowledge +``` + +### 6.2 Evaluation Metrics + +1. **Efficiency**: Information gained / Pages crawled +2. **Completeness**: Answerable queries / Total queries +3. **Redundancy**: 1 - (Unique information / Total information) +4. **Convergence Rate**: Pages to 95% completeness + +### 6.3 Ablation Studies + +- Impact of each score component (coverage, consistency, saturation) +- Sensitivity to threshold parameters +- Performance across different domain types + +## 7. Theoretical Properties + +### 7.1 Convergence Guarantee + +**Theorem**: For finite websites, ProgressiveCrawl converges to IS(K, Q) ≥ θ or exhausts all reachable pages. + +**Proof sketch**: IS(K, Q) is monotonically non-decreasing with each crawl, bounded above by 1. + +### 7.2 Optimality + +Under certain assumptions about link preview accuracy: +- Expected crawls ≤ 2 × optimal_crawls +- Approximation ratio improves with preview quality + +## 8. Implementation Design + +### 8.1 Core Components + +1. **CrawlState**: Maintains crawl history and metrics +2. **AdaptiveConfig**: Configuration parameters +3. **CrawlStrategy**: Pluggable strategy interface +4. **AdaptiveCrawler**: Main orchestrator + +### 8.2 Integration with Crawl4AI + +- Wraps existing AsyncWebCrawler +- Leverages link preview functionality +- Maintains backward compatibility + +### 8.3 Persistence + +Knowledge base serialization for: +- Resumable crawls +- Knowledge sharing +- Offline analysis + +## 9. Future Directions + +### 9.1 Advanced Scoring + +- Temporal information value +- Multi-query optimization +- Active learning from user feedback + +### 9.2 Distributed Crawling + +- Collaborative knowledge building +- Federated information sufficiency + +### 9.3 Domain Adaptation + +- Transfer learning across domains +- Meta-learning for threshold selection + +## 10. Conclusion + +Progressive crawling with adaptive information foraging provides a principled approach to efficient web information extraction. By combining coverage, consistency, and saturation metrics, we can determine information sufficiency without ground truth labels. The multi-strategy architecture allows graceful enhancement from pure statistical to LLM-assisted approaches based on requirements and resources. + +## References + +1. Manning, C. D., Raghavan, P., & Schütze, H. (2008). Introduction to Information Retrieval. Cambridge University Press. + +2. Robertson, S., & Zaragoza, H. (2009). The Probabilistic Relevance Framework: BM25 and Beyond. Foundations and Trends in Information Retrieval. + +3. Pirolli, P., & Card, S. (1999). Information Foraging. Psychological Review, 106(4), 643-675. + +4. Dasgupta, S. (2005). Analysis of a greedy active learning strategy. Advances in Neural Information Processing Systems. + +## Appendix A: Implementation Pseudocode + +```python +class StatisticalStrategy: + def calculate_confidence(self, state): + coverage = self.calculate_coverage(state) + consistency = self.calculate_consistency(state) + saturation = self.calculate_saturation(state) + return min(coverage, consistency, saturation) + + def calculate_coverage(self, state): + # BM25-based term coverage + term_scores = [] + for term in state.query.split(): + df = state.document_frequencies.get(term, 0) + idf = self.idf_cache.get(term, 1.0) + term_scores.append(log(df + 1) * idf) + return mean(term_scores) / max_possible_score + + def rank_links(self, state): + scored_links = [] + for link in state.pending_links: + relevance = self.bm25_score(link.preview_text, state.query) + novelty = self.calculate_novelty(link, state.knowledge_base) + authority = self.url_authority(link.href) + score = relevance * novelty * authority + scored_links.append((link, score)) + return sorted(scored_links, key=lambda x: x[1], reverse=True) +``` + +## Appendix B: Evaluation Protocol + +1. **Dataset Creation**: + - Select diverse domains (documentation, blogs, e-commerce) + - Generate 100 queries per domain using LLM + - Create query variations (5-10 per query) + +2. **Baseline Comparisons**: + - BFS crawler (depth-limited) + - DFS crawler (depth-limited) + - Random crawler + - Oracle (knows relevant pages) + +3. **Metrics Collection**: + - Pages crawled vs query answerability + - Time to sufficient confidence + - False positive/negative rates + +4. **Statistical Analysis**: + - ANOVA for strategy comparison + - Regression for parameter sensitivity + - Bootstrap for confidence intervals \ No newline at end of file diff --git a/README-first.md b/README-first.md new file mode 100644 index 0000000..2a21df3 --- /dev/null +++ b/README-first.md @@ -0,0 +1,809 @@ +# 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper. + +
+ +unclecode%2Fcrawl4ai | Trendshift + +[![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers) +[![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members) + +[![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai) +[![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai)](https://pypi.org/project/crawl4ai/) +[![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai) +[![GitHub Sponsors](https://img.shields.io/github/sponsors/unclecode?style=flat&logo=GitHub-Sponsors&label=Sponsors&color=pink)](https://github.com/sponsors/unclecode) + +

+ + Follow on X + + + Follow on LinkedIn + + + Join our Discord + +

+
+ +Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for LLMs, AI agents, and data pipelines. Open source, flexible, and built for real-time performance, Crawl4AI empowers developers with unmatched speed, precision, and deployment ease. + +[✨ Check out latest update v0.7.0](#-recent-updates) + +🎉 **Version 0.7.0 is now available!** The Adaptive Intelligence Update introduces groundbreaking features: Adaptive Crawling that learns website patterns, Virtual Scroll support for infinite pages, intelligent Link Preview with 3-layer scoring, Async URL Seeder for massive discovery, and significant performance improvements. [Read the release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.0.md) + +
+🤓 My Personal Story + +My journey with computers started in childhood when my dad, a computer scientist, introduced me to an Amstrad computer. Those early days sparked a fascination with technology, leading me to pursue computer science and specialize in NLP during my postgraduate studies. It was during this time that I first delved into web crawling, building tools to help researchers organize papers and extract information from publications a challenging yet rewarding experience that honed my skills in data extraction. + +Fast forward to 2023, I was working on a tool for a project and needed a crawler to convert a webpage into markdown. While exploring solutions, I found one that claimed to be open-source but required creating an account and generating an API token. Worse, it turned out to be a SaaS model charging $16, and its quality didn’t meet my standards. Frustrated, I realized this was a deeper problem. That frustration turned into turbo anger mode, and I decided to build my own solution. In just a few days, I created Crawl4AI. To my surprise, it went viral, earning thousands of GitHub stars and resonating with a global community. + +I made Crawl4AI open-source for two reasons. First, it’s my way of giving back to the open-source community that has supported me throughout my career. Second, I believe data should be accessible to everyone, not locked behind paywalls or monopolized by a few. Open access to data lays the foundation for the democratization of AI, a vision where individuals can train their own models and take ownership of their information. This library is the first step in a larger journey to create the best open-source data extraction and generation tool the world has ever seen, built collaboratively by a passionate community. + +Thank you to everyone who has supported this project, used it, and shared feedback. Your encouragement motivates me to dream even bigger. Join us, file issues, submit PRs, or spread the word. Together, we can build a tool that truly empowers people to access their own data and reshape the future of AI. +
+ +## 🧐 Why Crawl4AI? + +1. **Built for LLMs**: Creates smart, concise Markdown optimized for RAG and fine-tuning applications. +2. **Lightning Fast**: Delivers results faster with real-time, cost-efficient performance. +3. **Flexible Browser Control**: Offers session management, proxies, and custom hooks for seamless data access. +4. **Heuristic Intelligence**: Uses advanced algorithms for efficient extraction, reducing reliance on costly models. +5. **Open Source & Deployable**: Fully open-source with no API keys—ready for Docker and cloud integration. +6. **Thriving Community**: Actively maintained by a vibrant community and the #1 trending GitHub repository. + +## 🚀 Quick Start + +1. Install Crawl4AI: +```bash +# Install the package +pip install -U crawl4ai + +# For pre release versions +pip install crawl4ai --pre + +# Run post-installation setup +crawl4ai-setup + +# Verify your installation +crawl4ai-doctor +``` + +If you encounter any browser-related issues, you can install them manually: +```bash +python -m playwright install --with-deps chromium +``` + +2. Run a simple web crawl with Python: +```python +import asyncio +from crawl4ai import * + +async def main(): + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://www.nbcnews.com/business", + ) + print(result.markdown) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +3. Or use the new command-line interface: +```bash +# Basic crawl with markdown output +crwl https://www.nbcnews.com/business -o markdown + +# Deep crawl with BFS strategy, max 10 pages +crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10 + +# Use LLM extraction with a specific question +crwl https://www.example.com/products -q "Extract all product prices" +``` + +## ✨ Features + +
+📝 Markdown Generation + +- 🧹 **Clean Markdown**: Generates clean, structured Markdown with accurate formatting. +- 🎯 **Fit Markdown**: Heuristic-based filtering to remove noise and irrelevant parts for AI-friendly processing. +- 🔗 **Citations and References**: Converts page links into a numbered reference list with clean citations. +- 🛠️ **Custom Strategies**: Users can create their own Markdown generation strategies tailored to specific needs. +- 📚 **BM25 Algorithm**: Employs BM25-based filtering for extracting core information and removing irrelevant content. +
+ +
+📊 Structured Data Extraction + +- 🤖 **LLM-Driven Extraction**: Supports all LLMs (open-source and proprietary) for structured data extraction. +- 🧱 **Chunking Strategies**: Implements chunking (topic-based, regex, sentence-level) for targeted content processing. +- 🌌 **Cosine Similarity**: Find relevant content chunks based on user queries for semantic extraction. +- 🔎 **CSS-Based Extraction**: Fast schema-based data extraction using XPath and CSS selectors. +- 🔧 **Schema Definition**: Define custom schemas for extracting structured JSON from repetitive patterns. + +
+ +
+🌐 Browser Integration + +- 🖥️ **Managed Browser**: Use user-owned browsers with full control, avoiding bot detection. +- 🔄 **Remote Browser Control**: Connect to Chrome Developer Tools Protocol for remote, large-scale data extraction. +- 👤 **Browser Profiler**: Create and manage persistent profiles with saved authentication states, cookies, and settings. +- 🔒 **Session Management**: Preserve browser states and reuse them for multi-step crawling. +- 🧩 **Proxy Support**: Seamlessly connect to proxies with authentication for secure access. +- ⚙️ **Full Browser Control**: Modify headers, cookies, user agents, and more for tailored crawling setups. +- 🌍 **Multi-Browser Support**: Compatible with Chromium, Firefox, and WebKit. +- 📐 **Dynamic Viewport Adjustment**: Automatically adjusts the browser viewport to match page content, ensuring complete rendering and capturing of all elements. + +
+ +
+🔎 Crawling & Scraping + +- 🖼️ **Media Support**: Extract images, audio, videos, and responsive image formats like `srcset` and `picture`. +- 🚀 **Dynamic Crawling**: Execute JS and wait for async or sync for dynamic content extraction. +- 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis. +- 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`). +- 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content. +- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior. +- 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches. +- 📄 **Metadata Extraction**: Retrieve structured metadata from web pages. +- 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content. +- 🕵️ **Lazy Load Handling**: Waits for images to fully load, ensuring no content is missed due to lazy loading. +- 🔄 **Full-Page Scanning**: Simulates scrolling to load and capture all dynamic content, perfect for infinite scroll pages. + +
+ +
+🚀 Deployment + +- 🐳 **Dockerized Setup**: Optimized Docker image with FastAPI server for easy deployment. +- 🔑 **Secure Authentication**: Built-in JWT token authentication for API security. +- 🔄 **API Gateway**: One-click deployment with secure token authentication for API-based workflows. +- 🌐 **Scalable Architecture**: Designed for mass-scale production and optimized server performance. +- ☁️ **Cloud Deployment**: Ready-to-deploy configurations for major cloud platforms. + +
+ +
+🎯 Additional Features + +- 🕶️ **Stealth Mode**: Avoid bot detection by mimicking real users. +- 🏷️ **Tag-Based Content Extraction**: Refine crawling based on custom tags, headers, or metadata. +- 🔗 **Link Analysis**: Extract and analyze all links for detailed data exploration. +- 🛡️ **Error Handling**: Robust error management for seamless execution. +- 🔐 **CORS & Static Serving**: Supports filesystem-based caching and cross-origin requests. +- 📖 **Clear Documentation**: Simplified and updated guides for onboarding and advanced usage. +- 🙌 **Community Recognition**: Acknowledges contributors and pull requests for transparency. + +
+ +## Try it Now! + +✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing) + +✨ Visit our [Documentation Website](https://docs.crawl4ai.com/) + +## Installation 🛠️ + +Crawl4AI offers flexible installation options to suit various use cases. You can install it as a Python package or use Docker. + +
+🐍 Using pip + +Choose the installation option that best fits your needs: + +### Basic Installation + +For basic web crawling and scraping tasks: + +```bash +pip install crawl4ai +crawl4ai-setup # Setup the browser +``` + +By default, this will install the asynchronous version of Crawl4AI, using Playwright for web crawling. + +👉 **Note**: When you install Crawl4AI, the `crawl4ai-setup` should automatically install and set up Playwright. However, if you encounter any Playwright-related errors, you can manually install it using one of these methods: + +1. Through the command line: + + ```bash + playwright install + ``` + +2. If the above doesn't work, try this more specific command: + + ```bash + python -m playwright install chromium + ``` + +This second method has proven to be more reliable in some cases. + +--- + +### Installation with Synchronous Version + +The sync version is deprecated and will be removed in future versions. If you need the synchronous version using Selenium: + +```bash +pip install crawl4ai[sync] +``` + +--- + +### Development Installation + +For contributors who plan to modify the source code: + +```bash +git clone https://github.com/unclecode/crawl4ai.git +cd crawl4ai +pip install -e . # Basic installation in editable mode +``` + +Install optional features: + +```bash +pip install -e ".[torch]" # With PyTorch features +pip install -e ".[transformer]" # With Transformer features +pip install -e ".[cosine]" # With cosine similarity features +pip install -e ".[sync]" # With synchronous crawling (Selenium) +pip install -e ".[all]" # Install all optional features +``` + +
+ +
+🐳 Docker Deployment + +> 🚀 **Now Available!** Our completely redesigned Docker implementation is here! This new solution makes deployment more efficient and seamless than ever. + +### New Docker Features + +The new Docker implementation includes: +- **Browser pooling** with page pre-warming for faster response times +- **Interactive playground** to test and generate request code +- **MCP integration** for direct connection to AI tools like Claude Code +- **Comprehensive API endpoints** including HTML extraction, screenshots, PDF generation, and JavaScript execution +- **Multi-architecture support** with automatic detection (AMD64/ARM64) +- **Optimized resources** with improved memory management + +### Getting Started + +```bash +# Pull and run the latest release candidate +docker pull unclecode/crawl4ai:0.7.0 +docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:0.7.0 + +# Visit the playground at http://localhost:11235/playground +``` + +For complete documentation, see our [Docker Deployment Guide](https://docs.crawl4ai.com/core/docker-deployment/). + +
+ +--- + +### Quick Test + +Run a quick test (works for both Docker options): + +```python +import requests + +# Submit a crawl job +response = requests.post( + "http://localhost:11235/crawl", + json={"urls": ["https://example.com"], "priority": 10} +) +if response.status_code == 200: + print("Crawl job submitted successfully.") + +if "results" in response.json(): + results = response.json()["results"] + print("Crawl job completed. Results:") + for result in results: + print(result) +else: + task_id = response.json()["task_id"] + print(f"Crawl job submitted. Task ID:: {task_id}") + result = requests.get(f"http://localhost:11235/task/{task_id}") +``` + +For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://docs.crawl4ai.com/basic/docker-deployment/). + + + + +## 🔬 Advanced Usage Examples 🔬 + +You can check the project structure in the directory [docs/examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples). Over there, you can find a variety of examples; here, some popular examples are shared. + +
+📝 Heuristic Markdown Generation with Clean and Fit Markdown + +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode +from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator + +async def main(): + browser_config = BrowserConfig( + headless=True, + verbose=True, + ) + run_config = CrawlerRunConfig( + cache_mode=CacheMode.ENABLED, + markdown_generator=DefaultMarkdownGenerator( + content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed", min_word_threshold=0) + ), + # markdown_generator=DefaultMarkdownGenerator( + # content_filter=BM25ContentFilter(user_query="WHEN_WE_FOCUS_BASED_ON_A_USER_QUERY", bm25_threshold=1.0) + # ), + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun( + url="https://docs.micronaut.io/4.7.6/guide/", + config=run_config + ) + print(len(result.markdown.raw_markdown)) + print(len(result.markdown.fit_markdown)) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +
+ +
+🖥️ Executing JavaScript & Extract Structured Data without LLMs + +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode +from crawl4ai import JsonCssExtractionStrategy +import json + +async def main(): + schema = { + "name": "KidoCode Courses", + "baseSelector": "section.charge-methodology .w-tab-content > div", + "fields": [ + { + "name": "section_title", + "selector": "h3.heading-50", + "type": "text", + }, + { + "name": "section_description", + "selector": ".charge-content", + "type": "text", + }, + { + "name": "course_name", + "selector": ".text-block-93", + "type": "text", + }, + { + "name": "course_description", + "selector": ".course-content-text", + "type": "text", + }, + { + "name": "course_icon", + "selector": ".image-92", + "type": "attribute", + "attribute": "src" + } + } +} + + extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True) + + browser_config = BrowserConfig( + headless=False, + verbose=True + ) + run_config = CrawlerRunConfig( + extraction_strategy=extraction_strategy, + js_code=["""(async () => {const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");for(let tab of tabs) {tab.scrollIntoView();tab.click();await new Promise(r => setTimeout(r, 500));}})();"""], + cache_mode=CacheMode.BYPASS + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + + result = await crawler.arun( + url="https://www.kidocode.com/degrees/technology", + config=run_config + ) + + companies = json.loads(result.extracted_content) + print(f"Successfully extracted {len(companies)} companies") + print(json.dumps(companies[0], indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +
+ +
+📚 Extracting Structured Data with LLMs + +```python +import os +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig +from crawl4ai import LLMExtractionStrategy +from pydantic import BaseModel, Field + +class OpenAIModelFee(BaseModel): + model_name: str = Field(..., description="Name of the OpenAI model.") + input_fee: str = Field(..., description="Fee for input token for the OpenAI model.") + output_fee: str = Field(..., description="Fee for output token for the OpenAI model.") + +async def main(): + browser_config = BrowserConfig(verbose=True) + run_config = CrawlerRunConfig( + word_count_threshold=1, + extraction_strategy=LLMExtractionStrategy( + # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2 + # provider="ollama/qwen2", api_token="no-token", + llm_config = LLMConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')), + schema=OpenAIModelFee.schema(), + extraction_type="schema", + instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. + Do not miss any models in the entire content. One extracted model JSON format should look like this: + {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}.""" + ), + cache_mode=CacheMode.BYPASS, + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun( + url='https://openai.com/api/pricing/', + config=run_config + ) + print(result.extracted_content) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +
+ +
+🤖 Using Your own Browser with Custom User Profile + +```python +import os, sys +from pathlib import Path +import asyncio, time +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode + +async def test_news_crawl(): + # Create a persistent user data directory + user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile") + os.makedirs(user_data_dir, exist_ok=True) + + browser_config = BrowserConfig( + verbose=True, + headless=True, + user_data_dir=user_data_dir, + use_persistent_context=True, + ) + run_config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + url = "ADDRESS_OF_A_CHALLENGING_WEBSITE" + + result = await crawler.arun( + url, + config=run_config, + magic=True, + ) + + print(f"Successfully crawled {url}") + print(f"Content length: {len(result.markdown)}") +``` + +
+ +## ✨ Recent Updates + +### Version 0.7.0 Release Highlights - The Adaptive Intelligence Update + +- **🧠 Adaptive Crawling**: Your crawler now learns and adapts to website patterns automatically: + ```python + config = AdaptiveConfig( + confidence_threshold=0.7, # Min confidence to stop crawling + max_depth=5, # Maximum crawl depth + max_pages=20, # Maximum number of pages to crawl + strategy="statistical" + ) + + async with AsyncWebCrawler() as crawler: + adaptive_crawler = AdaptiveCrawler(crawler, config) + state = await adaptive_crawler.digest( + start_url="https://news.example.com", + query="latest news content" + ) + # Crawler learns patterns and improves extraction over time + ``` + +- **🌊 Virtual Scroll Support**: Complete content extraction from infinite scroll pages: + ```python + scroll_config = VirtualScrollConfig( + container_selector="[data-testid='feed']", + scroll_count=20, + scroll_by="container_height", + wait_after_scroll=1.0 + ) + + result = await crawler.arun(url, config=CrawlerRunConfig( + virtual_scroll_config=scroll_config + )) + ``` + +- **🔗 Intelligent Link Analysis**: 3-layer scoring system for smart link prioritization: + ```python + link_config = LinkPreviewConfig( + query="machine learning tutorials", + score_threshold=0.3, + concurrent_requests=10 + ) + + result = await crawler.arun(url, config=CrawlerRunConfig( + link_preview_config=link_config, + score_links=True + )) + # Links ranked by relevance and quality + ``` + +- **🎣 Async URL Seeder**: Discover thousands of URLs in seconds: + ```python + seeder = AsyncUrlSeeder(SeedingConfig( + source="sitemap+cc", + pattern="*/blog/*", + query="python tutorials", + score_threshold=0.4 + )) + + urls = await seeder.discover("https://example.com") + ``` + +- **⚡ Performance Boost**: Up to 3x faster with optimized resource handling and memory efficiency + +Read the full details in our [0.7.0 Release Notes](https://docs.crawl4ai.com/blog/release-v0.7.0) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md). + +## Version Numbering in Crawl4AI + +Crawl4AI follows standard Python version numbering conventions (PEP 440) to help users understand the stability and features of each release. + +### Version Numbers Explained + +Our version numbers follow this pattern: `MAJOR.MINOR.PATCH` (e.g., 0.4.3) + +#### Pre-release Versions +We use different suffixes to indicate development stages: + +- `dev` (0.4.3dev1): Development versions, unstable +- `a` (0.4.3a1): Alpha releases, experimental features +- `b` (0.4.3b1): Beta releases, feature complete but needs testing +- `rc` (0.4.3): Release candidates, potential final version + +#### Installation +- Regular installation (stable version): + ```bash + pip install -U crawl4ai + ``` + +- Install pre-release versions: + ```bash + pip install crawl4ai --pre + ``` + +- Install specific version: + ```bash + pip install crawl4ai==0.4.3b1 + ``` + +#### Why Pre-releases? +We use pre-releases to: +- Test new features in real-world scenarios +- Gather feedback before final releases +- Ensure stability for production users +- Allow early adopters to try new features + +For production environments, we recommend using the stable version. For testing new features, you can opt-in to pre-releases using the `--pre` flag. + +## 📖 Documentation & Roadmap + +> 🚨 **Documentation Update Alert**: We're undertaking a major documentation overhaul next week to reflect recent updates and improvements. Stay tuned for a more comprehensive and up-to-date guide! + +For current documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://docs.crawl4ai.com/). + +To check our development plans and upcoming features, visit our [Roadmap](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md). + +
+📈 Development TODOs + +- [x] 0. Graph Crawler: Smart website traversal using graph search algorithms for comprehensive nested page extraction +- [ ] 1. Question-Based Crawler: Natural language driven web discovery and content extraction +- [ ] 2. Knowledge-Optimal Crawler: Smart crawling that maximizes knowledge while minimizing data extraction +- [ ] 3. Agentic Crawler: Autonomous system for complex multi-step crawling operations +- [ ] 4. Automated Schema Generator: Convert natural language to extraction schemas +- [ ] 5. Domain-Specific Scrapers: Pre-configured extractors for common platforms (academic, e-commerce) +- [ ] 6. Web Embedding Index: Semantic search infrastructure for crawled content +- [ ] 7. Interactive Playground: Web UI for testing, comparing strategies with AI assistance +- [ ] 8. Performance Monitor: Real-time insights into crawler operations +- [ ] 9. Cloud Integration: One-click deployment solutions across cloud providers +- [ ] 10. Sponsorship Program: Structured support system with tiered benefits +- [ ] 11. Educational Content: "How to Crawl" video series and interactive tutorials + +
+ +## 🤝 Contributing + +We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information. + +I'll help modify the license section with badges. For the halftone effect, here's a version with it: + +Here's the updated license section: + +## 📄 License & Attribution + +This project is licensed under the Apache License 2.0, attribution is recommended via the badges below. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details. + +### Attribution Requirements +When using Crawl4AI, you must include one of the following attribution methods: + +#### 1. Badge Attribution (Recommended) +Add one of these badges to your README, documentation, or website: + +| Theme | Badge | +|-------|-------| +| **Disco Theme (Animated)** | Powered by Crawl4AI | +| **Night Theme (Dark with Neon)** | Powered by Crawl4AI | +| **Dark Theme (Classic)** | Powered by Crawl4AI | +| **Light Theme (Classic)** | Powered by Crawl4AI | + + +HTML code for adding the badges: +```html + + + Powered by Crawl4AI + + + + + Powered by Crawl4AI + + + + + Powered by Crawl4AI + + + + + Powered by Crawl4AI + + + + + Powered by Crawl4AI + +``` + +#### 2. Text Attribution +Add this line to your documentation: +``` +This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction. +``` + +## 📚 Citation + +If you use Crawl4AI in your research or project, please cite: + +```bibtex +@software{crawl4ai2024, + author = {UncleCode}, + title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper}, + year = {2024}, + publisher = {GitHub}, + journal = {GitHub Repository}, + howpublished = {\url{https://github.com/unclecode/crawl4ai}}, + commit = {Please use the commit hash you're working with} +} +``` + +Text citation format: +``` +UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software]. +GitHub. https://github.com/unclecode/crawl4ai +``` + +## 📧 Contact + +For questions, suggestions, or feedback, feel free to reach out: + +- GitHub: [unclecode](https://github.com/unclecode) +- Twitter: [@unclecode](https://twitter.com/unclecode) +- Website: [crawl4ai.com](https://crawl4ai.com) + +Happy Crawling! 🕸️🚀 + +## 💖 Support Crawl4AI + +> 🎉 **Sponsorship Program Just Launched!** Be among the first 50 **Founding Sponsors** and get permanent recognition in our Hall of Fame! + +Crawl4AI is the #1 trending open-source web crawler with 51K+ stars. Your support ensures we stay independent, innovative, and free forever. + +
+ +[![Become a Sponsor](https://img.shields.io/badge/Become%20a%20Sponsor-pink?style=for-the-badge&logo=github-sponsors&logoColor=white)](https://github.com/sponsors/unclecode) +[![Current Sponsors](https://img.shields.io/github/sponsors/unclecode?style=for-the-badge&logo=github&label=Current%20Sponsors&color=green)](https://github.com/sponsors/unclecode) + +
+ +### 🤝 Sponsorship Tiers + +- **🌱 Believer ($5/mo)**: Join the movement for data democratization +- **🚀 Builder ($50/mo)**: Get priority support and early feature access +- **💼 Growing Team ($500/mo)**: Bi-weekly syncs and optimization help +- **🏢 Data Infrastructure Partner ($2000/mo)**: Full partnership with dedicated support + +**Why sponsor?** Every tier includes real benefits. No more rate-limited APIs. Own your data pipeline. Build data sovereignty together. + +[View All Tiers & Benefits →](https://github.com/sponsors/unclecode) + +### 🏆 Our Sponsors + +#### 👑 Founding Sponsors (First 50) +*Be part of history - [Become a Founding Sponsor](https://github.com/sponsors/unclecode)* + + + +#### Current Sponsors +Thank you to all our sponsors who make this project possible! + + + +## 🗾 Mission + +Our mission is to unlock the value of personal and enterprise data by transforming digital footprints into structured, tradeable assets. Crawl4AI empowers individuals and organizations with open-source tools to extract and structure data, fostering a shared data economy. + +We envision a future where AI is powered by real human knowledge, ensuring data creators directly benefit from their contributions. By democratizing data and enabling ethical sharing, we are laying the foundation for authentic AI advancement. + +
+🔑 Key Opportunities + +- **Data Capitalization**: Transform digital footprints into measurable, valuable assets. +- **Authentic AI Data**: Provide AI systems with real human insights. +- **Shared Economy**: Create a fair data marketplace that benefits data creators. + +
+ +
+🚀 Development Pathway + +1. **Open-Source Tools**: Community-driven platforms for transparent data extraction. +2. **Digital Asset Structuring**: Tools to organize and value digital knowledge. +3. **Ethical Data Marketplace**: A secure, fair platform for exchanging structured data. + +For more details, see our [full mission statement](./MISSION.md). +
+ +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date) diff --git a/README.md b/README.md new file mode 100644 index 0000000..20367f5 --- /dev/null +++ b/README.md @@ -0,0 +1,1274 @@ +# 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper. + +
+ +unclecode%2Fcrawl4ai | Trendshift + +[![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers) +[![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members) + +[![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai) +[![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai)](https://pypi.org/project/crawl4ai/) +[![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai) +[![GitHub Sponsors](https://img.shields.io/github/sponsors/unclecode?style=flat&logo=GitHub-Sponsors&label=Sponsors&color=pink)](https://github.com/sponsors/unclecode) + +--- +#### 🚀 Crawl4AI Cloud API — Closed Beta (Launching Soon) +Reliable, large-scale web extraction, now built to be _**drastically more cost-effective**_ than any of the existing solutions. + +👉 **Apply [here](https://forms.gle/E9MyPaNXACnAMaqG7) for early access** +_We’ll be onboarding in phases and working closely with early users. +Limited slots._ + +--- + +

+ + Follow on X + + + Follow on LinkedIn + + + Join our Discord + +

+
+ +Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community. + +[✨ Check out latest update v0.9.1](#-recent-updates) + +✨ **New in v0.9.1**: Patch release with 12 bug fixes across Docker, browser, and core. Adds `preserve_classes`/`preserve_tags` whitelist for PruningContentFilter, fixes Windows browser crash, Docker auth gate UI, HTTP timeout unit mismatch, and more. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.1.md) + +✨ Recent v0.9.0: Major secure-by-default release of the Docker API server. Auth is on by default, the server binds loopback unless given a token, and the request body is now an untrusted trust boundary. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md) + +✨ Recent v0.8.7: Security-hardening release. Fixes critical Docker API vulnerabilities (RCE, SSRF, auth bypass, file write, XSS, hardcoded JWT secret), adds DomainMapper, and ships scraping, deep-crawl, and LLM fixes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.8.7.md) + +✨ Previous v0.8.0: Crash Recovery & Prefetch Mode! Deep crawl crash recovery with `resume_state` and `on_state_change` callbacks for long-running crawls. New `prefetch=True` mode for 5-10x faster URL discovery. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.8.0.md) + +✨ Previous v0.7.8: Stability & Bug Fix Release! 11 bug fixes addressing Docker API issues, LLM extraction improvements, URL handling fixes, and dependency updates. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.8.md) + +
+ 🤓 My Personal Story + +I grew up on an Amstrad, thanks to my dad, and never stopped building. In grad school I specialized in NLP and built crawlers for research. That’s where I learned how much extraction matters. + +In 2023, I needed web-to-Markdown. The “open source” option wanted an account, API token, and $16, and still under-delivered. I went turbo anger mode, built Crawl4AI in days, and it went viral. Now it’s the most-starred crawler on GitHub. + +I made it open source for **availability**, anyone can use it without a gate. Now I’m building the platform for **affordability**, anyone can run serious crawls without breaking the bank. If that resonates, join in, send feedback, or just crawl something amazing. +
+ + +
+ Why developers pick Crawl4AI + +- **LLM ready output**, smart Markdown with headings, tables, code, citation hints +- **Fast in practice**, async browser pool, caching, minimal hops +- **Full control**, sessions, proxies, cookies, user scripts, hooks +- **Adaptive intelligence**, learns site patterns, explores only what matters +- **Deploy anywhere**, zero keys, CLI and Docker, cloud friendly +
+ + +## 🚀 Quick Start + +1. Install Crawl4AI: +```bash +# Install the package +pip install -U crawl4ai + +# For pre release versions +pip install crawl4ai --pre + +# Run post-installation setup +crawl4ai-setup + +# Verify your installation +crawl4ai-doctor +``` + +If you encounter any browser-related issues, you can install them manually: +```bash +python -m playwright install --with-deps chromium +``` + +2. Run a simple web crawl with Python: +```python +import asyncio +from crawl4ai import * + +async def main(): + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://www.nbcnews.com/business", + ) + print(result.markdown) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +3. Or use the new command-line interface: +```bash +# Basic crawl with markdown output +crwl https://www.nbcnews.com/business -o markdown + +# Deep crawl with BFS strategy, max 10 pages +crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10 + +# Use LLM extraction with a specific question +crwl https://www.example.com/products -q "Extract all product prices" +``` + +## 💖 Support Crawl4AI + +> 🎉 **Sponsorship Program Now Open!** After powering 51K+ developers and 1 year of growth, Crawl4AI is launching dedicated support for **startups** and **enterprises**. Be among the first 50 **Founding Sponsors** for permanent recognition in our Hall of Fame. + +Crawl4AI is the #1 trending open-source web crawler on GitHub. Your support keeps it independent, innovative, and free for the community — while giving you direct access to premium benefits. + +
+ +[![Become a Sponsor](https://img.shields.io/badge/Become%20a%20Sponsor-pink?style=for-the-badge&logo=github-sponsors&logoColor=white)](https://github.com/sponsors/unclecode) +[![Current Sponsors](https://img.shields.io/github/sponsors/unclecode?style=for-the-badge&logo=github&label=Current%20Sponsors&color=green)](https://github.com/sponsors/unclecode) + +
+ +### 🤝 Sponsorship Tiers + +- **🌱 Believer ($5/mo)** — Join the movement for data democratization +- **🚀 Builder ($50/mo)** — Priority support & early access to features +- **💼 Growing Team ($500/mo)** — Bi-weekly syncs & optimization help +- **🏢 Data Infrastructure Partner ($2000/mo)** — Full partnership with dedicated support + *Custom arrangements available - see [SPONSORS.md](SPONSORS.md) for details & contact* + +**Why sponsor?** +No rate-limited APIs. No lock-in. Build and own your data pipeline with direct guidance from the creator of Crawl4AI. + +[See All Tiers & Benefits →](https://github.com/sponsors/unclecode) + + +## ✨ Features + +
+📝 Markdown Generation + +- 🧹 **Clean Markdown**: Generates clean, structured Markdown with accurate formatting. +- 🎯 **Fit Markdown**: Heuristic-based filtering to remove noise and irrelevant parts for AI-friendly processing. +- 🔗 **Citations and References**: Converts page links into a numbered reference list with clean citations. +- 🛠️ **Custom Strategies**: Users can create their own Markdown generation strategies tailored to specific needs. +- 📚 **BM25 Algorithm**: Employs BM25-based filtering for extracting core information and removing irrelevant content. +
+ +
+📊 Structured Data Extraction + +- 🤖 **LLM-Driven Extraction**: Supports all LLMs (open-source and proprietary) for structured data extraction. +- 🧱 **Chunking Strategies**: Implements chunking (topic-based, regex, sentence-level) for targeted content processing. +- 🌌 **Cosine Similarity**: Find relevant content chunks based on user queries for semantic extraction. +- 🔎 **CSS-Based Extraction**: Fast schema-based data extraction using XPath and CSS selectors. +- 🔧 **Schema Definition**: Define custom schemas for extracting structured JSON from repetitive patterns. + +
+ +
+🌐 Browser Integration + +- 🖥️ **Managed Browser**: Use user-owned browsers with full control, avoiding bot detection. +- 🔄 **Remote Browser Control**: Connect to Chrome Developer Tools Protocol for remote, large-scale data extraction. +- 👤 **Browser Profiler**: Create and manage persistent profiles with saved authentication states, cookies, and settings. +- 🔒 **Session Management**: Preserve browser states and reuse them for multi-step crawling. +- 🧩 **Proxy Support**: Seamlessly connect to proxies with authentication for secure access. +- ⚙️ **Full Browser Control**: Modify headers, cookies, user agents, and more for tailored crawling setups. +- 🌍 **Multi-Browser Support**: Compatible with Chromium, Firefox, and WebKit. +- 📐 **Dynamic Viewport Adjustment**: Automatically adjusts the browser viewport to match page content, ensuring complete rendering and capturing of all elements. + +
+ +
+🔎 Crawling & Scraping + +- 🖼️ **Media Support**: Extract images, audio, videos, and responsive image formats like `srcset` and `picture`. +- 🚀 **Dynamic Crawling**: Execute JS and wait for async or sync for dynamic content extraction. +- 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis. +- 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`). +- 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content. +- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior (supports both string and function-based APIs). +- 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches. +- 📄 **Metadata Extraction**: Retrieve structured metadata from web pages. +- 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content. +- 🕵️ **Lazy Load Handling**: Waits for images to fully load, ensuring no content is missed due to lazy loading. +- 🔄 **Full-Page Scanning**: Simulates scrolling to load and capture all dynamic content, perfect for infinite scroll pages. + +
+ +
+🚀 Deployment + +- 🐳 **Dockerized Setup**: Optimized Docker image with FastAPI server for easy deployment. +- 🔑 **Secure Authentication**: Built-in JWT token authentication for API security. +- 🔄 **API Gateway**: One-click deployment with secure token authentication for API-based workflows. +- 🌐 **Scalable Architecture**: Designed for mass-scale production and optimized server performance. +- ☁️ **Cloud Deployment**: Ready-to-deploy configurations for major cloud platforms. + +
+ +
+🎯 Additional Features + +- 🕶️ **Stealth Mode**: Avoid bot detection by mimicking real users. +- 🏷️ **Tag-Based Content Extraction**: Refine crawling based on custom tags, headers, or metadata. +- 🔗 **Link Analysis**: Extract and analyze all links for detailed data exploration. +- 🛡️ **Error Handling**: Robust error management for seamless execution. +- 🔐 **CORS & Static Serving**: Supports filesystem-based caching and cross-origin requests. +- 📖 **Clear Documentation**: Simplified and updated guides for onboarding and advanced usage. +- 🙌 **Community Recognition**: Acknowledges contributors and pull requests for transparency. + +
+ +## Try it Now! + +✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing) + +✨ Visit our [Documentation Website](https://docs.crawl4ai.com/) + +## Installation 🛠️ + +Crawl4AI offers flexible installation options to suit various use cases. You can install it as a Python package or use Docker. + +
+🐍 Using pip + +Choose the installation option that best fits your needs: + +### Basic Installation + +For basic web crawling and scraping tasks: + +```bash +pip install crawl4ai +crawl4ai-setup # Setup the browser +``` + +By default, this will install the asynchronous version of Crawl4AI, using Playwright for web crawling. + +👉 **Note**: When you install Crawl4AI, the `crawl4ai-setup` should automatically install and set up Playwright. However, if you encounter any Playwright-related errors, you can manually install it using one of these methods: + +1. Through the command line: + + ```bash + playwright install + ``` + +2. If the above doesn't work, try this more specific command: + + ```bash + python -m playwright install chromium + ``` + +This second method has proven to be more reliable in some cases. + +--- + +### Installation with Synchronous Version + +The sync version is deprecated and will be removed in future versions. If you need the synchronous version using Selenium: + +```bash +pip install crawl4ai[sync] +``` + +--- + +### Development Installation + +For contributors who plan to modify the source code: + +```bash +git clone https://github.com/unclecode/crawl4ai.git +cd crawl4ai +pip install -e . # Basic installation in editable mode +``` + +Install optional features: + +```bash +pip install -e ".[torch]" # With PyTorch features +pip install -e ".[transformer]" # With Transformer features +pip install -e ".[cosine]" # With cosine similarity features +pip install -e ".[sync]" # With synchronous crawling (Selenium) +pip install -e ".[all]" # Install all optional features +``` + +
+ +
+🐳 Docker Deployment + +> 🚀 **Now Available!** Our completely redesigned Docker implementation is here! This new solution makes deployment more efficient and seamless than ever. + +### New Docker Features + +The new Docker implementation includes: +- **Real-time Monitoring Dashboard** with live system metrics and browser pool visibility +- **Browser pooling** with page pre-warming for faster response times +- **Interactive playground** to test and generate request code +- **MCP integration** for direct connection to AI tools like Claude Code +- **Comprehensive API endpoints** including HTML extraction, screenshots, PDF generation, and JavaScript execution +- **Multi-architecture support** with automatic detection (AMD64/ARM64) +- **Optimized resources** with improved memory management + +### Getting Started + +```bash +# Pull and run the latest release +docker pull unclecode/crawl4ai:latest +docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:latest + +# Visit the monitoring dashboard at http://localhost:11235/dashboard +# Or the playground at http://localhost:11235/playground +``` + +### Quick Test + +Run a quick test (works for both Docker options): + +```python +import requests + +# Submit a crawl job +response = requests.post( + "http://localhost:11235/crawl", + json={"urls": ["https://example.com"], "priority": 10} +) +if response.status_code == 200: + print("Crawl job submitted successfully.") + +if "results" in response.json(): + results = response.json()["results"] + print("Crawl job completed. Results:") + for result in results: + print(result) +else: + task_id = response.json()["task_id"] + print(f"Crawl job submitted. Task ID:: {task_id}") + result = requests.get(f"http://localhost:11235/task/{task_id}") +``` + +For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, monitoring features, and production deployment, see our [Self-Hosting Guide](https://docs.crawl4ai.com/core/self-hosting/). + +
+ +--- + +## 🔬 Advanced Usage Examples 🔬 + +You can check the project structure in the directory [docs/examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples). Over there, you can find a variety of examples; here, some popular examples are shared. + +
+📝 Heuristic Markdown Generation with Clean and Fit Markdown + +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode +from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator + +async def main(): + browser_config = BrowserConfig( + headless=True, + verbose=True, + ) + run_config = CrawlerRunConfig( + cache_mode=CacheMode.ENABLED, + markdown_generator=DefaultMarkdownGenerator( + content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed", min_word_threshold=0) + ), + # markdown_generator=DefaultMarkdownGenerator( + # content_filter=BM25ContentFilter(user_query="WHEN_WE_FOCUS_BASED_ON_A_USER_QUERY", bm25_threshold=1.0) + # ), + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun( + url="https://docs.micronaut.io/4.9.9/guide/", + config=run_config + ) + print(len(result.markdown.raw_markdown)) + print(len(result.markdown.fit_markdown)) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +
+ +
+🖥️ Executing JavaScript & Extract Structured Data without LLMs + +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode +from crawl4ai import JsonCssExtractionStrategy +import json + +async def main(): + schema = { + "name": "KidoCode Courses", + "baseSelector": "section.charge-methodology .w-tab-content > div", + "fields": [ + { + "name": "section_title", + "selector": "h3.heading-50", + "type": "text", + }, + { + "name": "section_description", + "selector": ".charge-content", + "type": "text", + }, + { + "name": "course_name", + "selector": ".text-block-93", + "type": "text", + }, + { + "name": "course_description", + "selector": ".course-content-text", + "type": "text", + }, + { + "name": "course_icon", + "selector": ".image-92", + "type": "attribute", + "attribute": "src" + } + ] +} + + extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True) + + browser_config = BrowserConfig( + headless=False, + verbose=True + ) + run_config = CrawlerRunConfig( + extraction_strategy=extraction_strategy, + js_code=["""(async () => {const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");for(let tab of tabs) {tab.scrollIntoView();tab.click();await new Promise(r => setTimeout(r, 500));}})();"""], + cache_mode=CacheMode.BYPASS + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + + result = await crawler.arun( + url="https://www.kidocode.com/degrees/technology", + config=run_config + ) + + companies = json.loads(result.extracted_content) + print(f"Successfully extracted {len(companies)} companies") + print(json.dumps(companies[0], indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +
+ +
+📚 Extracting Structured Data with LLMs + +```python +import os +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig +from crawl4ai import LLMExtractionStrategy +from pydantic import BaseModel, Field + +class OpenAIModelFee(BaseModel): + model_name: str = Field(..., description="Name of the OpenAI model.") + input_fee: str = Field(..., description="Fee for input token for the OpenAI model.") + output_fee: str = Field(..., description="Fee for output token for the OpenAI model.") + +async def main(): + browser_config = BrowserConfig(verbose=True) + run_config = CrawlerRunConfig( + word_count_threshold=1, + extraction_strategy=LLMExtractionStrategy( + # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2 + # provider="ollama/qwen2", api_token="no-token", + llm_config = LLMConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')), + schema=OpenAIModelFee.schema(), + extraction_type="schema", + instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. + Do not miss any models in the entire content. One extracted model JSON format should look like this: + {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}.""" + ), + cache_mode=CacheMode.BYPASS, + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun( + url='https://openai.com/api/pricing/', + config=run_config + ) + print(result.extracted_content) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +
+ +
+🤖 Using Your own Browser with Custom User Profile + +```python +import os, sys +from pathlib import Path +import asyncio, time +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode + +async def test_news_crawl(): + # Create a persistent user data directory + user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile") + os.makedirs(user_data_dir, exist_ok=True) + + browser_config = BrowserConfig( + verbose=True, + headless=True, + user_data_dir=user_data_dir, + use_persistent_context=True, + ) + run_config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + url = "ADDRESS_OF_A_CHALLENGING_WEBSITE" + + result = await crawler.arun( + url, + config=run_config, + magic=True, + ) + + print(f"Successfully crawled {url}") + print(f"Content length: {len(result.markdown)}") +``` + +
+ +--- + +## ✨ Recent Updates + +
+Version 0.9.1 Release Highlights - Bug Fixes & PruningContentFilter Whitelist + +A patch release with 12 bug fixes and one new feature. The new `preserve_classes` / `preserve_tags` parameters for `PruningContentFilter` let you whitelist CSS classes or HTML tags that should never be pruned — useful for protecting short metadata elements like author names and timestamps. + +Bug fixes span Docker (auth gate UI, supervisord/redis dirs, FastAPI compatibility, redis auth), browser (Windows channel crash, context snapshot leak), core (HTTP timeout unit mismatch, best-first ordering), and extraction (html2text table attributes). + +```bash +pip install -U crawl4ai +``` + +[Full v0.9.1 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.1.md) + +
+ +
+Version 0.9.0 Release Highlights - Secure-by-Default Docker Server + +A major, secure-by-default release of the Docker API server. The out-of-the-box deployment is hardened with defense in depth: authentication is on by default, the server binds loopback unless you give it a token, and the network request body is treated as an untrusted trust boundary. + +```bash +pip install -U crawl4ai +``` + +[Migration Guide →](https://github.com/unclecode/crawl4ai/blob/main/deploy/docker/MIGRATION.md) · [Full v0.9.0 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md) + +
+ +
+Version 0.8.7 Release Highlights - Security Hardening, DomainMapper & Community Fixes + +A security-hardening release. Fixes critical Docker API vulnerabilities (AST sandbox escape RCE, hook sandbox RCE, hardcoded JWT secret, SSRF on webhook and crawl endpoints, arbitrary file write, monitor auth bypass, stored XSS, and unauthenticated JS execution), adds the DomainMapper feature, and ships a batch of scraping, deep-crawl, and LLM fixes. If you self-host the Docker API, upgrade immediately. + +```bash +pip install -U crawl4ai +``` + +[Full v0.8.7 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.8.7.md) + +
+ +
+Version 0.8.6 - Security Hotfix: litellm Supply Chain Fix + +Replaced `litellm` dependency with `unclecode-litellm` due to a PyPI supply chain compromise affecting the original package. If you're on v0.8.5 or earlier, upgrade immediately. + +```bash +pip install -U crawl4ai +``` + +
+ +
+Version 0.8.5 Release Highlights - Anti-Bot Detection, Shadow DOM & 60+ Bug Fixes + +Our biggest release since v0.8.0. Anti-bot detection with proxy escalation, Shadow DOM flattening, deep crawl cancellation, and over 60 bug fixes. + +- **🛡️ Anti-Bot Detection & Proxy Escalation**: + - 3-tier detection: known vendors, generic block indicators, structural integrity checks + - Automatic retry with proxy chain and fallback fetch function + ```python + from crawl4ai import CrawlerRunConfig + from crawl4ai.async_configs import ProxyConfig + + config = CrawlerRunConfig( + proxy_config=[ProxyConfig.DIRECT, ProxyConfig(server="http://my-proxy:8080")], + max_retries=2, + fallback_fetch_function=my_web_unlocker, + ) + ``` + +- **🌑 Shadow DOM Flattening**: + - Extract content hidden inside shadow DOM components + ```python + config = CrawlerRunConfig(flatten_shadow_dom=True) + ``` + +- **🛑 Deep Crawl Cancellation**: + - Stop long crawls gracefully with `cancel()` or `should_cancel` callback + - Works with BFS, DFS, and BestFirst strategies + +- **⚙️ Config Defaults API**: + - `set_defaults()` / `get_defaults()` / `reset_defaults()` on BrowserConfig and CrawlerRunConfig + +- **🔒 Critical Security Fixes**: + - RCE via deserialization in Docker `/crawl` endpoint — removed `eval()`, added allowlist + - Redis CVE-2025-49844 (CVSS 10.0) — upgraded to 7.2.7 + +- **60+ Bug Fixes** across browser management, proxy, deep crawling, extraction, CLI, and Docker + +[Full v0.8.5 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.8.5.md) + +
+ +
+Version 0.8.0 Release Highlights - Crash Recovery & Prefetch Mode + +This release introduces crash recovery for deep crawls, a new prefetch mode for fast URL discovery, and critical security fixes for Docker deployments. + +- **🔄 Deep Crawl Crash Recovery**: + - `on_state_change` callback fires after each URL for real-time state persistence + - `resume_state` parameter to continue from a saved checkpoint + - JSON-serializable state for Redis/database storage + - Works with BFS, DFS, and Best-First strategies + ```python + from crawl4ai.deep_crawling import BFSDeepCrawlStrategy + + strategy = BFSDeepCrawlStrategy( + max_depth=3, + resume_state=saved_state, # Continue from checkpoint + on_state_change=save_to_redis, # Called after each URL + ) + ``` + +- **⚡ Prefetch Mode for Fast URL Discovery**: + - `prefetch=True` skips markdown, extraction, and media processing + - 5-10x faster than full processing + - Perfect for two-phase crawling: discover first, process selectively + ```python + config = CrawlerRunConfig(prefetch=True) + result = await crawler.arun("https://example.com", config=config) + # Returns HTML and links only - no markdown generation + ``` + +- **🔒 Security Fixes (Docker API)**: + - Hooks disabled by default (`CRAWL4AI_HOOKS_ENABLED=false`) + - `file://` URLs blocked on API endpoints to prevent LFI + - `__import__` removed from hook execution sandbox + +[Full v0.8.0 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.8.0.md) + +
+ +
+Version 0.7.8 Release Highlights - Stability & Bug Fix Release + +This release focuses on stability with 11 bug fixes addressing issues reported by the community. No new features, but significant improvements to reliability. + +- **🐳 Docker API Fixes**: + - Fixed `ContentRelevanceFilter` deserialization in deep crawl requests (#1642) + - Fixed `ProxyConfig` JSON serialization in `BrowserConfig.to_dict()` (#1629) + - Fixed `.cache` folder permissions in Docker image (#1638) + +- **🤖 LLM Extraction Improvements**: + - Configurable rate limiter backoff with new `LLMConfig` parameters (#1269): + ```python + from crawl4ai import LLMConfig + + config = LLMConfig( + provider="openai/gpt-4o-mini", + backoff_base_delay=5, # Wait 5s on first retry + backoff_max_attempts=5, # Try up to 5 times + backoff_exponential_factor=3 # Multiply delay by 3 each attempt + ) + ``` + - HTML input format support for `LLMExtractionStrategy` (#1178): + ```python + from crawl4ai import LLMExtractionStrategy + + strategy = LLMExtractionStrategy( + llm_config=config, + instruction="Extract table data", + input_format="html" # Now supports: "html", "markdown", "fit_markdown" + ) + ``` + - Fixed raw HTML URL variable - extraction strategies now receive `"Raw HTML"` instead of HTML blob (#1116) + +- **🔗 URL Handling**: + - Fixed relative URL resolution after JavaScript redirects (#1268) + - Fixed import statement formatting in extracted code (#1181) + +- **📦 Dependency Updates**: + - Replaced deprecated PyPDF2 with pypdf (#1412) + - Pydantic v2 ConfigDict compatibility - no more deprecation warnings (#678) + +- **🧠 AdaptiveCrawler**: + - Fixed query expansion to actually use LLM instead of hardcoded mock data (#1621) + +[Full v0.7.8 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.8.md) + +
+ +
+Version 0.7.7 Release Highlights - The Self-Hosting & Monitoring Update + +- **📊 Real-time Monitoring Dashboard**: Interactive web UI with live system metrics and browser pool visibility + ```python + # Access the monitoring dashboard + # Visit: http://localhost:11235/dashboard + + # Real-time metrics include: + # - System health (CPU, memory, network, uptime) + # - Active and completed request tracking + # - Browser pool management (permanent/hot/cold) + # - Janitor cleanup events + # - Error monitoring with full context + ``` + +- **🔌 Comprehensive Monitor API**: Complete REST API for programmatic access to all monitoring data + ```python + import httpx + + async with httpx.AsyncClient() as client: + # System health + health = await client.get("http://localhost:11235/monitor/health") + + # Request tracking + requests = await client.get("http://localhost:11235/monitor/requests") + + # Browser pool status + browsers = await client.get("http://localhost:11235/monitor/browsers") + + # Endpoint statistics + stats = await client.get("http://localhost:11235/monitor/endpoints/stats") + ``` + +- **⚡ WebSocket Streaming**: Real-time updates every 2 seconds for custom dashboards +- **🔥 Smart Browser Pool**: 3-tier architecture (permanent/hot/cold) with automatic promotion and cleanup +- **🧹 Janitor System**: Automatic resource management with event logging +- **🎮 Control Actions**: Manual browser management (kill, restart, cleanup) via API +- **📈 Production Metrics**: 6 critical metrics for operational excellence with Prometheus integration +- **🐛 Critical Bug Fixes**: + - Fixed async LLM extraction blocking issue (#1055) + - Enhanced DFS deep crawl strategy (#1607) + - Fixed sitemap parsing in AsyncUrlSeeder (#1598) + - Resolved browser viewport configuration (#1495) + - Fixed CDP timing with exponential backoff (#1528) + - Security update for pyOpenSSL (>=25.3.0) + +[Full v0.7.7 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.7.md) + +
+ +
+Version 0.7.5 Release Highlights - The Docker Hooks & Security Update + +- **🔧 Docker Hooks System**: Complete pipeline customization with user-provided Python functions at 8 key points +- **✨ Function-Based Hooks API (NEW)**: Write hooks as regular Python functions with full IDE support: + ```python + from crawl4ai import hooks_to_string + from crawl4ai.docker_client import Crawl4aiDockerClient + + # Define hooks as regular Python functions + async def on_page_context_created(page, context, **kwargs): + """Block images to speed up crawling""" + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page + + async def before_goto(page, context, url, **kwargs): + """Add custom headers""" + await page.set_extra_http_headers({'X-Crawl4AI': 'v0.7.5'}) + return page + + # Option 1: Use hooks_to_string() utility for REST API + hooks_code = hooks_to_string({ + "on_page_context_created": on_page_context_created, + "before_goto": before_goto + }) + + # Option 2: Docker client with automatic conversion (Recommended) + client = Crawl4aiDockerClient(base_url="http://localhost:11235") + results = await client.crawl( + urls=["https://httpbin.org/html"], + hooks={ + "on_page_context_created": on_page_context_created, + "before_goto": before_goto + } + ) + # ✓ Full IDE support, type checking, and reusability! + ``` + +- **🤖 Enhanced LLM Integration**: Custom providers with temperature control and base_url configuration +- **🔒 HTTPS Preservation**: Secure internal link handling with `preserve_https_for_internal_links=True` +- **🐍 Python 3.10+ Support**: Modern language features and enhanced performance +- **🛠️ Bug Fixes**: Resolved multiple community-reported issues including URL processing, JWT authentication, and proxy configuration + +[Full v0.7.5 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.5.md) + +
+ +
+Version 0.7.4 Release Highlights - The Intelligent Table Extraction & Performance Update + +- **🚀 LLMTableExtraction**: Revolutionary table extraction with intelligent chunking for massive tables: + ```python + from crawl4ai import LLMTableExtraction, LLMConfig + + # Configure intelligent table extraction + table_strategy = LLMTableExtraction( + llm_config=LLMConfig(provider="openai/gpt-4.1-mini"), + enable_chunking=True, # Handle massive tables + chunk_token_threshold=5000, # Smart chunking threshold + overlap_threshold=100, # Maintain context between chunks + extraction_type="structured" # Get structured data output + ) + + config = CrawlerRunConfig(table_extraction_strategy=table_strategy) + result = await crawler.arun("https://complex-tables-site.com", config=config) + + # Tables are automatically chunked, processed, and merged + for table in result.tables: + print(f"Extracted table: {len(table['data'])} rows") + ``` + +- **⚡ Dispatcher Bug Fix**: Fixed sequential processing bottleneck in arun_many for fast-completing tasks +- **🧹 Memory Management Refactor**: Consolidated memory utilities into main utils module for cleaner architecture +- **🔧 Browser Manager Fixes**: Resolved race conditions in concurrent page creation with thread-safe locking +- **🔗 Advanced URL Processing**: Better handling of raw:// URLs and base tag link resolution +- **🛡️ Enhanced Proxy Support**: Flexible proxy configuration supporting both dict and string formats + +[Full v0.7.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.4.md) + +
+ +
+Version 0.7.3 Release Highlights - The Multi-Config Intelligence Update + +- **🕵️ Undetected Browser Support**: Bypass sophisticated bot detection systems: + ```python + from crawl4ai import AsyncWebCrawler, BrowserConfig + + browser_config = BrowserConfig( + browser_type="undetected", # Use undetected Chrome + headless=True, # Can run headless with stealth + extra_args=[ + "--disable-blink-features=AutomationControlled", + "--disable-web-security" + ] + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun("https://protected-site.com") + # Successfully bypass Cloudflare, Akamai, and custom bot detection + ``` + +- **🎨 Multi-URL Configuration**: Different strategies for different URL patterns in one batch: +```python +from crawl4ai import CrawlerRunConfig, MatchMode, CacheMode + + configs = [ + # Documentation sites - aggressive caching + CrawlerRunConfig( + url_matcher=["*docs*", "*documentation*"], + cache_mode=CacheMode.WRITE_ONLY, + markdown_generator_options={"include_links": True} + ), + + # News/blog sites - fresh content + CrawlerRunConfig( + url_matcher=lambda url: 'blog' in url or 'news' in url, + cache_mode=CacheMode.BYPASS + ), + + # Fallback for everything else + CrawlerRunConfig() + ] + + results = await crawler.arun_many(urls, config=configs) + # Each URL gets the perfect configuration automatically + ``` + +- **🧠 Memory Monitoring**: Track and optimize memory usage during crawling: + ```python + from crawl4ai.memory_utils import MemoryMonitor + + monitor = MemoryMonitor() + monitor.start_monitoring() + + results = await crawler.arun_many(large_url_list) + + report = monitor.get_report() + print(f"Peak memory: {report['peak_mb']:.1f} MB") + print(f"Efficiency: {report['efficiency']:.1f}%") + # Get optimization recommendations + ``` + +- **📊 Enhanced Table Extraction**: Direct DataFrame conversion from web tables: + ```python + result = await crawler.arun("https://site-with-tables.com") + + # New way - direct table access + if result.tables: + import pandas as pd + for table in result.tables: + df = pd.DataFrame(table['data']) + print(f"Table: {df.shape[0]} rows × {df.shape[1]} columns") + ``` + +- **💰 GitHub Sponsors**: 4-tier sponsorship system for project sustainability +- **🐳 Docker LLM Flexibility**: Configure providers via environment variables + +[Full v0.7.3 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.3.md) + +
+ +
+Version 0.7.0 Release Highlights - The Adaptive Intelligence Update + +- **🧠 Adaptive Crawling**: Your crawler now learns and adapts to website patterns automatically: + ```python + config = AdaptiveConfig( + confidence_threshold=0.7, # Min confidence to stop crawling + max_depth=5, # Maximum crawl depth + max_pages=20, # Maximum number of pages to crawl + strategy="statistical" + ) + + async with AsyncWebCrawler() as crawler: + adaptive_crawler = AdaptiveCrawler(crawler, config) + state = await adaptive_crawler.digest( + start_url="https://news.example.com", + query="latest news content" + ) + # Crawler learns patterns and improves extraction over time + ``` + +- **🌊 Virtual Scroll Support**: Complete content extraction from infinite scroll pages: + ```python + scroll_config = VirtualScrollConfig( + container_selector="[data-testid='feed']", + scroll_count=20, + scroll_by="container_height", + wait_after_scroll=1.0 + ) + + result = await crawler.arun(url, config=CrawlerRunConfig( + virtual_scroll_config=scroll_config + )) + ``` + +- **🔗 Intelligent Link Analysis**: 3-layer scoring system for smart link prioritization: + ```python + link_config = LinkPreviewConfig( + query="machine learning tutorials", + score_threshold=0.3, + concurrent_requests=10 + ) + + result = await crawler.arun(url, config=CrawlerRunConfig( + link_preview_config=link_config, + score_links=True + )) + # Links ranked by relevance and quality + ``` + +- **🎣 Async URL Seeder**: Discover thousands of URLs in seconds: + ```python + seeder = AsyncUrlSeeder(SeedingConfig( + source="sitemap+cc", + pattern="*/blog/*", + query="python tutorials", + score_threshold=0.4 + )) + + urls = await seeder.discover("https://example.com") + ``` + +- **⚡ Performance Boost**: Up to 3x faster with optimized resource handling and memory efficiency + +Read the full details in our [0.7.0 Release Notes](https://docs.crawl4ai.com/blog/release-v0.7.0) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md). + +
+ +## Version Numbering in Crawl4AI + +Crawl4AI follows standard Python version numbering conventions (PEP 440) to help users understand the stability and features of each release. + +
+📈 Version Numbers Explained + +Our version numbers follow this pattern: `MAJOR.MINOR.PATCH` (e.g., 0.4.3) + +#### Pre-release Versions +We use different suffixes to indicate development stages: + +- `dev` (0.4.3dev1): Development versions, unstable +- `a` (0.4.3a1): Alpha releases, experimental features +- `b` (0.4.3b1): Beta releases, feature complete but needs testing +- `rc` (0.4.3): Release candidates, potential final version + +#### Installation +- Regular installation (stable version): + ```bash + pip install -U crawl4ai + ``` + +- Install pre-release versions: + ```bash + pip install crawl4ai --pre + ``` + +- Install specific version: + ```bash + pip install crawl4ai==0.4.3b1 + ``` + +#### Why Pre-releases? +We use pre-releases to: +- Test new features in real-world scenarios +- Gather feedback before final releases +- Ensure stability for production users +- Allow early adopters to try new features + +For production environments, we recommend using the stable version. For testing new features, you can opt-in to pre-releases using the `--pre` flag. + +
+ +## 📖 Documentation & Roadmap + +> 🚨 **Documentation Update Alert**: We're undertaking a major documentation overhaul next week to reflect recent updates and improvements. Stay tuned for a more comprehensive and up-to-date guide! + +For current documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://docs.crawl4ai.com/). + +To check our development plans and upcoming features, visit our [Roadmap](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md). + +
+📈 Development TODOs + +- [x] 0. Graph Crawler: Smart website traversal using graph search algorithms for comprehensive nested page extraction +- [x] 1. Question-Based Crawler: Natural language driven web discovery and content extraction +- [x] 2. Knowledge-Optimal Crawler: Smart crawling that maximizes knowledge while minimizing data extraction +- [x] 3. Agentic Crawler: Autonomous system for complex multi-step crawling operations +- [x] 4. Automated Schema Generator: Convert natural language to extraction schemas +- [x] 5. Domain-Specific Scrapers: Pre-configured extractors for common platforms (academic, e-commerce) +- [x] 6. Web Embedding Index: Semantic search infrastructure for crawled content +- [x] 7. Interactive Playground: Web UI for testing, comparing strategies with AI assistance +- [x] 8. Performance Monitor: Real-time insights into crawler operations +- [ ] 9. Cloud Integration: One-click deployment solutions across cloud providers +- [x] 10. Sponsorship Program: Structured support system with tiered benefits +- [ ] 11. Educational Content: "How to Crawl" video series and interactive tutorials + +
+ +## 🤝 Contributing + +We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information. + +I'll help modify the license section with badges. For the halftone effect, here's a version with it: + +Here's the updated license section: + +## 📄 License & Attribution + +This project is licensed under the Apache License 2.0, attribution is recommended via the badges below. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details. + +### Attribution Requirements +When using Crawl4AI, you must include one of the following attribution methods: + +
+📈 1. Badge Attribution (Recommended) +Add one of these badges to your README, documentation, or website: + +| Theme | Badge | +|-------|-------| +| **Disco Theme (Animated)** | Powered by Crawl4AI | +| **Night Theme (Dark with Neon)** | Powered by Crawl4AI | +| **Dark Theme (Classic)** | Powered by Crawl4AI | +| **Light Theme (Classic)** | Powered by Crawl4AI | + + +HTML code for adding the badges: +```html + + + Powered by Crawl4AI + + + + + Powered by Crawl4AI + + + + + Powered by Crawl4AI + + + + + Powered by Crawl4AI + + + + + Powered by Crawl4AI + +``` + +
+ +
+📖 2. Text Attribution +Add this line to your documentation: +``` +This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction. +``` +
+ +## 📚 Citation + +If you use Crawl4AI in your research or project, please cite: + +```bibtex +@software{crawl4ai2024, + author = {UncleCode}, + title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper}, + year = {2024}, + publisher = {GitHub}, + journal = {GitHub Repository}, + howpublished = {\url{https://github.com/unclecode/crawl4ai}}, + commit = {Please use the commit hash you're working with} +} +``` + +Text citation format: +``` +UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software]. +GitHub. https://github.com/unclecode/crawl4ai +``` + +## 📧 Contact + +For questions, suggestions, or feedback, feel free to reach out: + +- GitHub: [unclecode](https://github.com/unclecode) +- Twitter: [@unclecode](https://twitter.com/unclecode) +- Website: [crawl4ai.com](https://crawl4ai.com) + +Happy Crawling! 🕸️🚀 + +## 🗾 Mission + +Our mission is to unlock the value of personal and enterprise data by transforming digital footprints into structured, tradeable assets. Crawl4AI empowers individuals and organizations with open-source tools to extract and structure data, fostering a shared data economy. + +We envision a future where AI is powered by real human knowledge, ensuring data creators directly benefit from their contributions. By democratizing data and enabling ethical sharing, we are laying the foundation for authentic AI advancement. + +
+🔑 Key Opportunities + +- **Data Capitalization**: Transform digital footprints into measurable, valuable assets. +- **Authentic AI Data**: Provide AI systems with real human insights. +- **Shared Economy**: Create a fair data marketplace that benefits data creators. + +
+ +
+🚀 Development Pathway + +1. **Open-Source Tools**: Community-driven platforms for transparent data extraction. +2. **Digital Asset Structuring**: Tools to organize and value digital knowledge. +3. **Ethical Data Marketplace**: A secure, fair platform for exchanging structured data. + +For more details, see our [full mission statement](./MISSION.md). +
+ +## 🌟 Current Sponsors + +### 🤝 Strategic Partners + +These companies provide core infrastructure and technology that power Crawl4AI’s capabilities — from web access and proxy networks to AI tooling and data pipelines. + +| Company | About | +|------|------| +| Massive | Massive is a web access API backed by millions of volunteer devices in 195+ countries. AI agents, models, and data pipelines use it to reach any site on the internet, reliably, in real time, and at scale. | + +### 🏢 Enterprise Sponsors + +Our enterprise sponsors support Crawl4AI and help scale it to power production-grade data pipelines. + +| Company | About | Sponsorship Tier | +|------|------|----------------------------| +| DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | +| Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | +| Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | + +--- + +### 💼 Become a Strategic Partner or Sponsor + +Interested in partnering with Crawl4AI? + +Whether you’re a proxy provider, AI infrastructure company, cloud platform, or an organization looking to support the Crawl4AI ecosystem, we’d love to hear from you. + +📩 Contact: hello@crawl4ai.com + + + +### 🧑‍🤝 Individual Sponsors + +A heartfelt thanks to our individual supporters! Every contribution helps us keep our opensource mission alive and thriving! + +

+ + + + + + + + +

+ +> Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/unclecode) + +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date) diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..e4f3c96 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`unclecode/crawl4ai` +- 原始仓库：https://github.com/unclecode/crawl4ai +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..0fd784c --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,503 @@ +# Crawl4AI Strategic Roadmap + +```mermaid +%%{init: {'themeVariables': { 'fontSize': '14px'}}}%% +graph TD + subgraph A1[Advanced Crawling Systems 🔧] + A["` + • Graph Crawler ✓ + • Question-Based Crawler + • Knowledge-Optimal Crawler + • Agentic Crawler + `"] + end + + subgraph A2[Specialized Features 🛠️] + B["` + • Automated Schema Generator + • Domain-Specific Scrapers + • + • + `"] + end + + subgraph A3[Development Tools 🔨] + C["` + • Interactive Playground + • Performance Monitor + • Cloud Integration + • + `"] + end + + subgraph A4[Community & Growth 🌱] + D["` + • Sponsorship Program + • Educational Content + • + • + `"] + end + + classDef default fill:#f9f9f9,stroke:#333,stroke-width:2px + classDef section fill:#f0f0f0,stroke:#333,stroke-width:4px,rx:10 + class A1,A2,A3,A4 section + + %% Layout hints + A1 --> A2[" "] + A3 --> A4[" "] + linkStyle 0,1 stroke:none +``` + +Crawl4AI is evolving to provide more intelligent, efficient, and versatile web crawling capabilities. This roadmap outlines the key developments and features planned for the project, organized into strategic sections that build upon our current foundation. + +## 1. Advanced Crawling Systems 🔧 + +This section introduces three powerful crawling systems that extend Crawl4AI's capabilities from basic web crawling to intelligent, purpose-driven data extraction. + +### 1.1 Question-Based Crawler +The Question-Based Crawler enhances our core engine by enabling automatic discovery and extraction of relevant web content based on natural language questions. + +Key Features: +- SerpiAPI integration for intelligent web search +- Relevancy scoring for search results +- Automatic URL discovery and prioritization +- Cross-source validation + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.discovery import QuestionBasedDiscovery + +async with AsyncWebCrawler() as crawler: + discovery = QuestionBasedDiscovery(crawler) + results = await discovery.arun( + question="What are the system requirements for major cloud providers' GPU instances?", + max_urls=5, + relevance_threshold=0.7 + ) + + for result in results: + print(f"Source: {result.url} (Relevance: {result.relevance_score})") + print(f"Content: {result.markdown}\n") +``` + +### 1.2 Knowledge-Optimal Crawler +An intelligent crawling system that solves the optimization problem of minimizing data extraction while maximizing knowledge acquisition for specific objectives. + +Key Features: +- Smart content prioritization +- Minimal data extraction for maximum knowledge +- Probabilistic relevance assessment +- Objective-driven crawling paths + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.optimization import KnowledgeOptimizer + +async with AsyncWebCrawler() as crawler: + optimizer = KnowledgeOptimizer( + objective="Understand GPU instance pricing and limitations across cloud providers", + required_knowledge=[ + "pricing structure", + "GPU specifications", + "usage limits", + "availability zones" + ], + confidence_threshold=0.85 + ) + + result = await crawler.arun( + urls=[ + "https://aws.amazon.com/ec2/pricing/", + "https://cloud.google.com/gpu", + "https://azure.microsoft.com/pricing/" + ], + optimizer=optimizer, + optimization_mode="minimal_extraction" + ) + + print(f"Knowledge Coverage: {result.knowledge_coverage}") + print(f"Data Efficiency: {result.efficiency_ratio}") + print(f"Extracted Content: {result.optimal_content}") +``` + +### 1.3 Agentic Crawler +An autonomous system capable of understanding complex goals and automatically planning and executing multi-step crawling operations. + +Key Features: +- Autonomous goal interpretation +- Dynamic step planning +- Interactive navigation capabilities +- Visual recognition and interaction +- Automatic error recovery + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.agents import CrawlerAgent + +async with AsyncWebCrawler() as crawler: + agent = CrawlerAgent(crawler) + + # Automatic planning and execution + result = await agent.arun( + goal="Find research papers about quantum computing published in 2023 with more than 50 citations", + auto_retry=True + ) + print("Generated Plan:", result.executed_steps) + print("Extracted Data:", result.data) + + # Using custom steps with automatic execution + result = await agent.arun( + goal="Extract conference deadlines from ML conferences", + custom_plan=[ + "Navigate to conference page", + "Find important dates section", + "Extract submission deadlines", + "Verify dates are for 2024" + ] + ) + + # Monitoring execution + print("Step Completion:", result.step_status) + print("Execution Time:", result.execution_time) + print("Success Rate:", result.success_rate) +``` + +# Section 2: Specialized Features 🛠️ + +This section introduces specialized tools and features that enhance Crawl4AI's capabilities for specific use cases and data extraction needs. + +### 2.1 Automated Schema Generator +A system that automatically generates JsonCssExtractionStrategy schemas from natural language descriptions, making structured data extraction accessible to all users. + +Key Features: +- Natural language schema generation +- Automatic pattern detection +- Predefined schema templates +- Chrome extension for visual schema building + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.schema import SchemaGenerator + +# Generate schema from natural language description +generator = SchemaGenerator() +schema = await generator.generate( + url="https://news-website.com", + description="For each news article on the page, I need the headline, publication date, and main image" +) + +# Use generated schema with crawler +async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://news-website.com", + extraction_strategy=schema + ) + +# Example of generated schema: +""" +{ + "name": "News Article Extractor", + "baseSelector": "article.news-item", + "fields": [ + { + "name": "headline", + "selector": "h2.article-title", + "type": "text" + }, + { + "name": "date", + "selector": "span.publish-date", + "type": "text" + }, + { + "name": "image", + "selector": "img.article-image", + "type": "attribute", + "attribute": "src" + } + ] +} +""" +``` + +### 2.2 Domain Specific Scrapers +Specialized extraction strategies optimized for common website types and platforms, providing consistent and reliable data extraction without additional configuration. + +Key Features: +- Pre-configured extractors for popular platforms +- Academic site specialization (arXiv, NCBI) +- E-commerce standardization +- Documentation site handling + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.extractors import AcademicExtractor, EcommerceExtractor + +async with AsyncWebCrawler() as crawler: + # Academic paper extraction + papers = await crawler.arun( + url="https://arxiv.org/list/cs.AI/recent", + extractor="academic", # Built-in extractor type + site_type="arxiv", # Specific site optimization + extract_fields=[ + "title", + "authors", + "abstract", + "citations" + ] + ) + + # E-commerce product data + products = await crawler.arun( + url="https://store.example.com/products", + extractor="ecommerce", + extract_fields=[ + "name", + "price", + "availability", + "reviews" + ] + ) +``` + +### 2.3 Web Embedding Index +Creates and maintains a semantic search infrastructure for crawled content, enabling efficient retrieval and querying of web content through vector embeddings. + +Key Features: +- Automatic embedding generation +- Intelligent content chunking +- Efficient vector storage and indexing +- Semantic search capabilities + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.indexing import WebIndex + +# Initialize and build index +index = WebIndex(model="efficient-mini") + +async with AsyncWebCrawler() as crawler: + # Crawl and index content + await index.build( + urls=["https://docs.example.com"], + crawler=crawler, + options={ + "chunk_method": "semantic", + "update_policy": "incremental", + "embedding_batch_size": 100 + } + ) + + # Search through indexed content + results = await index.search( + query="How to implement OAuth authentication?", + filters={ + "content_type": "technical", + "recency": "6months" + }, + top_k=5 + ) + + # Get similar content + similar = await index.find_similar( + url="https://docs.example.com/auth/oauth", + threshold=0.85 + ) +``` + +Each of these specialized features builds upon Crawl4AI's core functionality while providing targeted solutions for specific use cases. They can be used independently or combined for more complex data extraction and processing needs. + +# Section 3: Development Tools 🔧 + +This section covers tools designed to enhance the development experience, monitoring, and deployment of Crawl4AI applications. + +### 3.1 Crawl4AI Playground 🎮 + +The Crawl4AI Playground is an interactive web-based development environment that simplifies web scraping experimentation, development, and deployment. With its intuitive interface and AI-powered assistance, users can quickly prototype, test, and deploy web scraping solutions. + +#### Key Features 🌟 + +##### Visual Strategy Builder +- Interactive point-and-click interface for building extraction strategies +- Real-time preview of selected elements +- Side-by-side comparison of different extraction approaches +- Visual validation of CSS selectors and XPath queries + +##### AI Assistant Integration +- Strategy recommendations based on target website analysis +- Parameter optimization suggestions +- Best practices guidance for specific use cases +- Automated error detection and resolution +- Performance optimization tips + +##### Real-Time Testing & Validation +- Live preview of extraction results +- Side-by-side comparison of multiple strategies +- Performance metrics visualization +- Automatic validation of extracted data +- Error detection and debugging tools + +##### Project Management +- Save and organize multiple scraping projects +- Version control for configurations +- Export/import project settings +- Share configurations with team members +- Project templates for common use cases + +##### Deployment Pipeline +- One-click deployment to various environments +- Docker container generation +- Cloud deployment templates (AWS, GCP, Azure) +- Scaling configuration management +- Monitoring setup automation + + +### 3.2 Performance Monitoring System +A comprehensive monitoring solution providing real-time insights into crawler operations, resource usage, and system health through both CLI and GUI interfaces. + +Key Features: +- Real-time resource tracking +- Active crawl monitoring +- Performance statistics +- Customizable alerting system + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.monitor import CrawlMonitor + +# Initialize monitoring +monitor = CrawlMonitor() + +# Start monitoring with CLI interface +await monitor.start( + mode="cli", # or "gui" + refresh_rate="1s", + metrics={ + "resources": ["cpu", "memory", "network"], + "crawls": ["active", "queued", "completed"], + "performance": ["success_rate", "response_times"] + } +) + +# Example CLI output: +""" +Crawl4AI Monitor (Live) - Press Q to exit +──────────────────────────────────────── +System Usage: + ├─ CPU: ███████░░░ 70% + └─ Memory: ████░░░░░ 2.1GB/8GB + +Active Crawls: +ID URL Status Progress +001 docs.example.com 🟢 Active 75% +002 api.service.com 🟡 Queue - + +Metrics (Last 5min): + ├─ Success Rate: 98% + ├─ Avg Response: 0.6s + └─ Pages/sec: 8.5 +""" +``` + +### 3.3 Cloud Integration +Streamlined deployment tools for setting up Crawl4AI in various cloud environments, with support for scaling and monitoring. + +Key Features: +- One-click deployment solutions +- Auto-scaling configuration +- Load balancing setup +- Cloud-specific optimizations +- Monitoring integration + +```python +from crawl4ai import AsyncWebCrawler +from crawl4ai.deploy import CloudDeployer + +# Initialize deployer +deployer = CloudDeployer() + +# Deploy crawler service +deployment = await deployer.deploy( + service_name="crawler-cluster", + platform="aws", # or "gcp", "azure" + config={ + "instance_type": "compute-optimized", + "auto_scaling": { + "min_instances": 2, + "max_instances": 10, + "scale_based_on": "cpu_usage" + }, + "region": "us-east-1", + "monitoring": True + } +) + +# Get deployment status and endpoints +print(f"Service Status: {deployment.status}") +print(f"API Endpoint: {deployment.endpoint}") +print(f"Monitor URL: {deployment.monitor_url}") +``` + +These development tools work together to provide a comprehensive environment for developing, testing, monitoring, and deploying Crawl4AI applications. The Playground helps users experiment and generate optimal configurations, the Performance Monitor ensures smooth operation, and the Cloud Integration tools simplify deployment and scaling. + +# Section 4: Community & Growth 🌱 + +This section outlines initiatives designed to build and support the Crawl4AI community, provide educational resources, and ensure sustainable project growth. + +### 4.1 Sponsorship Program +A structured program to support ongoing development and maintenance of Crawl4AI while providing valuable benefits to sponsors. + +Key Features: +- Multiple sponsorship tiers +- Sponsor recognition system +- Priority support for sponsors +- Early access to new features +- Custom feature development opportunities + +Program Structure (not yet finalized): +``` +Sponsorship Tiers: + +🥉 Bronze Supporter +- GitHub Sponsor badge +- Priority issue response +- Community Discord role + +🥈 Silver Supporter +- All Bronze benefits +- Technical support channel +- Vote on roadmap priorities +- Early access to beta features + +🥇 Gold Supporter +- All Silver benefits +- Custom feature requests +- Direct developer access +- Private support sessions + +💎 Diamond Partner +- All Gold benefits +- Custom development +- On-demand consulting +- Integration support +``` + +### 4.2 "How to Crawl" Video Series +A comprehensive educational resource teaching users how to effectively use Crawl4AI for various web scraping and data extraction scenarios. + +Key Features: +- Step-by-step tutorials +- Real-world use cases +- Best practices +- Integration guides +- Advanced feature deep-dives + +These community initiatives are designed to: +- Provide comprehensive learning resources +- Foster a supportive user community +- Ensure sustainable project development +- Share knowledge and best practices +- Create opportunities for collaboration + +The combination of structured support through sponsorship, educational content through video series, and interactive learning through the playground creates a robust ecosystem for both new and experienced users of Crawl4AI. diff --git a/SECURITY-CREDITS.md b/SECURITY-CREDITS.md new file mode 100644 index 0000000..e2ae2bc --- /dev/null +++ b/SECURITY-CREDITS.md @@ -0,0 +1,20 @@ +# Security Credits + +We thank the following security researchers for their responsible disclosure: + +| Researcher | Contact | Vulnerability | Date Reported | +|---|---|---|---| +| Song Binglin (q1uf3ng) | q1uf3ng@proton.me | AST sandbox escape via gi_frame.f_back chain (CVSS 9.8) | 2026-03-29 | +| Jeongbean Jeon | wjswjdqls7@gmail.com | File write, SSRF, monitor auth bypass, stored XSS | 2026-04-13 | +| wulonchia | wulonchia@gmail.com | File write via output_path (independent report) | 2026-04-13 | +| by111 (August829) | GitHub: [August829](https://github.com/August829) | Hardcoded JWT secret, eval in /config/dump, /execute_js, hook sandbox escape | 2026-04-14 | +| secsys_codex | secsys_codex@163.com | SSRF via /md, /crawl, /llm endpoints (URL destination validation) | 2026-04-18 | +| Velayutham Selvaraj | [LinkedIn](https://www.linkedin.com/in/velayuthamselvaraj) | SSRF via missing host validation in validate_url_scheme (independent report) | 2026-05-06 | +| IcySun & Yashon | icysun@qq.com, liyaoyin@qq.com | SSRF, file write via output_path, missing auth by default, hook sandbox bypass via asyncio (independent report) | 2026-05-15 | +| Geo ([geo-chen](https://github.com/geo-chen)) | cve@sageby.com | LLM API key exfiltration via unvalidated base_url (0.8.8) | 2026-06-02 | +| Geo ([geo-chen](https://github.com/geo-chen)) | cve@sageby.com | SSRF via proxy_config.server bypassing the SSRF check (0.8.9) | 2026-06-04 | +| Y4tacker | y4tacker@gmail.com | Download path traversal -> file write; Chromium launch-arg injection via extra_args (0.9.0) | 2026-06-18 | +| KOH Jun Sheng ([seankohjs](https://github.com/seankohjs)) | jskoh.2023@scis.smu.edu.sg | SSRF on the streaming crawl path /crawl/stream (0.9.0) | 2026-06-18 | +| UDU_RisePho | GitHub: [hoanggxyuuki](https://github.com/hoanggxyuuki) | Chromium launch-flag RCE class via extra_args (0.9.0) | 2026-06-18 | +| Y4tacker | GitHub: [Y4tacker](https://github.com/Y4tacker) | Hook system exec() sandbox escape (MRO chain RCE), Chromium launch-arg injection (--utility-cmd-prefix RCE), HTTP crawler path traversal arbitrary file write | 2026-07-09 | +| Rafael | GitHub: [rafaelfiguereod-stack](https://github.com/rafaelfiguereod-stack) | Reported SSRF, LLM key exfiltration, and auth gaps (already fixed / not exploitable in current code) | 2026-07-09 | diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3245b9f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,134 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 0.8.x | :white_check_mark: | +| 0.7.x | :x: (upgrade recommended) | +| < 0.7 | :x: | + +## Reporting a Vulnerability + +We take security vulnerabilities seriously. If you discover a security issue, please report it responsibly. + +### How to Report + +**DO NOT** open a public GitHub issue for security vulnerabilities. + +Instead, please report via one of these methods: + +1. **GitHub Security Advisories (Preferred)** + - Go to [Security Advisories](https://github.com/unclecode/crawl4ai/security/advisories) + - Click "New draft security advisory" + - Fill in the details + +2. **Email** + - Send details to: unclecode@crawl4ai.com (CC: nasrin@crawl4ai.com and aravind@crawl4ai.com) + - Use subject: `[SECURITY] Brief description` + - Include: + - Description of the vulnerability + - Steps to reproduce + - Potential impact + - Any suggested fixes + +### What to Expect + +- **Acknowledgment**: Within 48 hours +- **Initial Assessment**: Within 7 days +- **Resolution Timeline**: Depends on severity + - Critical: 24-72 hours + - High: 7 days + - Medium: 30 days + - Low: 90 days + +### Disclosure Policy + +- We follow responsible disclosure practices +- We will coordinate with you on disclosure timing +- Credit will be given to reporters (unless anonymity is requested) +- We may request CVE assignment for significant vulnerabilities + +## Security Best Practices for Users + +### Docker API Deployment + +If you're running the Crawl4AI Docker API in production: + +1. **Enable Authentication** + ```yaml + # config.yml + security: + enabled: true + jwt_enabled: true + ``` + ```bash + # Set a strong secret key + export SECRET_KEY="your-secure-random-key-here" + ``` + +2. **Hooks are Disabled by Default** (v0.8.0+) + - Only enable if you trust all API users + - Set `CRAWL4AI_HOOKS_ENABLED=true` only when necessary + +3. **Network Security** + - Run behind a reverse proxy (nginx, traefik) + - Use HTTPS in production + - Restrict access to trusted IPs if possible + +4. **Container Security** + - Run as non-root user (default in our container) + - Use read-only filesystem where possible + - Limit container resources + +### Library Usage + +When using Crawl4AI as a Python library: + +1. **Validate URLs** before crawling untrusted input +2. **Sanitize extracted content** before using in other systems +3. **Be cautious with hooks** - they execute arbitrary code + +## Known Security Issues + +### Fixed in v0.8.0 + +| ID | Severity | Description | Fix | +|----|----------|-------------|-----| +| CVE-pending-1 | CRITICAL | RCE via hooks `__import__` | Removed from allowed builtins | +| CVE-pending-2 | HIGH | LFI via `file://` URLs | URL scheme validation added | + +### Fixed in v0.8.1 + +| ID | Severity | Description | Fix | +|----|----------|-------------|-----| +| CVE-pending-3 | CRITICAL | RCE via deserialization + `eval()` in `/crawl` endpoint | Allowlisted deserializable types; AST-validated computed field expressions | + +See [Security Advisory](https://github.com/unclecode/crawl4ai/security/advisories) for details. + +## Security Features + +### v0.8.1+ + +- **Deserialization Allowlist**: Only known-safe types can be instantiated via API config +- **Safe Expression Evaluation**: Computed fields use AST validation (no `__import__`, no dunder access) + +### v0.8.0+ + +- **URL Scheme Validation**: Blocks `file://`, `javascript:`, `data:` URLs on API +- **Hooks Disabled by Default**: Opt-in via `CRAWL4AI_HOOKS_ENABLED=true` +- **Restricted Hook Builtins**: No `__import__`, `eval`, `exec`, `open` +- **JWT Authentication**: Optional but recommended for production +- **Rate Limiting**: Configurable request limits +- **Security Headers**: X-Frame-Options, CSP, HSTS when enabled + +## Acknowledgments + +We thank the following security researchers for responsibly disclosing vulnerabilities: + +- **Alec M** — RCE via deserialization in `/crawl` endpoint (January 2026) +- **[Neo by ProjectDiscovery](https://projectdiscovery.io/blog/introducing-neo)** — RCE and LFI vulnerabilities (December 2025) + +--- + +*Last updated: February 2026* diff --git a/SPONSORS.md b/SPONSORS.md new file mode 100644 index 0000000..6503cdb --- /dev/null +++ b/SPONSORS.md @@ -0,0 +1,65 @@ +# 💖 Sponsors & Supporters + +Thank you to everyone supporting Crawl4AI! Your sponsorship helps keep this project open-source and actively maintained. + +## 👑 Founding Sponsors +*The first 50 sponsors who believed in our vision - permanently recognized* + + +🎉 **Become a Founding Sponsor!** Only [X/50] spots remaining! [Join now →](https://github.com/sponsors/unclecode) + +--- + +## 🏢 Data Infrastructure Partners ($2000/month) +*These organizations are building their data sovereignty with Crawl4AI at the core* + + +*Be the first Data Infrastructure Partner! [Join us →](https://github.com/sponsors/unclecode)* + +--- + +## 💼 Growing Teams ($500/month) +*Teams scaling their data extraction with Crawl4AI* + + +*Your team could be here! [Become a sponsor →](https://github.com/sponsors/unclecode)* + +--- + +## 🚀 Builders ($50/month) +*Developers and entrepreneurs building with Crawl4AI* + + +*Join the builders! [Start sponsoring →](https://github.com/sponsors/unclecode)* + +--- + +## 🌱 Believers ($5/month) +*The community supporting data democratization* + + +*Thank you to all our community believers!* + +--- + +## 🤝 Want to Sponsor? + +Crawl4AI is the #1 trending open-source web crawler. We're building the future of data extraction - where organizations own their data pipelines instead of relying on rate-limited APIs. + +### Available Sponsorship Tiers: +- **🌱 Believer** ($5/mo) - Support the movement +- **🚀 Builder** ($50/mo) - Priority support & early access +- **💼 Growing Team** ($500/mo) - Bi-weekly syncs & optimization +- **🏢 Data Infrastructure Partner** ($2000/mo) - Full partnership & dedicated support + +[View all tiers and benefits →](https://github.com/sponsors/unclecode) + +### Enterprise & Custom Partnerships + +Building data extraction at scale? Need dedicated support or infrastructure? Let's talk about a custom partnership. + +📧 Contact: [hello@crawl4ai.com](mailto:hello@crawl4ai.com) | 📅 [Schedule a call](https://calendar.app.google/rEpvi2UBgUQjWHfJ9) + +--- + +*This list is updated regularly. Sponsors at $50+ tiers can submit their logos via [hello@crawl4ai.com](mailto:hello@crawl4ai.com)* \ No newline at end of file diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 0000000..a2cd8cd --- /dev/null +++ b/cliff.toml @@ -0,0 +1,24 @@ +[changelog] +# Template format +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n +""" + +# Organize commits by type +[git] +conventional_commits = true +filter_unconventional = true +commit_parsers = [ + { message = "^feat", group = "Added"}, + { message = "^fix", group = "Fixed"}, + { message = "^doc", group = "Documentation"}, + { message = "^perf", group = "Performance"}, + { message = "^refactor", group = "Changed"}, + { message = "^style", group = "Changed"}, + { message = "^test", group = "Testing"}, + { message = "^chore\\(release\\): prepare for", skip = true}, + { message = "^chore", group = "Miscellaneous Tasks"}, +] \ No newline at end of file diff --git a/crawl4ai/__init__.py b/crawl4ai/__init__.py new file mode 100644 index 0000000..0d1071a --- /dev/null +++ b/crawl4ai/__init__.py @@ -0,0 +1,236 @@ +# __init__.py +import warnings + +from .async_webcrawler import AsyncWebCrawler, CacheMode +# MODIFIED: Add SeedingConfig and VirtualScrollConfig here +from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig, LLMConfig, ProxyConfig, GeolocationConfig, SeedingConfig, VirtualScrollConfig, LinkPreviewConfig, MatchMode, DomainMapperConfig + +from .content_scraping_strategy import ( + ContentScrapingStrategy, + LXMLWebScrapingStrategy, + WebScrapingStrategy, # Backward compatibility alias +) +from .processors.pdf import PDFContentScrapingStrategy +from .async_logger import ( + AsyncLoggerBase, + AsyncLogger, +) +from .proxy_strategy import ( + ProxyRotationStrategy, + RoundRobinProxyStrategy, +) +from .extraction_strategy import ( + ExtractionStrategy, + LLMExtractionStrategy, + CosineStrategy, + JsonCssExtractionStrategy, + JsonXPathExtractionStrategy, + JsonLxmlExtractionStrategy, + RegexExtractionStrategy +) +from .chunking_strategy import ChunkingStrategy, RegexChunking +from .markdown_generation_strategy import DefaultMarkdownGenerator +from .table_extraction import ( + TableExtractionStrategy, + DefaultTableExtraction, + NoTableExtraction, + LLMTableExtraction, +) +from .content_filter_strategy import ( + PruningContentFilter, + BM25ContentFilter, + LLMContentFilter, + RelevantContentFilter, +) +from .models import CrawlResult, MarkdownGenerationResult, DisplayMode +from .components.crawler_monitor import CrawlerMonitor +from .link_preview import LinkPreview +from .async_dispatcher import ( + MemoryAdaptiveDispatcher, + SemaphoreDispatcher, + RateLimiter, + BaseDispatcher, +) +from .docker_client import Crawl4aiDockerClient +from .hub import CrawlerHub +from .browser_profiler import BrowserProfiler +from .deep_crawling import ( + DeepCrawlStrategy, + BFSDeepCrawlStrategy, + FilterChain, + URLPatternFilter, + DomainFilter, + ContentTypeFilter, + URLFilter, + FilterStats, + SEOFilter, + KeywordRelevanceScorer, + URLScorer, + CompositeScorer, + DomainAuthorityScorer, + FreshnessScorer, + PathDepthScorer, + BestFirstCrawlingStrategy, + DFSDeepCrawlStrategy, + DeepCrawlDecorator, + ContentRelevanceFilter, + ContentTypeScorer, +) +# NEW: Import AsyncUrlSeeder +from .async_url_seeder import AsyncUrlSeeder +from .domain_mapper import DomainMapper +# Adaptive Crawler +from .adaptive_crawler import ( + AdaptiveCrawler, + AdaptiveConfig, + CrawlState, + CrawlStrategy, + StatisticalStrategy +) + +# C4A Script Language Support +from .script import ( + compile as c4a_compile, + validate as c4a_validate, + compile_file as c4a_compile_file, + CompilationResult, + ValidationResult, + ErrorDetail +) + +# Browser Adapters +from .browser_adapter import ( + BrowserAdapter, + PlaywrightAdapter, + UndetectedAdapter +) + +from .utils import ( + start_colab_display_server, + setup_colab_environment, + hooks_to_string +) + +__all__ = [ + "AsyncLoggerBase", + "AsyncLogger", + "AsyncWebCrawler", + "BrowserProfiler", + "LLMConfig", + "GeolocationConfig", + # NEW: Add SeedingConfig and VirtualScrollConfig + "SeedingConfig", + "VirtualScrollConfig", + # NEW: Add AsyncUrlSeeder + "AsyncUrlSeeder", + # DomainMapper + "DomainMapper", + "DomainMapperConfig", + # Adaptive Crawler + "AdaptiveCrawler", + "AdaptiveConfig", + "CrawlState", + "CrawlStrategy", + "StatisticalStrategy", + "DeepCrawlStrategy", + "BFSDeepCrawlStrategy", + "BestFirstCrawlingStrategy", + "DFSDeepCrawlStrategy", + "FilterChain", + "URLPatternFilter", + "ContentTypeFilter", + "DomainFilter", + "FilterStats", + "URLFilter", + "SEOFilter", + "KeywordRelevanceScorer", + "URLScorer", + "CompositeScorer", + "DomainAuthorityScorer", + "FreshnessScorer", + "PathDepthScorer", + "DeepCrawlDecorator", + "CrawlResult", + "CrawlerHub", + "CacheMode", + "MatchMode", + "ContentScrapingStrategy", + "WebScrapingStrategy", + "LXMLWebScrapingStrategy", + "BrowserConfig", + "CrawlerRunConfig", + "HTTPCrawlerConfig", + "ExtractionStrategy", + "LLMExtractionStrategy", + "CosineStrategy", + "JsonCssExtractionStrategy", + "JsonXPathExtractionStrategy", + "JsonLxmlExtractionStrategy", + "RegexExtractionStrategy", + "ChunkingStrategy", + "RegexChunking", + "DefaultMarkdownGenerator", + "TableExtractionStrategy", + "DefaultTableExtraction", + "NoTableExtraction", + "LLMTableExtraction", + "RelevantContentFilter", + "PruningContentFilter", + "BM25ContentFilter", + "LLMContentFilter", + "BaseDispatcher", + "MemoryAdaptiveDispatcher", + "SemaphoreDispatcher", + "RateLimiter", + "CrawlerMonitor", + "LinkPreview", + "DisplayMode", + "MarkdownGenerationResult", + "Crawl4aiDockerClient", + "ProxyRotationStrategy", + "RoundRobinProxyStrategy", + "ProxyConfig", + "start_colab_display_server", + "setup_colab_environment", + "hooks_to_string", + # C4A Script additions + "c4a_compile", + "c4a_validate", + "c4a_compile_file", + "CompilationResult", + "ValidationResult", + "ErrorDetail", + # Browser Adapters + "BrowserAdapter", + "PlaywrightAdapter", + "UndetectedAdapter", + "LinkPreviewConfig" +] + + +# def is_sync_version_installed(): +# try: +# import selenium # noqa + +# return True +# except ImportError: +# return False + + +# if is_sync_version_installed(): +# try: +# from .web_crawler import WebCrawler + +# __all__.append("WebCrawler") +# except ImportError: +# print( +# "Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies." +# ) +# else: +# WebCrawler = None +# # import warnings +# # print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.") + +# Disable all Pydantic warnings +warnings.filterwarnings("ignore", module="pydantic") +# pydantic_warnings.filter_warnings() \ No newline at end of file diff --git a/crawl4ai/__version__.py b/crawl4ai/__version__.py new file mode 100644 index 0000000..6dac946 --- /dev/null +++ b/crawl4ai/__version__.py @@ -0,0 +1,8 @@ +# crawl4ai/__version__.py + +# This is the version that will be used for stable releases +__version__ = "0.9.1" + +# For nightly builds, this gets set during build process +__nightly_version__ = None + diff --git a/crawl4ai/adaptive_crawler.py b/crawl4ai/adaptive_crawler.py new file mode 100644 index 0000000..6aa1d3c --- /dev/null +++ b/crawl4ai/adaptive_crawler.py @@ -0,0 +1,1923 @@ +""" +Adaptive Web Crawler for Crawl4AI + +This module implements adaptive information foraging for efficient web crawling. +It determines when sufficient information has been gathered to answer a query, +avoiding unnecessary crawls while ensuring comprehensive coverage. +""" + +from abc import ABC, abstractmethod +from typing import Dict, List, Optional, Set, Tuple, Any, Union +from dataclasses import dataclass, field +import asyncio +import pickle +import os +import json +import math +from collections import defaultdict, Counter +import re +from pathlib import Path + +from crawl4ai.async_webcrawler import AsyncWebCrawler +from crawl4ai.async_configs import CrawlerRunConfig, LinkPreviewConfig, LLMConfig +from crawl4ai.models import Link, CrawlResult +import numpy as np + +@dataclass +class CrawlState: + """Tracks the current state of adaptive crawling""" + crawled_urls: Set[str] = field(default_factory=set) + knowledge_base: List[CrawlResult] = field(default_factory=list) + pending_links: List[Link] = field(default_factory=list) + query: str = "" + metrics: Dict[str, float] = field(default_factory=dict) + + # Statistical tracking + term_frequencies: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) + document_frequencies: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) + documents_with_terms: Dict[str, Set[int]] = field(default_factory=lambda: defaultdict(set)) + total_documents: int = 0 + + # History tracking for saturation + new_terms_history: List[int] = field(default_factory=list) + crawl_order: List[str] = field(default_factory=list) + + # Embedding-specific tracking (only if strategy is embedding) + kb_embeddings: Optional[Any] = None # Will be numpy array + query_embeddings: Optional[Any] = None # Will be numpy array + expanded_queries: List[str] = field(default_factory=list) + coverage_shape: Optional[Any] = None # Alpha shape + semantic_gaps: List[Tuple[List[float], float]] = field(default_factory=list) # Serializable + embedding_model: str = "" + + def save(self, path: Union[str, Path]): + """Save state to disk for persistence""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + # Convert CrawlResult objects to dicts for serialization + state_dict = { + 'crawled_urls': list(self.crawled_urls), + 'knowledge_base': [self._crawl_result_to_dict(cr) for cr in self.knowledge_base], + 'pending_links': [link.model_dump() for link in self.pending_links], + 'query': self.query, + 'metrics': self.metrics, + 'term_frequencies': dict(self.term_frequencies), + 'document_frequencies': dict(self.document_frequencies), + 'documents_with_terms': {k: list(v) for k, v in self.documents_with_terms.items()}, + 'total_documents': self.total_documents, + 'new_terms_history': self.new_terms_history, + 'crawl_order': self.crawl_order, + # Embedding-specific fields (convert numpy arrays to lists for JSON) + 'kb_embeddings': self.kb_embeddings.tolist() if self.kb_embeddings is not None else None, + 'query_embeddings': self.query_embeddings.tolist() if self.query_embeddings is not None else None, + 'expanded_queries': self.expanded_queries, + 'semantic_gaps': self.semantic_gaps, + 'embedding_model': self.embedding_model + } + + with open(path, 'w') as f: + json.dump(state_dict, f, indent=2) + + @classmethod + def load(cls, path: Union[str, Path]) -> 'CrawlState': + """Load state from disk""" + path = Path(path) + with open(path, 'r') as f: + state_dict = json.load(f) + + state = cls() + state.crawled_urls = set(state_dict['crawled_urls']) + state.knowledge_base = [cls._dict_to_crawl_result(d) for d in state_dict['knowledge_base']] + state.pending_links = [Link(**link_dict) for link_dict in state_dict['pending_links']] + state.query = state_dict['query'] + state.metrics = state_dict['metrics'] + state.term_frequencies = defaultdict(int, state_dict['term_frequencies']) + state.document_frequencies = defaultdict(int, state_dict['document_frequencies']) + state.documents_with_terms = defaultdict(set, {k: set(v) for k, v in state_dict['documents_with_terms'].items()}) + state.total_documents = state_dict['total_documents'] + state.new_terms_history = state_dict['new_terms_history'] + state.crawl_order = state_dict['crawl_order'] + + # Load embedding-specific fields (convert lists back to numpy arrays) + + state.kb_embeddings = np.array(state_dict['kb_embeddings']) if state_dict.get('kb_embeddings') is not None else None + state.query_embeddings = np.array(state_dict['query_embeddings']) if state_dict.get('query_embeddings') is not None else None + state.expanded_queries = state_dict.get('expanded_queries', []) + state.semantic_gaps = state_dict.get('semantic_gaps', []) + state.embedding_model = state_dict.get('embedding_model', '') + + return state + + @staticmethod + def _crawl_result_to_dict(cr: CrawlResult) -> Dict: + """Convert CrawlResult to serializable dict""" + # Extract markdown content safely + markdown_content = "" + if hasattr(cr, 'markdown') and cr.markdown: + if hasattr(cr.markdown, 'raw_markdown'): + markdown_content = cr.markdown.raw_markdown + else: + markdown_content = str(cr.markdown) + + return { + 'url': cr.url, + 'content': markdown_content, + 'links': cr.links if hasattr(cr, 'links') else {}, + 'metadata': cr.metadata if hasattr(cr, 'metadata') else {} + } + + @staticmethod + def _dict_to_crawl_result(d: Dict): + """Convert dict back to CrawlResult""" + # Create a mock object that has the minimal interface we need + class MockMarkdown: + def __init__(self, content): + self.raw_markdown = content + + class MockCrawlResult: + def __init__(self, url, content, links, metadata): + self.url = url + self.markdown = MockMarkdown(content) + self.links = links + self.metadata = metadata + + return MockCrawlResult( + url=d['url'], + content=d.get('content', ''), + links=d.get('links', {}), + metadata=d.get('metadata', {}) + ) + + +@dataclass +class AdaptiveConfig: + """Configuration for adaptive crawling""" + confidence_threshold: float = 0.7 + max_depth: int = 5 + max_pages: int = 20 + top_k_links: int = 3 + min_gain_threshold: float = 0.1 + strategy: str = "statistical" # statistical, embedding, llm + + # Advanced parameters + saturation_threshold: float = 0.8 + consistency_threshold: float = 0.7 + coverage_weight: float = 0.4 + consistency_weight: float = 0.3 + saturation_weight: float = 0.3 + + # Link scoring parameters + relevance_weight: float = 0.5 + novelty_weight: float = 0.3 + authority_weight: float = 0.2 + + # Persistence + save_state: bool = False + state_path: Optional[str] = None + + # Embedding strategy parameters + embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2" + embedding_llm_config: Optional[Union[LLMConfig, Dict]] = None # Separate config for embeddings + query_llm_config: Optional[Union[LLMConfig, Dict]] = None # Config for query expansion (chat completion) + n_query_variations: int = 10 + coverage_threshold: float = 0.85 + alpha_shape_alpha: float = 0.5 + + # Minimum confidence threshold for relevance + embedding_min_confidence_threshold: float = 0.1 # Below this, content is considered completely irrelevant + # Example: If confidence < 0.1, stop immediately as query and content are unrelated + + # Embedding confidence calculation parameters + embedding_coverage_radius: float = 0.2 # Distance threshold for "covered" query points + # Example: With radius=0.2, a query point is considered covered if ANY document + # is within cosine distance 0.2 (very similar). Smaller = stricter coverage requirement + + embedding_k_exp: float = 1.0 # Exponential decay factor for distance-to-score mapping + # Example: score = exp(-k_exp * distance). With k_exp=1, distance 0.2 → score 0.82, + # distance 0.5 → score 0.61. Higher k_exp = steeper decay = more emphasis on very close matches + + embedding_nearest_weight: float = 0.7 # Weight for nearest neighbor in hybrid scoring + embedding_top_k_weight: float = 0.3 # Weight for top-k average in hybrid scoring + # Example: If nearest doc has score 0.9 and top-3 avg is 0.6, final = 0.7*0.9 + 0.3*0.6 = 0.81 + # Higher nearest_weight = more focus on best match vs neighborhood density + + # Embedding link selection parameters + embedding_overlap_threshold: float = 0.85 # Similarity threshold for penalizing redundant links + # Example: Links with >0.85 similarity to existing KB get penalized to avoid redundancy + # Lower = more aggressive deduplication, Higher = allow more similar content + + # Link preview timeout (seconds) + link_preview_timeout: float = 5.0 + + # Embedding stopping criteria parameters + embedding_min_relative_improvement: float = 0.1 # Minimum relative improvement to continue + # Example: If confidence is 0.6, need improvement > 0.06 per batch to continue crawling + # Lower = more patient crawling, Higher = stop earlier when progress slows + + embedding_validation_min_score: float = 0.3 # Minimum validation score to trust convergence + # Example: Even if learning converged, keep crawling if validation score < 0.4 + # This prevents premature stopping when we haven't truly covered the query space + + # Quality confidence mapping parameters (for display to user) + embedding_quality_min_confidence: float = 0.7 # Minimum confidence for validated systems + embedding_quality_max_confidence: float = 0.95 # Maximum realistic confidence + embedding_quality_scale_factor: float = 0.833 # Scaling factor for confidence mapping + # Example: Validated system with learning_score=0.5 → confidence = 0.7 + (0.5-0.4)*0.833 = 0.78 + # These control how internal scores map to user-friendly confidence percentages + + def validate(self): + """Validate configuration parameters""" + assert 0 <= self.confidence_threshold <= 1, "confidence_threshold must be between 0 and 1" + assert self.max_depth > 0, "max_depth must be positive" + assert self.max_pages > 0, "max_pages must be positive" + assert self.top_k_links > 0, "top_k_links must be positive" + assert 0 <= self.min_gain_threshold <= 1, "min_gain_threshold must be between 0 and 1" + + # Check weights sum to 1 + weight_sum = self.coverage_weight + self.consistency_weight + self.saturation_weight + assert abs(weight_sum - 1.0) < 0.001, f"Coverage weights must sum to 1, got {weight_sum}" + + weight_sum = self.relevance_weight + self.novelty_weight + self.authority_weight + assert abs(weight_sum - 1.0) < 0.001, f"Link scoring weights must sum to 1, got {weight_sum}" + + # Validate embedding parameters + assert 0 < self.embedding_coverage_radius < 1, "embedding_coverage_radius must be between 0 and 1" + assert self.embedding_k_exp > 0, "embedding_k_exp must be positive" + assert 0 <= self.embedding_nearest_weight <= 1, "embedding_nearest_weight must be between 0 and 1" + assert 0 <= self.embedding_top_k_weight <= 1, "embedding_top_k_weight must be between 0 and 1" + assert abs(self.embedding_nearest_weight + self.embedding_top_k_weight - 1.0) < 0.001, "Embedding weights must sum to 1" + assert 0 <= self.embedding_overlap_threshold <= 1, "embedding_overlap_threshold must be between 0 and 1" + assert 0 < self.embedding_min_relative_improvement < 1, "embedding_min_relative_improvement must be between 0 and 1" + assert 0 <= self.embedding_validation_min_score <= 1, "embedding_validation_min_score must be between 0 and 1" + assert 0 <= self.embedding_quality_min_confidence <= 1, "embedding_quality_min_confidence must be between 0 and 1" + assert 0 <= self.embedding_quality_max_confidence <= 1, "embedding_quality_max_confidence must be between 0 and 1" + assert self.embedding_quality_scale_factor > 0, "embedding_quality_scale_factor must be positive" + assert 0 <= self.embedding_min_confidence_threshold <= 1, "embedding_min_confidence_threshold must be between 0 and 1" + + @property + def _embedding_llm_config_dict(self) -> Optional[Dict]: + """Convert LLMConfig to dict format for backward compatibility.""" + if self.embedding_llm_config is None: + return None + if isinstance(self.embedding_llm_config, dict): + return self.embedding_llm_config + return self.embedding_llm_config.to_dict() + + @property + def _query_llm_config_dict(self) -> Optional[Dict]: + """Convert query LLM config to dict format.""" + if self.query_llm_config is None: + return None + if isinstance(self.query_llm_config, dict): + return self.query_llm_config + return self.query_llm_config.to_dict() + + +class CrawlStrategy(ABC): + """Abstract base class for crawling strategies""" + + @abstractmethod + async def calculate_confidence(self, state: CrawlState) -> float: + """Calculate overall confidence that we have sufficient information""" + pass + + @abstractmethod + async def rank_links(self, state: CrawlState, config: AdaptiveConfig) -> List[Tuple[Link, float]]: + """Rank pending links by expected information gain""" + pass + + @abstractmethod + async def should_stop(self, state: CrawlState, config: AdaptiveConfig) -> bool: + """Determine if crawling should stop""" + pass + + @abstractmethod + async def update_state(self, state: CrawlState, new_results: List[CrawlResult]) -> None: + """Update state with new crawl results""" + pass + + +class StatisticalStrategy(CrawlStrategy): + """Pure statistical approach - no LLM, no embeddings""" + + def __init__(self): + self.idf_cache = {} + self.bm25_k1 = 1.2 # BM25 parameter + self.bm25_b = 0.75 # BM25 parameter + + async def calculate_confidence(self, state: CrawlState) -> float: + """Calculate confidence using coverage, consistency, and saturation""" + if not state.knowledge_base: + return 0.0 + + coverage = self._calculate_coverage(state) + consistency = self._calculate_consistency(state) + saturation = self._calculate_saturation(state) + + # Store individual metrics + state.metrics['coverage'] = coverage + state.metrics['consistency'] = consistency + state.metrics['saturation'] = saturation + + # Weighted combination (weights from config not accessible here, using defaults) + confidence = 0.4 * coverage + 0.3 * consistency + 0.3 * saturation + + return confidence + + def _calculate_coverage(self, state: CrawlState) -> float: + """Coverage scoring - measures query term presence across knowledge base + + Returns a score between 0 and 1, where: + - 0 means no query terms found + - 1 means excellent coverage of all query terms + """ + if not state.query or state.total_documents == 0: + return 0.0 + + query_terms = self._tokenize(state.query.lower()) + if not query_terms: + return 0.0 + + term_scores = [] + max_tf = max(state.term_frequencies.values()) if state.term_frequencies else 1 + + for term in query_terms: + tf = state.term_frequencies.get(term, 0) + df = state.document_frequencies.get(term, 0) + + if df > 0: + # Document coverage: what fraction of docs contain this term + doc_coverage = df / state.total_documents + + # Frequency signal: normalized log frequency + freq_signal = math.log(1 + tf) / math.log(1 + max_tf) if max_tf > 0 else 0 + + # Combined score: document coverage with frequency boost + term_score = doc_coverage * (1 + 0.5 * freq_signal) + term_scores.append(term_score) + else: + term_scores.append(0.0) + + # Average across all query terms + coverage = sum(term_scores) / len(term_scores) + + # Apply square root curve to make score more intuitive + # This helps differentiate between partial and good coverage + return min(1.0, math.sqrt(coverage)) + + def _calculate_consistency(self, state: CrawlState) -> float: + """Information overlap between pages - high overlap suggests coherent topic coverage""" + if len(state.knowledge_base) < 2: + return 1.0 # Single or no documents are perfectly consistent + + # Calculate pairwise term overlap + overlaps = [] + + for i in range(len(state.knowledge_base)): + for j in range(i + 1, len(state.knowledge_base)): + # Get terms from both documents + terms_i = set(self._get_document_terms(state.knowledge_base[i])) + terms_j = set(self._get_document_terms(state.knowledge_base[j])) + + if terms_i and terms_j: + # Jaccard similarity + overlap = len(terms_i & terms_j) / len(terms_i | terms_j) + overlaps.append(overlap) + + if overlaps: + # Average overlap as consistency measure + consistency = sum(overlaps) / len(overlaps) + else: + consistency = 0.0 + + return consistency + + def _calculate_saturation(self, state: CrawlState) -> float: + """Diminishing returns indicator - are we still discovering new information?""" + if not state.new_terms_history: + return 0.0 + + if len(state.new_terms_history) < 2: + return 0.0 # Not enough history + + # Calculate rate of new term discovery + recent_rate = state.new_terms_history[-1] if state.new_terms_history[-1] > 0 else 1 + initial_rate = state.new_terms_history[0] if state.new_terms_history[0] > 0 else 1 + + # Saturation increases as rate decreases + saturation = 1 - (recent_rate / initial_rate) + + return max(0.0, min(saturation, 1.0)) + + async def rank_links(self, state: CrawlState, config: AdaptiveConfig) -> List[Tuple[Link, float]]: + """Rank links by expected information gain""" + scored_links = [] + + for link in state.pending_links: + # Skip already crawled URLs + if link.href in state.crawled_urls: + continue + + # Calculate component scores + relevance = self._calculate_relevance(link, state) + novelty = self._calculate_novelty(link, state) + authority = 1.0 + # authority = self._calculate_authority(link) + + # Combined score + score = (config.relevance_weight * relevance + + config.novelty_weight * novelty + + config.authority_weight * authority) + + scored_links.append((link, score)) + + # Sort by score descending + scored_links.sort(key=lambda x: x[1], reverse=True) + + return scored_links + + def _calculate_relevance(self, link: Link, state: CrawlState) -> float: + """BM25 relevance score between link preview and query""" + if not state.query or not link: + return 0.0 + + # Combine available text from link + link_text = ' '.join(filter(None, [ + link.text or '', + link.title or '', + link.head_data.get('meta', {}).get('title', '') if link.head_data else '', + link.head_data.get('meta', {}).get('description', '') if link.head_data else '', + link.head_data.get('meta', {}).get('keywords', '') if link.head_data else '' + ])).lower() + + if not link_text: + return 0.0 + + # Use contextual score if available (from BM25 scoring during crawl) + # if link.contextual_score is not None: + if link.contextual_score and link.contextual_score > 0: + return link.contextual_score + + # Otherwise, calculate simple term overlap + query_terms = set(self._tokenize(state.query.lower())) + link_terms = set(self._tokenize(link_text)) + + if not query_terms: + return 0.0 + + overlap = len(query_terms & link_terms) / len(query_terms) + return overlap + + def _calculate_novelty(self, link: Link, state: CrawlState) -> float: + """Estimate how much new information this link might provide""" + if not state.knowledge_base: + return 1.0 # First links are maximally novel + + # Get terms from link preview + link_text = ' '.join(filter(None, [ + link.text or '', + link.title or '', + link.head_data.get('title', '') if link.head_data else '', + link.head_data.get('description', '') if link.head_data else '', + link.head_data.get('keywords', '') if link.head_data else '' + ])).lower() + + link_terms = set(self._tokenize(link_text)) + if not link_terms: + return 0.5 # Unknown novelty + + # Calculate what percentage of link terms are new + existing_terms = set(state.term_frequencies.keys()) + new_terms = link_terms - existing_terms + + novelty = len(new_terms) / len(link_terms) if link_terms else 0.0 + + return novelty + + def _calculate_authority(self, link: Link) -> float: + """Simple authority score based on URL structure and link attributes""" + score = 0.5 # Base score + + if not link.href: + return 0.0 + + url = link.href.lower() + + # Positive indicators + if '/docs/' in url or '/documentation/' in url: + score += 0.2 + if '/api/' in url or '/reference/' in url: + score += 0.2 + if '/guide/' in url or '/tutorial/' in url: + score += 0.1 + + # Check for file extensions + if url.endswith('.pdf'): + score += 0.1 + elif url.endswith(('.jpg', '.png', '.gif')): + score -= 0.3 # Reduce score for images + + # Use intrinsic score if available + if link.intrinsic_score is not None: + score = 0.7 * score + 0.3 * link.intrinsic_score + + return min(score, 1.0) + + async def should_stop(self, state: CrawlState, config: AdaptiveConfig) -> bool: + """Determine if crawling should stop""" + # Check confidence threshold + confidence = state.metrics.get('confidence', 0.0) + if confidence >= config.confidence_threshold: + return True + + # Check resource limits + if len(state.crawled_urls) >= config.max_pages: + return True + + # Check if we have any links left + if not state.pending_links: + return True + + # Check saturation + if state.metrics.get('saturation', 0.0) >= config.saturation_threshold: + return True + + return False + + async def update_state(self, state: CrawlState, new_results: List[CrawlResult]) -> None: + """Update state with new crawl results""" + for result in new_results: + # Track new terms + old_term_count = len(state.term_frequencies) + + # Extract and process content - try multiple fields + try: + content = result.markdown.raw_markdown + except AttributeError: + print(f"Warning: CrawlResult {result.url} has no markdown content") + content = "" + # content = "" + # if hasattr(result, 'extracted_content') and result.extracted_content: + # content = result.extracted_content + # elif hasattr(result, 'markdown') and result.markdown: + # content = result.markdown.raw_markdown + # elif hasattr(result, 'cleaned_html') and result.cleaned_html: + # content = result.cleaned_html + # elif hasattr(result, 'html') and result.html: + # # Use raw HTML as last resort + # content = result.html + + + terms = self._tokenize(content.lower()) + + # Update term frequencies + term_set = set() + for term in terms: + state.term_frequencies[term] += 1 + term_set.add(term) + + # Update document frequencies + doc_id = state.total_documents + for term in term_set: + if term not in state.documents_with_terms[term]: + state.document_frequencies[term] += 1 + state.documents_with_terms[term].add(doc_id) + + # Track new terms discovered + new_term_count = len(state.term_frequencies) + new_terms = new_term_count - old_term_count + state.new_terms_history.append(new_terms) + + # Update document count + state.total_documents += 1 + + # Add to crawl order + state.crawl_order.append(result.url) + + def _tokenize(self, text: str) -> List[str]: + """Simple tokenization - can be enhanced""" + # Remove punctuation and split + text = re.sub(r'[^\w\s]', ' ', text) + tokens = text.split() + + # Filter short tokens and stop words (basic) + tokens = [t for t in tokens if len(t) > 2] + + return tokens + + def _get_document_terms(self, crawl_result: CrawlResult) -> List[str]: + """Extract terms from a crawl result""" + content = crawl_result.markdown.raw_markdown or "" + return self._tokenize(content.lower()) + + +class EmbeddingStrategy(CrawlStrategy): + """Embedding-based adaptive crawling using semantic space coverage""" + + def __init__(self, embedding_model: str = None, llm_config: Union[LLMConfig, Dict] = None, query_llm_config: Union[LLMConfig, Dict] = None): + self.embedding_model = embedding_model or "sentence-transformers/all-MiniLM-L6-v2" + self.llm_config = llm_config + self.query_llm_config = query_llm_config + self._embedding_cache = {} + self._link_embedding_cache = {} # Cache for link embeddings + self._validation_passed = False # Track if validation passed + + # Performance optimization caches + self._distance_matrix_cache = None # Cache for query-KB distances + self._kb_embeddings_hash = None # Track KB changes + self._validation_embeddings_cache = None # Cache validation query embeddings + self._kb_similarity_threshold = 0.95 # Threshold for deduplication + + def _get_embedding_llm_config_dict(self) -> Optional[Dict]: + """Get embedding LLM config as dict, or None for local embeddings.""" + if hasattr(self, 'config') and self.config: + config_dict = self.config._embedding_llm_config_dict + if config_dict: + return config_dict + + # Return None to use local sentence-transformers embeddings + return None + + def _get_query_llm_config_dict(self) -> Optional[Dict]: + """Get query LLM config as dict for chat completion calls. + + Fallback chain: + 1. self.query_llm_config (explicit query config on strategy) + 2. self.config._query_llm_config_dict (from AdaptiveConfig) + 3. self.llm_config (legacy: single config for both) + 4. None (caller uses hardcoded defaults) + """ + # 1. Explicit query config on strategy instance + if self.query_llm_config is not None: + if isinstance(self.query_llm_config, dict): + return self.query_llm_config + return self.query_llm_config.to_dict() + + # 2. From AdaptiveConfig + if hasattr(self, 'config') and self.config: + config_dict = self.config._query_llm_config_dict + if config_dict: + return config_dict + + # 3. Legacy fallback: use embedding/shared llm_config for backward compat + if self.llm_config is not None: + if isinstance(self.llm_config, dict): + return self.llm_config + return self.llm_config.to_dict() + + # 4. None — caller applies hardcoded defaults + return None + + async def _get_embeddings(self, texts: List[str]) -> Any: + """Get embeddings using configured method""" + from .utils import get_text_embeddings + embedding_llm_config = self._get_embedding_llm_config_dict() + return await get_text_embeddings( + texts, + embedding_llm_config, + self.embedding_model + ) + + def _compute_distance_matrix(self, query_embeddings: Any, kb_embeddings: Any) -> Any: + """Compute distance matrix using vectorized operations""" + + + if kb_embeddings is None or len(kb_embeddings) == 0: + return None + + # Ensure proper shapes + if len(query_embeddings.shape) == 1: + query_embeddings = query_embeddings.reshape(1, -1) + if len(kb_embeddings.shape) == 1: + kb_embeddings = kb_embeddings.reshape(1, -1) + + # Vectorized cosine distance: 1 - cosine_similarity + # Normalize vectors + query_norm = query_embeddings / np.linalg.norm(query_embeddings, axis=1, keepdims=True) + kb_norm = kb_embeddings / np.linalg.norm(kb_embeddings, axis=1, keepdims=True) + + # Compute cosine similarity matrix + similarity_matrix = np.dot(query_norm, kb_norm.T) + + # Convert to distance + distance_matrix = 1 - similarity_matrix + + return distance_matrix + + def _get_cached_distance_matrix(self, query_embeddings: Any, kb_embeddings: Any) -> Any: + """Get distance matrix with caching""" + + + if kb_embeddings is None or len(kb_embeddings) == 0: + return None + + # Check if KB has changed + kb_hash = hash(kb_embeddings.tobytes()) if kb_embeddings is not None else None + + if (self._distance_matrix_cache is None or + kb_hash != self._kb_embeddings_hash): + # Recompute matrix + self._distance_matrix_cache = self._compute_distance_matrix(query_embeddings, kb_embeddings) + self._kb_embeddings_hash = kb_hash + + return self._distance_matrix_cache + + async def map_query_semantic_space(self, query: str, n_synthetic: int = 10) -> Any: + """Generate a point cloud representing the semantic neighborhood of the query""" + from .utils import perform_completion_with_backoff + + # Generate more variations than needed for train/val split + n_total = int(n_synthetic * 1.3) # Generate 30% more for validation + + # Generate variations using LLM + prompt = f"""Generate {n_total} variations of this query that explore different aspects: '{query}' + + These should be queries a user might ask when looking for similar information. + Include different phrasings, related concepts, and specific aspects. + + Return as a JSON array of strings.""" + + # Use a chat completion model for query generation + llm_config_dict = self._get_query_llm_config_dict() + + provider = llm_config_dict.get('provider', 'openai/gpt-4o-mini') if llm_config_dict else 'openai/gpt-4o-mini' + api_token = llm_config_dict.get('api_token') if llm_config_dict else None + base_url = llm_config_dict.get('base_url') if llm_config_dict else None + + response = perform_completion_with_backoff( + provider=provider, + prompt_with_variables=prompt, + api_token=api_token, + json_response=True, + base_url=base_url, + base_delay=llm_config_dict.get('backoff_base_delay', 2) if llm_config_dict else 2, + max_attempts=llm_config_dict.get('backoff_max_attempts', 3) if llm_config_dict else 3, + exponential_factor=llm_config_dict.get('backoff_exponential_factor', 2) if llm_config_dict else 2, + ) + + variations = json.loads(response.choices[0].message.content) + + + # # Mock data with more variations for split + # variations ={'queries': ['what are the best vegetables to use in fried rice?', 'how do I make vegetable fried rice from scratch?', 'can you provide a quick recipe for vegetable fried rice?', 'what cooking techniques are essential for perfect fried rice with vegetables?', 'how to add flavor to vegetable fried rice?', 'are there any tips for making healthy fried rice with vegetables?']} + + + # variations = {'queries': [ + # 'How do async and await work with coroutines in Python?', + # 'What is the role of event loops in asynchronous programming?', + # 'Can you explain the differences between async/await and traditional callback methods?', + # 'How do coroutines interact with event loops in JavaScript?', + # 'What are the benefits of using async await over promises in Node.js?', + # 'How to manage multiple coroutines with an event loop?', + # 'What are some common pitfalls when using async await with coroutines?', + # 'How do different programming languages implement async await and event loops?', + # 'What happens when an async function is called without await?', + # 'How does the event loop handle blocking operations?', + # 'Can you nest async functions and how does that affect the event loop?', + # 'What is the performance impact of using async/await?' + # ]} + + # Split into train and validation + # all_queries = [query] + variations['queries'] + + # Randomly shuffle for proper train/val split (keeping original query in training) + import random + + # Keep original query always in training + other_queries = variations['queries'].copy() + random.shuffle(other_queries) + + # Split: 80% for training, 20% for validation + n_validation = max(2, int(len(other_queries) * 0.2)) # At least 2 for validation + val_queries = other_queries[-n_validation:] + train_queries = [query] + other_queries[:-n_validation] + + # Embed only training queries for now (faster) + train_embeddings = await self._get_embeddings(train_queries) + + # Store validation queries for later (don't embed yet to save time) + self._validation_queries = val_queries + + return train_embeddings, train_queries + + def compute_coverage_shape(self, query_points: Any, alpha: float = 0.5): + """Find the minimal shape that covers all query points using alpha shape""" + try: + + + if len(query_points) < 3: + return None + + # For high-dimensional embeddings (e.g., 384-dim, 768-dim), + # alpha shapes require exponentially more points than available. + # Instead, use a statistical coverage model + query_points = np.array(query_points) + + # Store coverage as centroid + radius model + coverage = { + 'center': np.mean(query_points, axis=0), + 'std': np.std(query_points, axis=0), + 'points': query_points, + 'radius': np.max(np.linalg.norm(query_points - np.mean(query_points, axis=0), axis=1)) + } + return coverage + except Exception: + # Fallback if computation fails + return None + + def _sample_boundary_points(self, shape, n_samples: int = 20) -> List[Any]: + """Sample points from the boundary of a shape""" + + + # Simplified implementation - in practice would sample from actual shape boundary + # For now, return empty list if shape is None + if shape is None: + return [] + + # This is a placeholder - actual implementation would depend on shape type + return [] + + def find_coverage_gaps(self, kb_embeddings: Any, query_embeddings: Any) -> List[Tuple[Any, float]]: + """Calculate gap distances for all query variations using vectorized operations""" + + + gaps = [] + + if kb_embeddings is None or len(kb_embeddings) == 0: + # If no KB yet, all query points have maximum gap + for q_emb in query_embeddings: + gaps.append((q_emb, 1.0)) + return gaps + + # Use cached distance matrix + distance_matrix = self._get_cached_distance_matrix(query_embeddings, kb_embeddings) + + if distance_matrix is None: + # Fallback + for q_emb in query_embeddings: + gaps.append((q_emb, 1.0)) + return gaps + + # Find minimum distance for each query (vectorized) + min_distances = np.min(distance_matrix, axis=1) + + # Create gaps list + for i, q_emb in enumerate(query_embeddings): + gaps.append((q_emb, min_distances[i])) + + return gaps + + async def select_links_for_expansion( + self, + candidate_links: List[Link], + gaps: List[Tuple[Any, float]], + kb_embeddings: Any + ) -> List[Tuple[Link, float]]: + """Select links that most efficiently fill the gaps""" + from .utils import cosine_distance, cosine_similarity, get_text_embeddings + + import hashlib + + scored_links = [] + + # Prepare for embedding - separate cached vs uncached + links_to_embed = [] + texts_to_embed = [] + link_embeddings_map = {} + + for link in candidate_links: + # Extract text from link + link_text = ' '.join(filter(None, [ + link.text or '', + link.title or '', + link.meta.get('description', '') if hasattr(link, 'meta') and link.meta else '', + link.head_data.get('meta', {}).get('description', '') if link.head_data else '' + ])) + + if not link_text.strip(): + continue + + # Create cache key from URL + text content + cache_key = hashlib.md5(f"{link.href}:{link_text}".encode()).hexdigest() + + # Check cache + if cache_key in self._link_embedding_cache: + link_embeddings_map[link.href] = self._link_embedding_cache[cache_key] + else: + links_to_embed.append(link) + texts_to_embed.append(link_text) + + # Batch embed only uncached links + if texts_to_embed: + embedding_llm_config = self._get_embedding_llm_config_dict() + new_embeddings = await get_text_embeddings(texts_to_embed, embedding_llm_config, self.embedding_model) + + # Cache the new embeddings + for link, text, embedding in zip(links_to_embed, texts_to_embed, new_embeddings): + cache_key = hashlib.md5(f"{link.href}:{text}".encode()).hexdigest() + self._link_embedding_cache[cache_key] = embedding + link_embeddings_map[link.href] = embedding + + # Get coverage radius from config + coverage_radius = self.config.embedding_coverage_radius if hasattr(self, 'config') else 0.2 + + # Score each link + for link in candidate_links: + if link.href not in link_embeddings_map: + continue # Skip links without embeddings + + link_embedding = link_embeddings_map[link.href] + + if not gaps: + score = 0.0 + else: + # Calculate how many gaps this link helps with + gaps_helped = 0 + total_improvement = 0 + + for gap_point, gap_distance in gaps: + # Only consider gaps that actually need filling (outside coverage radius) + if gap_distance > coverage_radius: + new_distance = cosine_distance(link_embedding, gap_point) + if new_distance < gap_distance: + # This link helps this gap + improvement = gap_distance - new_distance + # Scale improvement - moving from 0.5 to 0.3 is valuable + scaled_improvement = improvement * 2 # Amplify the signal + total_improvement += scaled_improvement + gaps_helped += 1 + + # Average improvement per gap that needs help + gaps_needing_help = sum(1 for _, d in gaps if d > coverage_radius) + if gaps_needing_help > 0: + gap_reduction_score = total_improvement / gaps_needing_help + else: + gap_reduction_score = 0 + + # Check overlap with existing KB (vectorized) + if kb_embeddings is not None and len(kb_embeddings) > 0: + # Normalize embeddings + link_norm = link_embedding / np.linalg.norm(link_embedding) + kb_norm = kb_embeddings / np.linalg.norm(kb_embeddings, axis=1, keepdims=True) + + # Compute all similarities at once + similarities = np.dot(kb_norm, link_norm) + max_similarity = np.max(similarities) + + # Only penalize if very similar (above threshold) + overlap_threshold = self.config.embedding_overlap_threshold if hasattr(self, 'config') else 0.85 + if max_similarity > overlap_threshold: + overlap_penalty = (max_similarity - overlap_threshold) * 2 # 0 to 0.3 range + else: + overlap_penalty = 0 + else: + overlap_penalty = 0 + + # Final score - emphasize gap reduction + score = gap_reduction_score * (1 - overlap_penalty) + + # Add contextual score boost if available + if hasattr(link, 'contextual_score') and link.contextual_score: + score = score * 0.8 + link.contextual_score * 0.2 + + scored_links.append((link, score)) + + return sorted(scored_links, key=lambda x: x[1], reverse=True) + + async def calculate_confidence(self, state: CrawlState) -> float: + """Coverage-based learning score (0–1).""" + # Guard clauses + if state.kb_embeddings is None or state.query_embeddings is None: + return 0.0 + if len(state.kb_embeddings) == 0 or len(state.query_embeddings) == 0: + return 0.0 + + # Prepare L2-normalised arrays + Q = np.asarray(state.query_embeddings, dtype=np.float32) + D = np.asarray(state.kb_embeddings, dtype=np.float32) + Q /= np.linalg.norm(Q, axis=1, keepdims=True) + 1e-8 + D /= np.linalg.norm(D, axis=1, keepdims=True) + 1e-8 + + # Best cosine per query + best = (Q @ D.T).max(axis=1) + + # Mean similarity or hit-rate above tau + tau = getattr(self.config, 'coverage_tau', None) + score = float((best >= tau).mean()) if tau is not None else float(best.mean()) + + # Store quick metrics + state.metrics['coverage_score'] = score + state.metrics['avg_best_similarity'] = float(best.mean()) + state.metrics['median_best_similarity'] = float(np.median(best)) + + return score + + + + # async def calculate_confidence(self, state: CrawlState) -> float: + # """Calculate learning score for adaptive crawling (used for stopping)""" + # + + # if state.kb_embeddings is None or state.query_embeddings is None: + # return 0.0 + + # if len(state.kb_embeddings) == 0: + # return 0.0 + + # # Get cached distance matrix + # distance_matrix = self._get_cached_distance_matrix(state.query_embeddings, state.kb_embeddings) + + # if distance_matrix is None: + # return 0.0 + + # # Vectorized analysis for all queries at once + # all_query_metrics = [] + + # for i in range(len(state.query_embeddings)): + # # Get distances for this query + # distances = distance_matrix[i] + # sorted_distances = np.sort(distances) + + # # Store metrics for this query + # query_metric = { + # 'min_distance': sorted_distances[0], + # 'top_3_distances': sorted_distances[:3], + # 'top_5_distances': sorted_distances[:5], + # 'close_neighbors': np.sum(distances < 0.3), + # 'very_close_neighbors': np.sum(distances < 0.2), + # 'all_distances': distances + # } + # all_query_metrics.append(query_metric) + + # # Hybrid approach with density (exponential base) + # k_exp = self.config.embedding_k_exp if hasattr(self, 'config') else 1.0 + # coverage_scores_hybrid_exp = [] + + # for metric in all_query_metrics: + # # Base score from nearest neighbor + # nearest_score = np.exp(-k_exp * metric['min_distance']) + + # # Top-k average (top 3) + # top_k = min(3, len(metric['all_distances'])) + # top_k_avg = np.mean([np.exp(-k_exp * d) for d in metric['top_3_distances'][:top_k]]) + + # # Combine using configured weights + # nearest_weight = self.config.embedding_nearest_weight if hasattr(self, 'config') else 0.7 + # top_k_weight = self.config.embedding_top_k_weight if hasattr(self, 'config') else 0.3 + # hybrid_score = nearest_weight * nearest_score + top_k_weight * top_k_avg + # coverage_scores_hybrid_exp.append(hybrid_score) + + # learning_score = np.mean(coverage_scores_hybrid_exp) + + # # Store as learning score + # state.metrics['learning_score'] = learning_score + + # # Store embedding-specific metrics + # state.metrics['avg_min_distance'] = np.mean([m['min_distance'] for m in all_query_metrics]) + # state.metrics['avg_close_neighbors'] = np.mean([m['close_neighbors'] for m in all_query_metrics]) + # state.metrics['avg_very_close_neighbors'] = np.mean([m['very_close_neighbors'] for m in all_query_metrics]) + # state.metrics['total_kb_docs'] = len(state.kb_embeddings) + + # # Store query-level metrics for detailed analysis + # self._query_metrics = all_query_metrics + + # # For stopping criteria, return learning score + # return float(learning_score) + + async def rank_links(self, state: CrawlState, config: AdaptiveConfig) -> List[Tuple[Link, float]]: + """Main entry point for link ranking""" + # Store config for use in other methods + self.config = config + + # Filter out already crawled URLs and remove duplicates + seen_urls = set() + uncrawled_links = [] + + for link in state.pending_links: + if link.href not in state.crawled_urls and link.href not in seen_urls: + uncrawled_links.append(link) + seen_urls.add(link.href) + + if not uncrawled_links: + return [] + + # Get gaps in coverage (no threshold needed anymore) + gaps = self.find_coverage_gaps( + state.kb_embeddings, + state.query_embeddings + ) + state.semantic_gaps = [(g[0].tolist(), g[1]) for g in gaps] # Store as list for serialization + + # Select links that fill gaps (only from uncrawled) + return await self.select_links_for_expansion( + uncrawled_links, + gaps, + state.kb_embeddings + ) + + async def validate_coverage(self, state: CrawlState) -> float: + """Validate coverage using held-out queries with caching""" + if not hasattr(self, '_validation_queries') or not self._validation_queries: + return state.metrics.get('confidence', 0.0) + + + + # Cache validation embeddings (only embed once!) + if self._validation_embeddings_cache is None: + self._validation_embeddings_cache = await self._get_embeddings(self._validation_queries) + + val_embeddings = self._validation_embeddings_cache + + # Use vectorized distance computation + if state.kb_embeddings is None or len(state.kb_embeddings) == 0: + return 0.0 + + # Compute distance matrix for validation queries + distance_matrix = self._compute_distance_matrix(val_embeddings, state.kb_embeddings) + + if distance_matrix is None: + return 0.0 + + # Find minimum distance for each validation query (vectorized) + min_distances = np.min(distance_matrix, axis=1) + scores = 1.0 - min_distances # Convert distances to scores (0-1 range) + + # Compute scores using same exponential as training + # k_exp = self.config.embedding_k_exp if hasattr(self, 'config') else 1.0 + # scores = np.exp(-k_exp * min_distances) + + validation_confidence = np.mean(scores) + state.metrics['validation_confidence'] = validation_confidence + + return validation_confidence + + async def should_stop(self, state: CrawlState, config: AdaptiveConfig) -> bool: + """Stop based on learning curve convergence""" + confidence = state.metrics.get('confidence', 0.0) + + # Check if confidence is below minimum threshold (completely irrelevant) + min_confidence_threshold = config.embedding_min_confidence_threshold if hasattr(config, 'embedding_min_confidence_threshold') else 0.1 + if confidence < min_confidence_threshold and len(state.crawled_urls) > 0: + state.metrics['stopped_reason'] = 'below_minimum_relevance_threshold' + state.metrics['is_irrelevant'] = True + return True + + # Basic limits + if len(state.crawled_urls) >= config.max_pages or not state.pending_links: + return True + + # Track confidence history + if not hasattr(state, 'confidence_history'): + state.confidence_history = [] + + state.confidence_history.append(confidence) + + # Need at least 3 iterations to check convergence + if len(state.confidence_history) < 2: + return False + + improvement_diffs = list(zip(state.confidence_history[:-1], state.confidence_history[1:])) + + # Calculate average improvement + avg_improvement = sum(abs(b - a) for a, b in improvement_diffs) / len(improvement_diffs) + state.metrics['avg_improvement'] = avg_improvement + + min_relative_improvement = self.config.embedding_min_relative_improvement * confidence if hasattr(self, 'config') else 0.1 * confidence + if avg_improvement < min_relative_improvement: + # Converged - validate before stopping + val_score = await self.validate_coverage(state) + + # Only stop if validation is reasonable + validation_min = self.config.embedding_validation_min_score if hasattr(self, 'config') else 0.4 + # k_exp = self.config.embedding_k_exp if hasattr(self, 'config') else 1.0 + # validation_min = np.exp(-k_exp * validation_min) + + if val_score > validation_min: + state.metrics['stopped_reason'] = 'converged_validated' + self._validation_passed = True + return True + else: + state.metrics['stopped_reason'] = 'low_validation' + # Continue crawling despite convergence + + return False + + def get_quality_confidence(self, state: CrawlState) -> float: + """Calculate quality-based confidence score for display""" + learning_score = state.metrics.get('learning_score', 0.0) + validation_score = state.metrics.get('validation_confidence', 0.0) + + # Get config values + validation_min = self.config.embedding_validation_min_score if hasattr(self, 'config') else 0.4 + quality_min = self.config.embedding_quality_min_confidence if hasattr(self, 'config') else 0.7 + quality_max = self.config.embedding_quality_max_confidence if hasattr(self, 'config') else 0.95 + scale_factor = self.config.embedding_quality_scale_factor if hasattr(self, 'config') else 0.833 + + if self._validation_passed and validation_score > validation_min: + # Validated systems get boosted scores + # Map 0.4-0.7 learning → quality_min-quality_max confidence + if learning_score < 0.4: + confidence = quality_min # Minimum for validated systems + elif learning_score > 0.7: + confidence = quality_max # Maximum realistic confidence + else: + # Linear mapping in between + confidence = quality_min + (learning_score - 0.4) * scale_factor + else: + # Not validated = conservative mapping + confidence = learning_score * 0.8 + + return confidence + + async def update_state(self, state: CrawlState, new_results: List[CrawlResult]) -> None: + """Update embeddings and coverage metrics with deduplication""" + from .utils import get_text_embeddings + + + # Extract text from results + new_texts = [] + valid_results = [] + for result in new_results: + content = result.markdown.raw_markdown if hasattr(result, 'markdown') and result.markdown else "" + if content: # Only process non-empty content + new_texts.append(content[:5000]) # Limit text length + valid_results.append(result) + + if not new_texts: + return + + # Get embeddings for new texts + embedding_llm_config = self._get_embedding_llm_config_dict() + new_embeddings = await get_text_embeddings(new_texts, embedding_llm_config, self.embedding_model) + + # Deduplicate embeddings before adding to KB + if state.kb_embeddings is None: + # First batch - no deduplication needed + state.kb_embeddings = new_embeddings + deduplicated_indices = list(range(len(new_embeddings))) + else: + # Check for duplicates using vectorized similarity + deduplicated_embeddings = [] + deduplicated_indices = [] + + for i, new_emb in enumerate(new_embeddings): + # Compute similarities with existing KB + new_emb_normalized = new_emb / np.linalg.norm(new_emb) + kb_normalized = state.kb_embeddings / np.linalg.norm(state.kb_embeddings, axis=1, keepdims=True) + similarities = np.dot(kb_normalized, new_emb_normalized) + + # Only add if not too similar to existing content + if np.max(similarities) < self._kb_similarity_threshold: + deduplicated_embeddings.append(new_emb) + deduplicated_indices.append(i) + + # Add deduplicated embeddings + if deduplicated_embeddings: + state.kb_embeddings = np.vstack([state.kb_embeddings, np.array(deduplicated_embeddings)]) + + # Update crawl order only for non-duplicate results + for idx in deduplicated_indices: + state.crawl_order.append(valid_results[idx].url) + + # Invalidate distance matrix cache since KB changed + self._kb_embeddings_hash = None + self._distance_matrix_cache = None + + # Update coverage shape if needed + if hasattr(state, 'query_embeddings') and state.query_embeddings is not None: + state.coverage_shape = self.compute_coverage_shape(state.query_embeddings, self.config.alpha_shape_alpha if hasattr(self, 'config') else 0.5) + + +class AdaptiveCrawler: + """Main adaptive crawler that orchestrates the crawling process""" + + def __init__(self, + crawler: Optional[AsyncWebCrawler] = None, + config: Optional[AdaptiveConfig] = None, + strategy: Optional[CrawlStrategy] = None): + self.crawler = crawler + self.config = config or AdaptiveConfig() + self.config.validate() + + # Create strategy based on config + if strategy: + self.strategy = strategy + else: + self.strategy = self._create_strategy(self.config.strategy) + + # Initialize state + self.state: Optional[CrawlState] = None + + # Track if we own the crawler (for cleanup) + self._owns_crawler = crawler is None + + def _create_strategy(self, strategy_name: str) -> CrawlStrategy: + """Create strategy instance based on name""" + if strategy_name == "statistical": + return StatisticalStrategy() + elif strategy_name == "embedding": + strategy = EmbeddingStrategy( + embedding_model=self.config.embedding_model, + llm_config=self.config.embedding_llm_config, + query_llm_config=self.config.query_llm_config, + ) + strategy.config = self.config # Pass config to strategy + return strategy + else: + raise ValueError(f"Unknown strategy: {strategy_name}") + + async def digest(self, + start_url: str, + query: str, + resume_from: Optional[str] = None) -> CrawlState: + """Main entry point for adaptive crawling""" + # Initialize or resume state + if resume_from: + self.state = CrawlState.load(resume_from) + self.state.query = query # Update query in case it changed + else: + self.state = CrawlState( + crawled_urls=set(), + knowledge_base=[], + pending_links=[], + query=query, + metrics={} + ) + + # Create crawler if needed + if not self.crawler: + self.crawler = AsyncWebCrawler() + await self.crawler.__aenter__() + + self.strategy.config = self.config # Pass config to strategy + + # If using embedding strategy and not resuming, expand query space + if isinstance(self.strategy, EmbeddingStrategy) and not resume_from: + # Generate query space + query_embeddings, expanded_queries = await self.strategy.map_query_semantic_space( + query, + self.config.n_query_variations + ) + self.state.query_embeddings = query_embeddings + self.state.expanded_queries = expanded_queries[1:] # Skip original query + self.state.embedding_model = self.strategy.embedding_model + + try: + # Initial crawl if not resuming + if start_url not in self.state.crawled_urls: + result = await self._crawl_with_preview(start_url, query) + if result and hasattr(result, 'success') and result.success: + self.state.knowledge_base.append(result) + self.state.crawled_urls.add(start_url) + # Extract links from result - handle both dict and Links object formats + if hasattr(result, 'links') and result.links: + if isinstance(result.links, dict): + # Extract internal and external links from dict + internal_links = [Link(**link) for link in result.links.get('internal', [])] + self.state.pending_links.extend(internal_links) + else: + # Handle Links object + self.state.pending_links.extend(result.links.internal) + + # Update state + await self.strategy.update_state(self.state, [result]) + + # adaptive expansion + depth = 0 + while depth < self.config.max_depth: + # Calculate confidence + confidence = await self.strategy.calculate_confidence(self.state) + self.state.metrics['confidence'] = confidence + + # Check stopping criteria + if await self.strategy.should_stop(self.state, self.config): + break + + # Rank candidate links + ranked_links = await self.strategy.rank_links(self.state, self.config) + + if not ranked_links: + break + + # Check minimum gain threshold + if ranked_links[0][1] < self.config.min_gain_threshold: + break + + # Select top K links + to_crawl = [(link, score) for link, score in ranked_links[:self.config.top_k_links] + if link.href not in self.state.crawled_urls] + + if not to_crawl: + break + + # Crawl selected links + new_results = await self._crawl_batch(to_crawl, query) + + if new_results: + # Update knowledge base + self.state.knowledge_base.extend(new_results) + + # Update crawled URLs and pending links + for result, (link, _) in zip(new_results, to_crawl): + if result: + self.state.crawled_urls.add(link.href) + # Extract links from result - handle both dict and Links object formats + if hasattr(result, 'links') and result.links: + new_links = [] + if isinstance(result.links, dict): + # Extract internal and external links from dict + internal_links = [Link(**link_data) for link_data in result.links.get('internal', [])] + new_links = internal_links + else: + # Handle Links object + new_links = result.links.internal + + # Add new links to pending + for new_link in new_links: + if new_link.href not in self.state.crawled_urls: + self.state.pending_links.append(new_link) + + # Update state with new results + await self.strategy.update_state(self.state, new_results) + + depth += 1 + + # Save state if configured + if self.config.save_state and self.config.state_path: + self.state.save(self.config.state_path) + + # Final confidence calculation + learning_score = await self.strategy.calculate_confidence(self.state) + + # For embedding strategy, get quality-based confidence + if isinstance(self.strategy, EmbeddingStrategy): + self.state.metrics['confidence'] = self.strategy.get_quality_confidence(self.state) + else: + # For statistical strategy, use the same as before + self.state.metrics['confidence'] = learning_score + + self.state.metrics['pages_crawled'] = len(self.state.crawled_urls) + self.state.metrics['depth_reached'] = depth + + # Final save + if self.config.save_state and self.config.state_path: + self.state.save(self.config.state_path) + + return self.state + + finally: + # Cleanup if we created the crawler + if self._owns_crawler and self.crawler: + await self.crawler.__aexit__(None, None, None) + + async def _crawl_with_preview(self, url: str, query: str) -> Optional[CrawlResult]: + """Crawl a URL with link preview enabled""" + config = CrawlerRunConfig( + link_preview_config=LinkPreviewConfig( + include_internal=True, + include_external=False, + query=query, # For BM25 scoring + concurrency=5, + timeout=self.config.link_preview_timeout, + max_links=50, # Reasonable limit + verbose=False + ), + score_links=True # Enable intrinsic scoring + ) + + try: + result = await self.crawler.arun(url=url, config=config) + # Extract the actual CrawlResult from the container + if hasattr(result, '_results') and result._results: + result = result._results[0] + + # Filter our all links do not have head_date + if hasattr(result, 'links') and result.links: + result.links['internal'] = [link for link in result.links['internal'] if link.get('head_data')] + # For now let's ignore external links without head_data + # result.links['external'] = [link for link in result.links['external'] if link.get('head_data')] + + return result + except Exception as e: + print(f"Error crawling {url}: {e}") + return None + + async def _crawl_batch(self, links_with_scores: List[Tuple[Link, float]], query: str) -> List[CrawlResult]: + """Crawl multiple URLs in parallel""" + tasks = [] + for link, score in links_with_scores: + task = self._crawl_with_preview(link.href, query) + tasks.append(task) + + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Filter out exceptions and failed crawls + valid_results = [] + for result in results: + if isinstance(result, CrawlResult): + # Only include successful crawls + if hasattr(result, 'success') and result.success: + valid_results.append(result) + else: + print(f"Skipping failed crawl: {result.url if hasattr(result, 'url') else 'unknown'}") + elif isinstance(result, Exception): + print(f"Error in batch crawl: {result}") + + return valid_results + + # Status properties + @property + def confidence(self) -> float: + """Current confidence level""" + if self.state: + return self.state.metrics.get('confidence', 0.0) + return 0.0 + + @property + def coverage_stats(self) -> Dict[str, Any]: + """Detailed coverage statistics""" + if not self.state: + return {} + + total_content_length = sum( + len(result.markdown.raw_markdown or "") + for result in self.state.knowledge_base + ) + + return { + 'pages_crawled': len(self.state.crawled_urls), + 'total_content_length': total_content_length, + 'unique_terms': len(self.state.term_frequencies), + 'total_terms': sum(self.state.term_frequencies.values()), + 'pending_links': len(self.state.pending_links), + 'confidence': self.confidence, + 'coverage': self.state.metrics.get('coverage', 0.0), + 'consistency': self.state.metrics.get('consistency', 0.0), + 'saturation': self.state.metrics.get('saturation', 0.0) + } + + @property + def is_sufficient(self) -> bool: + """Check if current knowledge is sufficient""" + if isinstance(self.strategy, EmbeddingStrategy): + # For embedding strategy, sufficient = validation passed + return self.strategy._validation_passed + else: + # For statistical strategy, use threshold + return self.confidence >= self.config.confidence_threshold + + def print_stats(self, detailed: bool = False) -> None: + """Print comprehensive statistics about the knowledge base + + Args: + detailed: If True, show detailed statistics including top terms + """ + if not self.state: + print("No crawling state available.") + return + + # Import here to avoid circular imports + try: + from rich.console import Console + from rich.table import Table + console = Console() + use_rich = True + except ImportError: + use_rich = False + + if not detailed and use_rich: + # Summary view with nice table (like original) + table = Table(title=f"Adaptive Crawl Stats - Query: '{self.state.query}'") + table.add_column("Metric", style="cyan", no_wrap=True) + table.add_column("Value", style="magenta") + + # Basic stats + stats = self.coverage_stats + table.add_row("Pages Crawled", str(stats.get('pages_crawled', 0))) + table.add_row("Unique Terms", str(stats.get('unique_terms', 0))) + table.add_row("Total Terms", str(stats.get('total_terms', 0))) + table.add_row("Content Length", f"{stats.get('total_content_length', 0):,} chars") + table.add_row("Pending Links", str(stats.get('pending_links', 0))) + table.add_row("", "") # Spacer + + # Strategy-specific metrics + if isinstance(self.strategy, EmbeddingStrategy): + # Embedding-specific metrics + table.add_row("Confidence", f"{stats.get('confidence', 0):.2%}") + table.add_row("Avg Min Distance", f"{self.state.metrics.get('avg_min_distance', 0):.3f}") + table.add_row("Avg Close Neighbors", f"{self.state.metrics.get('avg_close_neighbors', 0):.1f}") + table.add_row("Validation Score", f"{self.state.metrics.get('validation_confidence', 0):.2%}") + table.add_row("", "") # Spacer + table.add_row("Is Sufficient?", "[green]Yes (Validated)[/green]" if self.is_sufficient else "[red]No[/red]") + else: + # Statistical strategy metrics + table.add_row("Confidence", f"{stats.get('confidence', 0):.2%}") + table.add_row("Coverage", f"{stats.get('coverage', 0):.2%}") + table.add_row("Consistency", f"{stats.get('consistency', 0):.2%}") + table.add_row("Saturation", f"{stats.get('saturation', 0):.2%}") + table.add_row("", "") # Spacer + table.add_row("Is Sufficient?", "[green]Yes[/green]" if self.is_sufficient else "[red]No[/red]") + + console.print(table) + else: + # Detailed view or fallback when rich not available + print("\n" + "="*80) + print(f"Adaptive Crawl Statistics - Query: '{self.state.query}'") + print("="*80) + + # Basic stats + print("\n[*] Basic Statistics:") + print(f" Pages Crawled: {len(self.state.crawled_urls)}") + print(f" Pending Links: {len(self.state.pending_links)}") + print(f" Total Documents: {self.state.total_documents}") + + # Content stats + total_content_length = sum( + len(self._get_content_from_result(result)) + for result in self.state.knowledge_base + ) + total_words = sum(self.state.term_frequencies.values()) + unique_terms = len(self.state.term_frequencies) + + print(f"\n[*] Content Statistics:") + print(f" Total Content: {total_content_length:,} characters") + print(f" Total Words: {total_words:,}") + print(f" Unique Terms: {unique_terms:,}") + if total_words > 0: + print(f" Vocabulary Richness: {unique_terms/total_words:.2%}") + + # Strategy-specific output + if isinstance(self.strategy, EmbeddingStrategy): + # Semantic coverage for embedding strategy + print(f"\n[*] Semantic Coverage Analysis:") + print(f" Average Min Distance: {self.state.metrics.get('avg_min_distance', 0):.3f}") + print(f" Avg Close Neighbors (< 0.3): {self.state.metrics.get('avg_close_neighbors', 0):.1f}") + print(f" Avg Very Close Neighbors (< 0.2): {self.state.metrics.get('avg_very_close_neighbors', 0):.1f}") + + # Confidence metrics + print(f"\n[*] Confidence Metrics:") + if self.is_sufficient: + if use_rich: + console.print(f" Overall Confidence: {self.confidence:.2%} [green][VALIDATED][/green]") + else: + print(f" Overall Confidence: {self.confidence:.2%} [VALIDATED]") + else: + if use_rich: + console.print(f" Overall Confidence: {self.confidence:.2%} [red][NOT VALIDATED][/red]") + else: + print(f" Overall Confidence: {self.confidence:.2%} [NOT VALIDATED]") + + print(f" Learning Score: {self.state.metrics.get('learning_score', 0):.2%}") + print(f" Validation Score: {self.state.metrics.get('validation_confidence', 0):.2%}") + + else: + # Query coverage for statistical strategy + print(f"\n[*] Query Coverage:") + query_terms = self.strategy._tokenize(self.state.query.lower()) + for term in query_terms: + tf = self.state.term_frequencies.get(term, 0) + df = self.state.document_frequencies.get(term, 0) + if df > 0: + if use_rich: + console.print(f" '{term}': found in {df}/{self.state.total_documents} docs ([green]{df/self.state.total_documents:.0%}[/green]), {tf} occurrences") + else: + print(f" '{term}': found in {df}/{self.state.total_documents} docs ({df/self.state.total_documents:.0%}), {tf} occurrences") + else: + if use_rich: + console.print(f" '{term}': [red][X] not found[/red]") + else: + print(f" '{term}': [X] not found") + + # Confidence metrics + print(f"\n[*] Confidence Metrics:") + status = "[OK]" if self.is_sufficient else "[!!]" + if use_rich: + status_colored = "[green][OK][/green]" if self.is_sufficient else "[red][!!][/red]" + console.print(f" Overall Confidence: {self.confidence:.2%} {status_colored}") + else: + print(f" Overall Confidence: {self.confidence:.2%} {status}") + print(f" Coverage Score: {self.state.metrics.get('coverage', 0):.2%}") + print(f" Consistency Score: {self.state.metrics.get('consistency', 0):.2%}") + print(f" Saturation Score: {self.state.metrics.get('saturation', 0):.2%}") + + # Crawl efficiency + if self.state.new_terms_history: + avg_new_terms = sum(self.state.new_terms_history) / len(self.state.new_terms_history) + print(f"\n[*] Crawl Efficiency:") + print(f" Avg New Terms per Page: {avg_new_terms:.1f}") + print(f" Information Saturation: {self.state.metrics.get('saturation', 0):.2%}") + + if detailed: + print("\n" + "-"*80) + if use_rich: + console.print("[bold cyan]DETAILED STATISTICS[/bold cyan]") + else: + print("DETAILED STATISTICS") + print("-"*80) + + # Top terms + print("\n[+] Top 20 Terms by Frequency:") + top_terms = sorted(self.state.term_frequencies.items(), key=lambda x: x[1], reverse=True)[:20] + for i, (term, freq) in enumerate(top_terms, 1): + df = self.state.document_frequencies.get(term, 0) + if use_rich: + console.print(f" {i:2d}. [yellow]'{term}'[/yellow]: {freq} occurrences in {df} docs") + else: + print(f" {i:2d}. '{term}': {freq} occurrences in {df} docs") + + # URLs crawled + print(f"\n[+] URLs Crawled ({len(self.state.crawled_urls)}):") + for i, url in enumerate(self.state.crawl_order, 1): + new_terms = self.state.new_terms_history[i-1] if i <= len(self.state.new_terms_history) else 0 + if use_rich: + console.print(f" {i}. [cyan]{url}[/cyan]") + console.print(f" -> Added [green]{new_terms}[/green] new terms") + else: + print(f" {i}. {url}") + print(f" -> Added {new_terms} new terms") + + # Document frequency distribution + print("\n[+] Document Frequency Distribution:") + df_counts = {} + for df in self.state.document_frequencies.values(): + df_counts[df] = df_counts.get(df, 0) + 1 + + for df in sorted(df_counts.keys()): + count = df_counts[df] + print(f" Terms in {df} docs: {count} terms") + + # Embedding stats + if self.state.embedding_model: + print("\n[+] Semantic Coverage Analysis:") + print(f" Embedding Model: {self.state.embedding_model}") + print(f" Query Variations: {len(self.state.expanded_queries)}") + if self.state.kb_embeddings is not None: + print(f" Knowledge Embeddings: {self.state.kb_embeddings.shape}") + else: + print(f" Knowledge Embeddings: None") + print(f" Semantic Gaps: {len(self.state.semantic_gaps)}") + print(f" Coverage Achievement: {self.confidence:.2%}") + + # Show sample expanded queries + if self.state.expanded_queries: + print("\n[+] Query Space (samples):") + for i, eq in enumerate(self.state.expanded_queries[:5], 1): + if use_rich: + console.print(f" {i}. [yellow]{eq}[/yellow]") + else: + print(f" {i}. {eq}") + + print("\n" + "="*80) + + def _get_content_from_result(self, result) -> str: + """Helper to safely extract content from result""" + if hasattr(result, 'markdown') and result.markdown: + if hasattr(result.markdown, 'raw_markdown'): + return result.markdown.raw_markdown or "" + return str(result.markdown) + return "" + + def export_knowledge_base(self, filepath: Union[str, Path], format: str = "jsonl") -> None: + """Export the knowledge base to a file + + Args: + filepath: Path to save the file + format: Export format - currently supports 'jsonl' + """ + if not self.state or not self.state.knowledge_base: + print("No knowledge base to export.") + return + + filepath = Path(filepath) + filepath.parent.mkdir(parents=True, exist_ok=True) + + if format == "jsonl": + # Export as JSONL - one CrawlResult per line + with open(filepath, 'w', encoding='utf-8') as f: + for result in self.state.knowledge_base: + # Convert CrawlResult to dict + result_dict = self._crawl_result_to_export_dict(result) + # Write as single line JSON + f.write(json.dumps(result_dict, ensure_ascii=False) + '\n') + + print(f"Exported {len(self.state.knowledge_base)} documents to {filepath}") + else: + raise ValueError(f"Unsupported export format: {format}") + + def _crawl_result_to_export_dict(self, result) -> Dict[str, Any]: + """Convert CrawlResult to a dictionary for export""" + # Extract all available fields + export_dict = { + 'url': getattr(result, 'url', ''), + 'timestamp': getattr(result, 'timestamp', None), + 'success': getattr(result, 'success', True), + 'query': self.state.query if self.state else '', + } + + # Extract content + if hasattr(result, 'markdown') and result.markdown: + if hasattr(result.markdown, 'raw_markdown'): + export_dict['content'] = result.markdown.raw_markdown + else: + export_dict['content'] = str(result.markdown) + else: + export_dict['content'] = '' + + # Extract metadata + if hasattr(result, 'metadata'): + export_dict['metadata'] = result.metadata + + # Extract links if available + if hasattr(result, 'links'): + export_dict['links'] = result.links + + # Add crawl-specific metadata + if self.state: + export_dict['crawl_metadata'] = { + 'crawl_order': self.state.crawl_order.index(export_dict['url']) + 1 if export_dict['url'] in self.state.crawl_order else 0, + 'confidence_at_crawl': self.state.metrics.get('confidence', 0), + 'total_documents': self.state.total_documents + } + + return export_dict + + async def import_knowledge_base(self, filepath: Union[str, Path], format: str = "jsonl") -> None: + """Import a knowledge base from a file + + Args: + filepath: Path to the file to import + format: Import format - currently supports 'jsonl' + """ + filepath = Path(filepath) + if not filepath.exists(): + raise FileNotFoundError(f"File not found: {filepath}") + + if format == "jsonl": + imported_results = [] + with open(filepath, 'r', encoding='utf-8') as f: + for line in f: + if line.strip(): + data = json.loads(line) + # Convert back to a mock CrawlResult + mock_result = self._import_dict_to_crawl_result(data) + imported_results.append(mock_result) + + # Initialize state if needed + if not self.state: + self.state = CrawlState() + + # Add imported results + self.state.knowledge_base.extend(imported_results) + + # Update state with imported data + await self.strategy.update_state(self.state, imported_results) + + print(f"Imported {len(imported_results)} documents from {filepath}") + else: + raise ValueError(f"Unsupported import format: {format}") + + def _import_dict_to_crawl_result(self, data: Dict[str, Any]): + """Convert imported dict back to a mock CrawlResult""" + class MockMarkdown: + def __init__(self, content): + self.raw_markdown = content + + class MockCrawlResult: + def __init__(self, data): + self.url = data.get('url', '') + self.markdown = MockMarkdown(data.get('content', '')) + self.links = data.get('links', {}) + self.metadata = data.get('metadata', {}) + self.success = data.get('success', True) + self.timestamp = data.get('timestamp') + + return MockCrawlResult(data) + + def get_relevant_content(self, top_k: int = 5) -> List[Dict[str, Any]]: + """Get most relevant content for the query""" + if not self.state or not self.state.knowledge_base: + return [] + + # Simple relevance ranking based on term overlap + scored_docs = [] + query_terms = set(self.state.query.lower().split()) + + for i, result in enumerate(self.state.knowledge_base): + content = (result.markdown.raw_markdown or "").lower() + content_terms = set(content.split()) + + # Calculate relevance score + overlap = len(query_terms & content_terms) + score = overlap / len(query_terms) if query_terms else 0.0 + + scored_docs.append({ + 'url': result.url, + 'score': score, + 'content': result.markdown.raw_markdown, + 'index': i + }) + + # Sort by score and return top K + scored_docs.sort(key=lambda x: x['score'], reverse=True) + return scored_docs[:top_k] \ No newline at end of file diff --git a/crawl4ai/antibot_detector.py b/crawl4ai/antibot_detector.py new file mode 100644 index 0000000..228c1b2 --- /dev/null +++ b/crawl4ai/antibot_detector.py @@ -0,0 +1,281 @@ +""" +Anti-bot detection heuristics for crawl results. + +Examines HTTP status codes and HTML content patterns to determine +if a crawl was blocked by anti-bot protection. + +Detection philosophy: false positives are cheap (the fallback mechanism +rescues them), false negatives are catastrophic (user gets garbage). +Err on the side of detection. + +Detection is layered: +- HTTP 403/503 with HTML content → always blocked (these are never desired content) +- Tier 1 patterns (structural markers) trigger on any page size +- Tier 2 patterns (generic terms) trigger on short pages or any error status +- Tier 3 structural integrity catches silent blocks and empty shells +""" + +import re +from typing import Optional, Tuple + + +# --------------------------------------------------------------------------- +# Tier 1: High-confidence structural markers (single signal sufficient) +# These are unique to block pages and virtually never appear in real content. +# --------------------------------------------------------------------------- +_TIER1_PATTERNS = [ + # Akamai — full reference pattern: Reference #18.2d351ab8.1557333295.a4e16ab + (re.compile(r"Reference\s*#\s*[\d]+\.[0-9a-f]+\.\d+\.[0-9a-f]+", re.IGNORECASE), + "Akamai block (Reference #)"), + # Akamai — "Pardon Our Interruption" challenge page + (re.compile(r"Pardon\s+Our\s+Interruption", re.IGNORECASE), + "Akamai challenge (Pardon Our Interruption)"), + # Cloudflare — challenge form with anti-bot token + (re.compile(r'challenge-form.*?__cf_chl_f_tk=', re.IGNORECASE | re.DOTALL), + "Cloudflare challenge form"), + # Cloudflare — error code spans (1020 Access Denied, 1010, 1012, 1015) + (re.compile(r'\d{4}', re.IGNORECASE), + "Cloudflare firewall block"), + # Cloudflare — IUAM challenge script + (re.compile(r'/cdn-cgi/challenge-platform/\S+orchestrate', re.IGNORECASE), + "Cloudflare JS challenge"), + # PerimeterX / HUMAN — block page with app ID assignment (not prose mentions) + (re.compile(r"window\._pxAppId\s*=", re.IGNORECASE), + "PerimeterX block"), + # PerimeterX — captcha CDN + (re.compile(r"captcha\.px-cdn\.net", re.IGNORECASE), + "PerimeterX captcha"), + # DataDome — captcha delivery domain (structural, not the word "datadome") + (re.compile(r"captcha-delivery\.com", re.IGNORECASE), + "DataDome captcha"), + # Imperva/Incapsula — resource iframe + (re.compile(r"_Incapsula_Resource", re.IGNORECASE), + "Imperva/Incapsula block"), + # Imperva/Incapsula — incident ID + (re.compile(r"Incapsula\s+incident\s+ID", re.IGNORECASE), + "Imperva/Incapsula incident"), + # Sucuri firewall + (re.compile(r"Sucuri\s+WebSite\s+Firewall", re.IGNORECASE), + "Sucuri firewall block"), + # Kasada + (re.compile(r"KPSDK\.scriptStart\s*=\s*KPSDK\.now\(\)", re.IGNORECASE), + "Kasada challenge"), + # Network security block — Reddit and other platforms serve large SPA shells + # with this message buried under 100KB+ of CSS/JS + (re.compile(r"blocked\s+by\s+network\s+security", re.IGNORECASE), + "Network security block"), +] + +# --------------------------------------------------------------------------- +# Tier 2: Medium-confidence patterns — only match on SHORT pages (< 10KB) +# These terms appear in real content (articles, login forms, security blogs) +# so we require the page to be small to avoid false positives. +# --------------------------------------------------------------------------- +_TIER2_PATTERNS = [ + # Akamai / generic — "Access Denied" (extremely common on legit 403s too) + (re.compile(r"Access\s+Denied", re.IGNORECASE), + "Access Denied on short page"), + # Cloudflare — "Just a moment" / "Checking your browser" + (re.compile(r"Checking\s+your\s+browser", re.IGNORECASE), + "Cloudflare browser check"), + (re.compile(r"\s*Just\s+a\s+moment", re.IGNORECASE), + "Cloudflare interstitial"), + # CAPTCHA on a block page (not a login form — login forms are big pages) + (re.compile(r'class=["\']g-recaptcha["\']', re.IGNORECASE), + "reCAPTCHA on block page"), + (re.compile(r'class=["\']h-captcha["\']', re.IGNORECASE), + "hCaptcha on block page"), + # PerimeterX block page title + (re.compile(r"Access\s+to\s+This\s+Page\s+Has\s+Been\s+Blocked", re.IGNORECASE), + "PerimeterX block page"), + # Generic block phrases (only on short pages to avoid matching articles) + (re.compile(r"blocked\s+by\s+security", re.IGNORECASE), + "Blocked by security"), + (re.compile(r"Request\s+unsuccessful", re.IGNORECASE), + "Request unsuccessful (Imperva)"), +] + +_TIER2_MAX_SIZE = 10000 # Only check tier 2 patterns on pages under 10KB + +# --------------------------------------------------------------------------- +# Tier 3: Structural integrity — catches silent blocks, anti-bot redirects, +# incomplete renders that pass pattern detection but are structurally broken +# --------------------------------------------------------------------------- +_STRUCTURAL_MAX_SIZE = 50000 # Only check pages under 50KB +_CONTENT_ELEMENTS_RE = re.compile( + r'<(?:p|h[1-6]|article|section|li|td|a|pre)\b', re.IGNORECASE +) +_SCRIPT_TAG_RE = re.compile(r'<script\b', re.IGNORECASE) +_STYLE_TAG_RE = re.compile(r'<style\b[\s\S]*?</style>', re.IGNORECASE) +_SCRIPT_BLOCK_RE = re.compile(r'<script\b[\s\S]*?</script>', re.IGNORECASE) +_TAG_RE = re.compile(r'<[^>]+>') +_BODY_RE = re.compile(r'<body\b', re.IGNORECASE) + +# --------------------------------------------------------------------------- +# Thresholds +# --------------------------------------------------------------------------- +_BLOCK_PAGE_MAX_SIZE = 5000 # 403 + short page = likely block +_EMPTY_CONTENT_THRESHOLD = 100 # 200 + near-empty = JS-blocked render + + +def _looks_like_data(html: str) -> bool: + """Check if content looks like a JSON/XML API response (not an HTML block page).""" + stripped = html.strip() + if not stripped: + return False + # Raw JSON/XML (not wrapped in HTML) + if stripped[0] in ('{', '['): + return True + # Browser-rendered JSON: browsers wrap raw JSON in <html><body><pre>{...}</pre> + if stripped[:10].lower().startswith(('<html', '<!')): + if re.search(r'<body[^>]*>\s*<pre[^>]*>\s*[{\[]', stripped[:500], re.IGNORECASE): + return True + return False + # Other XML-like content + return stripped[0] == '<' + + +def _structural_integrity_check(html: str) -> Tuple[bool, str]: + """ + Tier 3: Structural integrity check for pages that pass pattern detection + but are structurally broken — incomplete renders, anti-bot redirects, empty shells. + + Only applies to pages < 50KB that aren't JSON/XML. + + Returns: + Tuple of (is_blocked, reason). + """ + html_len = len(html) + + # Skip large pages (unlikely to be block pages) and data responses + if html_len > _STRUCTURAL_MAX_SIZE or _looks_like_data(html): + return False, "" + + signals = [] + + # Signal 1: No <body> tag — definitive structural failure + if not _BODY_RE.search(html): + return True, f"Structural: no <body> tag ({html_len} bytes)" + + # Signal 2: Minimal visible text after stripping scripts/styles/tags + body_match = re.search(r'<body\b[^>]*>([\s\S]*)</body>', html, re.IGNORECASE) + body_content = body_match.group(1) if body_match else html + stripped = _SCRIPT_BLOCK_RE.sub('', body_content) + stripped = _STYLE_TAG_RE.sub('', stripped) + visible_text = _TAG_RE.sub('', stripped).strip() + visible_len = len(visible_text) + if visible_len < 50: + signals.append("minimal_text") + + # Signal 3: No content elements (semantic HTML) + content_elements = len(_CONTENT_ELEMENTS_RE.findall(html)) + if content_elements == 0: + signals.append("no_content_elements") + + # Signal 4: Script-heavy shell — scripts present but no content + script_count = len(_SCRIPT_TAG_RE.findall(html)) + if script_count > 0 and content_elements == 0 and visible_len < 100: + signals.append("script_heavy_shell") + + # Scoring + signal_count = len(signals) + if signal_count >= 2: + return True, f"Structural: {', '.join(signals)} ({html_len} bytes, {visible_len} chars visible)" + + if signal_count == 1 and html_len < 5000: + return True, f"Structural: {signals[0]} on small page ({html_len} bytes, {visible_len} chars visible)" + + return False, "" + + +def is_blocked( + status_code: Optional[int], + html: str, + error_message: Optional[str] = None, +) -> Tuple[bool, str]: + """ + Detect if a crawl result indicates anti-bot blocking. + + Uses layered detection to maximize coverage while minimizing false positives: + - Tier 1 patterns (structural markers) trigger on any page size + - Tier 2 patterns (generic terms) only trigger on short pages (< 10KB) + - Tier 3 structural integrity catches silent blocks and empty shells + - Status-code checks require corroborating content signals + + Args: + status_code: HTTP status code from the response. + html: Raw HTML content from the response. + error_message: Error message from the crawl result, if any. + + Returns: + Tuple of (is_blocked, reason). reason is empty string when not blocked. + """ + html = html or "" + html_len = len(html) + + # --- HTTP 429 is always rate limiting --- + if status_code == 429: + return True, "HTTP 429 Too Many Requests" + + # --- Check for tier 1 patterns (high confidence, any page size) --- + # First check the raw start of the page (fast path for small pages). + # Then, for large pages, also check a stripped version (scripts/styles + # removed) because modern block pages bury text under 100KB+ of CSS/JS. + snippet = html[:15000] + if snippet: + for pattern, reason in _TIER1_PATTERNS: + if pattern.search(snippet): + return True, reason + + # Large-page deep scan: strip scripts/styles and re-check tier 1 + if html_len > 15000: + _stripped_for_t1 = _SCRIPT_BLOCK_RE.sub('', html[:500000]) + _stripped_for_t1 = _STYLE_TAG_RE.sub('', _stripped_for_t1) + _deep_snippet = _stripped_for_t1[:30000] + for pattern, reason in _TIER1_PATTERNS: + if pattern.search(_deep_snippet): + return True, reason + + # --- HTTP 403/503 — always blocked for non-data HTML responses --- + # Rationale: 403/503 are never the content the user wants. Modern block pages + # (Reddit, LinkedIn, etc.) serve full SPA shells that exceed 100KB, so + # size-based filtering misses them. Even for a legitimate auth error, the + # fallback (Web Unlocker) will also get 403 and we correctly report failure. + # False positives are cheap — the fallback mechanism rescues them. + if status_code in (403, 503) and not _looks_like_data(html): + if html_len < _EMPTY_CONTENT_THRESHOLD: + return True, f"HTTP {status_code} with near-empty response ({html_len} bytes)" + # For large pages, strip scripts/styles to find block text in the + # actual content (Reddit hides it under 180KB of inline CSS). + # Check tier 2 patterns regardless of page size. + if html_len > _TIER2_MAX_SIZE: + _stripped = _SCRIPT_BLOCK_RE.sub('', html[:500000]) + _stripped = _STYLE_TAG_RE.sub('', _stripped) + _check_snippet = _stripped[:30000] + else: + _check_snippet = snippet + for pattern, reason in _TIER2_PATTERNS: + if pattern.search(_check_snippet): + return True, f"{reason} (HTTP {status_code}, {html_len} bytes)" + # Even without a pattern match, a non-data 403/503 HTML page is + # almost certainly a block. Flag it so the fallback gets a chance. + return True, f"HTTP {status_code} with HTML content ({html_len} bytes)" + + # --- Tier 2 patterns on other 4xx/5xx + short page --- + if status_code and status_code >= 400 and html_len < _TIER2_MAX_SIZE: + for pattern, reason in _TIER2_PATTERNS: + if pattern.search(snippet): + return True, f"{reason} (HTTP {status_code}, {html_len} bytes)" + + # --- HTTP 200 + near-empty content (JS-rendered empty page) --- + if status_code == 200: + stripped = html.strip() + if len(stripped) < _EMPTY_CONTENT_THRESHOLD and not _looks_like_data(html): + return True, f"Near-empty content ({len(stripped)} bytes) with HTTP 200" + + # --- Tier 3: Structural integrity (catches silent blocks, redirects, incomplete renders) --- + _blocked, _reason = _structural_integrity_check(html) + if _blocked: + return True, _reason + + return False, "" diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py new file mode 100644 index 0000000..27320fd --- /dev/null +++ b/crawl4ai/async_configs.py @@ -0,0 +1,2527 @@ +import copy +import functools +import importlib +import os +import warnings +import requests +from .config import ( + DEFAULT_PROVIDER, + DEFAULT_PROVIDER_API_KEY, + MIN_WORD_THRESHOLD, + IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, + PROVIDER_MODELS, + PROVIDER_MODELS_PREFIXES, + SCREENSHOT_HEIGHT_TRESHOLD, + PAGE_TIMEOUT, + IMAGE_SCORE_THRESHOLD, + SOCIAL_MEDIA_DOMAINS, +) + +from .user_agent_generator import UAGen, ValidUAGenerator # , OnlineUAGenerator +from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy +from .chunking_strategy import ChunkingStrategy, RegexChunking + +from .markdown_generation_strategy import MarkdownGenerationStrategy, DefaultMarkdownGenerator +from .content_scraping_strategy import ContentScrapingStrategy, LXMLWebScrapingStrategy +from .deep_crawling import DeepCrawlStrategy +from .table_extraction import TableExtractionStrategy, DefaultTableExtraction + +from .cache_context import CacheMode +from .proxy_strategy import ProxyRotationStrategy + +import inspect +from typing import Any, Awaitable, Callable, Dict, List, Optional, Union +from enum import Enum + +# Type alias for URL matching +UrlMatcher = Union[str, Callable[[str], bool], List[Union[str, Callable[[str], bool]]]] + + +def _with_defaults(cls): + """Class decorator: adds set_defaults/get_defaults/reset_defaults classmethods. + + After decorating, every new instance resolves parameters as: + explicit arg > class-level user defaults > hardcoded default + + Usage:: + + BrowserConfig.set_defaults(headless=False, viewport_width=1920) + cfg = BrowserConfig() # headless=False, viewport_width=1920 + cfg = BrowserConfig(headless=True) # explicit wins → headless=True + """ + original_init = cls.__init__ + sig = inspect.signature(original_init) + param_names = [p for p in sig.parameters if p != "self"] + valid_params = frozenset(param_names) + + @functools.wraps(original_init) + def wrapped_init(self, *args, **kwargs): + user_defaults = type(self)._user_defaults + if user_defaults: + # Determine which params the caller passed explicitly + explicit = set(kwargs.keys()) + for i in range(len(args)): + if i < len(param_names): + explicit.add(param_names[i]) + # Inject user defaults for non-explicit params + for key, value in user_defaults.items(): + if key not in explicit: + kwargs[key] = copy.deepcopy(value) + original_init(self, *args, **kwargs) + + cls.__init__ = wrapped_init + cls._user_defaults = {} + + @classmethod + def set_defaults(klass, **kwargs): + """Set class-level default overrides for new instances. + + Args: + **kwargs: Parameter names and their default values. + + Raises: + ValueError: If any key is not a valid ``__init__`` parameter. + """ + invalid = set(kwargs) - valid_params + if invalid: + raise ValueError( + f"Invalid parameter(s) for {klass.__name__}: {invalid}" + ) + for k, v in kwargs.items(): + klass._user_defaults[k] = copy.deepcopy(v) + + @classmethod + def get_defaults(klass): + """Return a deep copy of the current class-level defaults.""" + return copy.deepcopy(klass._user_defaults) + + @classmethod + def reset_defaults(klass, *names): + """Clear class-level defaults. + + With no arguments, removes all overrides. + With arguments, removes only the named overrides. + """ + if names: + for n in names: + klass._user_defaults.pop(n, None) + else: + klass._user_defaults.clear() + + cls.set_defaults = set_defaults + cls.get_defaults = get_defaults + cls.reset_defaults = reset_defaults + return cls + + +class MatchMode(Enum): + OR = "or" + AND = "and" + +# from .proxy_strategy import ProxyConfig + +# Allowlist of types that can be deserialized via from_serializable_dict(). +# This prevents arbitrary class instantiation from untrusted input (e.g. API requests). +ALLOWED_DESERIALIZE_TYPES = { + # Config classes + "BrowserConfig", "CrawlerRunConfig", "HTTPCrawlerConfig", + "LLMConfig", "ProxyConfig", "GeolocationConfig", + "SeedingConfig", "VirtualScrollConfig", "LinkPreviewConfig", "DomainMapperConfig", + # Extraction strategies + "JsonCssExtractionStrategy", "JsonXPathExtractionStrategy", + "JsonLxmlExtractionStrategy", "LLMExtractionStrategy", + "CosineStrategy", "RegexExtractionStrategy", + # Markdown / content + "DefaultMarkdownGenerator", + "PruningContentFilter", "BM25ContentFilter", "LLMContentFilter", + # Scraping + "LXMLWebScrapingStrategy", "PDFContentScrapingStrategy", + # Chunking + "RegexChunking", + # Deep crawl + "BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy", + # Filters & scorers + "FilterChain", "URLPatternFilter", "DomainFilter", + "ContentTypeFilter", "URLFilter", "SEOFilter", "ContentRelevanceFilter", + "KeywordRelevanceScorer", "URLScorer", "CompositeScorer", + "DomainAuthorityScorer", "FreshnessScorer", "PathDepthScorer", + # Enums + "CacheMode", "MatchMode", "DisplayMode", + # Dispatchers + "MemoryAdaptiveDispatcher", "SemaphoreDispatcher", + # Table extraction + "DefaultTableExtraction", "NoTableExtraction", "LLMTableExtraction", + # Proxy + "RoundRobinProxyStrategy", +} + + +# ───────────────────────── untrusted-input trust boundary ───────────────────────── +# When a config is deserialized from an untrusted source (a network request body +# on the Docker server), it may only construct a strict subset of types and may +# only set scalar, non-power fields, which are then clamped. SDK / in-process use +# is TRUSTED by default, so this changes nothing for library callers. + + +class Provenance(Enum): + """Where deserialized config data came from. + + TRUSTED - SDK / in-process construction (unchanged behavior). + UNTRUSTED - a network request body; gated by the allowlists below. + """ + TRUSTED = "trusted" + UNTRUSTED = "untrusted" + + +class UntrustedConfigError(ValueError): + """An untrusted request tried to construct a forbidden type or set a + forbidden power-field. The Docker layer maps this to HTTP 400.""" + + +# Types an untrusted body may construct - a strict subset of +# ALLOWED_DESERIALIZE_TYPES. Deliberately EXCLUDES everything that can run code, +# read secrets, route traffic, or recurse the crawl: LLMConfig, +# LLMExtractionStrategy, LLMContentFilter, LLMTableExtraction, ProxyConfig, +# RoundRobinProxyStrategy, all DeepCrawl*/Filter/Scorer/Dispatcher classes, +# SeedingConfig and DomainMapperConfig. +UNTRUSTED_ALLOWED_TYPES = { + "CrawlerRunConfig", "BrowserConfig", "HTTPCrawlerConfig", + "GeolocationConfig", "VirtualScrollConfig", "LinkPreviewConfig", + # non-LLM extraction / markdown / scraping / chunking strategies + "JsonCssExtractionStrategy", "JsonXPathExtractionStrategy", + "JsonLxmlExtractionStrategy", "RegexExtractionStrategy", "CosineStrategy", + "DefaultMarkdownGenerator", "PruningContentFilter", "BM25ContentFilter", + "LXMLWebScrapingStrategy", "PDFContentScrapingStrategy", + "RegexChunking", + "DefaultTableExtraction", "NoTableExtraction", + # safe scalar enums + "CacheMode", "MatchMode", "DisplayMode", +} + +# Fields that must NEVER come from an untrusted body. Presence => 400 (loud), so +# a client smuggling these gets an explicit error rather than a silent drop. +UNTRUSTED_FORBIDDEN_FIELDS = { + "BrowserConfig": { + "proxy", "proxy_config", "extra_args", "user_data_dir", "channel", + "chrome_channel", "cdp_url", "debugging_port", "host", "storage_state", + "cookies", "headers", "init_scripts", "browser_context_id", "target_id", + }, + "CrawlerRunConfig": { + "js_code", "js_code_before_wait", "c4a_script", "deep_crawl_strategy", + "proxy_config", "proxy_rotation_strategy", "proxy_session_id", + "proxy_session_ttl", "proxy_session_auto_release", + "fallback_fetch_function", "experimental", "base_url", "simulate_user", + "override_navigator", "magic", "process_in_browser", "shared_data", + "session_id", + }, +} + +# Scalar knobs an untrusted body MAY set, per class. A field not listed here is +# dropped (forward-compatible: new SDK fields don't open a hole). Strategy-object +# fields are kept so the nested object is itself type-gated by the recursion. +UNTRUSTED_FIELD_ALLOWLIST = { + "BrowserConfig": { + "browser_type", "headless", "browser_mode", "viewport_width", + "viewport_height", "viewport", "device_scale_factor", "accept_downloads", + "java_script_enabled", "text_mode", "light_mode", "enable_stealth", + "avoid_ads", "avoid_css", "user_agent", "user_agent_mode", + "user_agent_generator_config", "verbose", "memory_saving_mode", + "max_pages_before_recycle", + }, + "CrawlerRunConfig": { + # content selection / cleaning + "word_count_threshold", "only_text", "css_selector", "target_elements", + "excluded_tags", "excluded_selector", "keep_data_attributes", "keep_attrs", + "remove_forms", "prettiify", "parser_type", + # strategy objects (nested type is gated by the recursion) + "extraction_strategy", "chunking_strategy", "markdown_generator", + "scraping_strategy", "table_extraction", + # locale / geo + "locale", "timezone_id", "geolocation", + # cache + "cache_mode", "bypass_cache", "disable_cache", "no_cache_read", + "no_cache_write", "check_cache_freshness", "cache_validation_timeout", + "fetch_ssl_certificate", + # timing / waiting + "wait_until", "page_timeout", "wait_for", "wait_for_timeout", + "wait_for_images", "delay_before_return_html", "mean_delay", "max_range", + # scrolling / rendering + "ignore_body_visibility", "scan_full_page", "scroll_delay", + "max_scroll_steps", "process_iframes", "flatten_shadow_dom", + "remove_overlay_elements", "remove_consent_popups", + "adjust_viewport_to_content", "virtual_scroll_config", + # media / capture + "screenshot", "screenshot_wait_for", "screenshot_height_threshold", + "force_viewport_screenshot", "pdf", "capture_mhtml", + "image_description_min_word_threshold", "image_score_threshold", + "table_score_threshold", + # links / images filtering + "exclude_external_images", "exclude_all_images", + "exclude_social_media_domains", "exclude_external_links", + "exclude_social_media_links", "exclude_domains", "exclude_internal_links", + "score_links", "preserve_https_for_internal_links", "link_preview_config", + # misc safe knobs + "verbose", "log_console", "capture_network_requests", + "capture_console_messages", "method", "stream", "prefetch", "url", + "check_robots_txt", "user_agent", "user_agent_mode", + "user_agent_generator_config", "url_matcher", "match_mode", "max_retries", + }, +} + +# Upper bounds applied to attacker-influenced quantities after filtering. +_MAX_TIMEOUT_MS = 60_000 +_MAX_SCROLL_STEPS = 1000 +_MAX_VIEWPORT = 4000 + + +def _filter_untrusted_fields(type_name: str, params: dict) -> dict: + """Drop non-allowlisted fields and raise on forbidden (power) fields.""" + forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set()) + allowlist = UNTRUSTED_FIELD_ALLOWLIST.get(type_name) # None => keep all non-forbidden + out = {} + for key, value in params.items(): + if key in forbidden: + raise UntrustedConfigError( + f"field '{key}' is not permitted on {type_name} from an untrusted request" + ) + if allowlist is not None and key not in allowlist: + continue # silently drop unknown/unsafe fields (forward-compatible) + out[key] = value + return out + + +def _clamp_untrusted(type_name: str, params: dict) -> dict: + """Clamp attacker-influenced quantities to safe upper bounds.""" + def _cap_timeout(v): + # 0 historically meant "no timeout"; treat as the cap, never unbounded. + if not isinstance(v, (int, float)) or v <= 0: + return _MAX_TIMEOUT_MS + return min(int(v), _MAX_TIMEOUT_MS) + + if type_name == "CrawlerRunConfig": + for f in ("page_timeout", "wait_for_timeout"): + if f in params: + params[f] = _cap_timeout(params[f]) + if isinstance(params.get("max_scroll_steps"), int): + params["max_scroll_steps"] = min(params["max_scroll_steps"], _MAX_SCROLL_STEPS) + elif type_name == "BrowserConfig": + for f in ("viewport_width", "viewport_height"): + if isinstance(params.get(f), int): + params[f] = max(1, min(params[f], _MAX_VIEWPORT)) + return params + + +def _enforce_untrusted(type_name: str, params: dict) -> dict: + """Filter then clamp a parameter dict for an untrusted construction.""" + return _clamp_untrusted(type_name, _filter_untrusted_fields(type_name, params)) + + +def to_serializable_dict(obj: Any, ignore_default_value : bool = False): + """ + Recursively convert an object to a serializable dictionary using {type, params} structure + for complex objects. + """ + if obj is None: + return None + + # Handle basic types + if isinstance(obj, (str, int, float, bool)): + return obj + + # Handle Enum + if isinstance(obj, Enum): + return {"type": obj.__class__.__name__, "params": obj.value} + + # Handle datetime objects + if hasattr(obj, "isoformat"): + return obj.isoformat() + + # Handle lists, tuples, and sets, and basically any iterable + if isinstance(obj, (list, tuple, set)) or hasattr(obj, '__iter__') and not isinstance(obj, dict): + return [to_serializable_dict(item) for item in obj] + + # Handle frozensets, which are not iterable + if isinstance(obj, frozenset): + return [to_serializable_dict(item) for item in list(obj)] + + # Handle dictionaries - preserve them as-is + if isinstance(obj, dict): + return { + "type": "dict", # Mark as plain dictionary + "value": {str(k): to_serializable_dict(v) for k, v in obj.items()}, + } + + _type = obj.__class__.__name__ + + # Handle class instances + if hasattr(obj, "__class__"): + # Skip types that cannot be deserialized (e.g. logging.Logger, callables). + # Only serialize objects whose type is in ALLOWED_DESERIALIZE_TYPES so that + # from_serializable_dict can reconstruct them on the other side. + if _type not in ALLOWED_DESERIALIZE_TYPES: + return None + + # Get constructor signature + sig = inspect.signature(obj.__class__.__init__) + params = sig.parameters + + # Get current values + current_values = {} + for name, param in params.items(): + if name == "self": + continue + + value = getattr(obj, name, param.default) + + # Only include if different from default, considering empty values + if not (is_empty_value(value) and is_empty_value(param.default)): + if value != param.default and not ignore_default_value: + current_values[name] = to_serializable_dict(value) + + # Don't serialize private __slots__ - they're internal implementation details + # not constructor parameters. This was causing URLPatternFilter to fail + # because _simple_suffixes was being serialized as 'simple_suffixes' + # if hasattr(obj, '__slots__'): + # for slot in obj.__slots__: + # if slot.startswith('_'): # Handle private slots + # attr_name = slot[1:] # Remove leading '_' + # value = getattr(obj, slot, None) + # if value is not None: + # current_values[attr_name] = to_serializable_dict(value) + + return { + "type": obj.__class__.__name__, + "params": current_values + } + + return str(obj) + + +def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any: + """ + Recursively convert a serializable dictionary back to an object instance. + + provenance defaults to TRUSTED (SDK/in-process). When UNTRUSTED, every typed + object must be in UNTRUSTED_ALLOWED_TYPES and its params are filtered (drop + unknown, raise on forbidden) and clamped before construction. + """ + if provenance is None: + provenance = Provenance.TRUSTED + + if data is None: + return None + + # Handle basic types + if isinstance(data, (str, int, float, bool)): + return data + + # Handle typed data. + # Only enter the typed-object path for dicts that match the shapes produced + # by to_serializable_dict(): {"type": "<ClassName>", "params": {...}} or + # {"type": "dict", "value": {...}}. Plain business dicts that happen to + # carry a "type" key (e.g. JSON-Schema fragments, JsonCss field specs like + # {"type": "text", "name": "..."}) have neither "params" nor "value" and + # must fall through to the raw-dict path below so they are passed as data. + if ( + isinstance(data, dict) + and "type" in data + and ("params" in data or (data["type"] == "dict" and "value" in data)) + ): + # Handle plain dictionaries + if data["type"] == "dict" and "value" in data: + return {k: from_serializable_dict(v, provenance) for k, v in data["value"].items()} + + # Security: only allow known-safe types to be deserialized. + # Unknown types (e.g. logging.Logger serialized by older clients) are + # silently dropped (returned as None) instead of crashing the request. + type_name = data["type"] + if type_name not in ALLOWED_DESERIALIZE_TYPES: + return None + + # Untrusted bodies may only construct the strict subset of types and + # may not set forbidden power-fields. + if provenance == Provenance.UNTRUSTED and type_name not in UNTRUSTED_ALLOWED_TYPES: + raise UntrustedConfigError( + f"type '{type_name}' may not be constructed from an untrusted request" + ) + + cls = None + module_paths = ["crawl4ai"] + for module_path in module_paths: + try: + mod = importlib.import_module(module_path) + if hasattr(mod, type_name): + cls = getattr(mod, type_name) + break + except (ImportError, AttributeError): + continue + + if cls is not None: + # Handle Enum + if issubclass(cls, Enum): + return cls(data["params"]) + + if "params" in data: + params = data["params"] + if provenance == Provenance.UNTRUSTED: + params = _enforce_untrusted(type_name, dict(params)) + # Handle class instances + constructor_args = { + k: from_serializable_dict(v, provenance) for k, v in params.items() + } + return cls(**constructor_args) + + # Handle lists + if isinstance(data, list): + return [from_serializable_dict(item, provenance) for item in data] + + # Handle raw dictionaries (legacy support) + if isinstance(data, dict): + return {k: from_serializable_dict(v, provenance) for k, v in data.items()} + + return data + + +def is_empty_value(value: Any) -> bool: + """Check if a value is effectively empty/null.""" + if value is None: + return True + if isinstance(value, (list, tuple, set, dict, str)) and len(value) == 0: + return True + return False + +class GeolocationConfig: + def __init__( + self, + latitude: float, + longitude: float, + accuracy: Optional[float] = 0.0 + ): + """Configuration class for geolocation settings. + + Args: + latitude: Latitude coordinate (e.g., 37.7749) + longitude: Longitude coordinate (e.g., -122.4194) + accuracy: Accuracy in meters. Default: 0.0 + """ + self.latitude = latitude + self.longitude = longitude + self.accuracy = accuracy + + @staticmethod + def from_dict(geo_dict: Dict) -> "GeolocationConfig": + """Create a GeolocationConfig from a dictionary.""" + return GeolocationConfig( + latitude=geo_dict.get("latitude"), + longitude=geo_dict.get("longitude"), + accuracy=geo_dict.get("accuracy", 0.0) + ) + + def to_dict(self) -> Dict: + """Convert to dictionary representation.""" + return { + "latitude": self.latitude, + "longitude": self.longitude, + "accuracy": self.accuracy + } + + def clone(self, **kwargs) -> "GeolocationConfig": + """Create a copy of this configuration with updated values. + + Args: + **kwargs: Key-value pairs of configuration options to update + + Returns: + GeolocationConfig: A new instance with the specified updates + """ + config_dict = self.to_dict() + config_dict.update(kwargs) + return GeolocationConfig.from_dict(config_dict) + +class ProxyConfig: + DIRECT = "direct" # Sentinel: use in proxy_config list to mean "no proxy" + + def __init__( + self, + server: str, + username: Optional[str] = None, + password: Optional[str] = None, + ip: Optional[str] = None, + ): + """Configuration class for a single proxy. + + Args: + server: Proxy server URL (e.g., "http://127.0.0.1:8080") + username: Optional username for proxy authentication + password: Optional password for proxy authentication + ip: Optional IP address for verification purposes + """ + self.server = server + self.username = username + self.password = password + + # Extract IP from server if not explicitly provided + self.ip = ip or self._extract_ip_from_server() + + def _extract_ip_from_server(self) -> Optional[str]: + """Extract IP address from server URL.""" + try: + # Simple extraction assuming http://ip:port format + if "://" in self.server: + parts = self.server.split("://")[1].split(":") + return parts[0] + else: + parts = self.server.split(":") + return parts[0] + except Exception: + return None + + @staticmethod + def from_string(proxy_str: str) -> "ProxyConfig": + """Create a ProxyConfig from a string. + + Supported formats: + - 'http://username:password@ip:port' + - 'http://ip:port' + - 'socks5://ip:port' + - 'ip:port:username:password' + - 'ip:port' + """ + s = (proxy_str or "").strip() + # URL with credentials + if "@" in s and "://" in s: + auth_part, server_part = s.split("@", 1) + protocol, credentials = auth_part.split("://", 1) + if ":" in credentials: + username, password = credentials.split(":", 1) + return ProxyConfig( + server=f"{protocol}://{server_part}", + username=username, + password=password, + ) + # URL without credentials (keep scheme) + if "://" in s and "@" not in s: + return ProxyConfig(server=s) + # Colon separated forms + parts = s.split(":") + if len(parts) == 4: + ip, port, username, password = parts + return ProxyConfig(server=f"http://{ip}:{port}", username=username, password=password) + if len(parts) == 2: + ip, port = parts + return ProxyConfig(server=f"http://{ip}:{port}") + raise ValueError(f"Invalid proxy string format: {proxy_str}") + + @staticmethod + def from_dict(proxy_dict: Dict) -> "ProxyConfig": + """Create a ProxyConfig from a dictionary.""" + return ProxyConfig( + server=proxy_dict.get("server"), + username=proxy_dict.get("username"), + password=proxy_dict.get("password"), + ip=proxy_dict.get("ip"), + ) + + @staticmethod + def from_env(env_var: str = "PROXIES") -> List["ProxyConfig"]: + """Load proxies from environment variable. + + Args: + env_var: Name of environment variable containing comma-separated proxy strings + + Returns: + List of ProxyConfig objects + """ + proxies = [] + try: + proxy_list = os.getenv(env_var, "").split(",") + for proxy in proxy_list: + if not proxy: + continue + proxies.append(ProxyConfig.from_string(proxy)) + except Exception as e: + print(f"Error loading proxies from environment: {e}") + return proxies + + def to_dict(self) -> Dict: + """Convert to dictionary representation.""" + return { + "server": self.server, + "username": self.username, + "password": self.password, + "ip": self.ip, + } + + def clone(self, **kwargs) -> "ProxyConfig": + """Create a copy of this configuration with updated values. + + Args: + **kwargs: Key-value pairs of configuration options to update + + Returns: + ProxyConfig: A new instance with the specified updates + """ + config_dict = self.to_dict() + config_dict.update(kwargs) + return ProxyConfig.from_dict(config_dict) + +@_with_defaults +class BrowserConfig: + """ + Configuration class for setting up a browser instance and its context in AsyncPlaywrightCrawlerStrategy. + + This class centralizes all parameters that affect browser and context creation. Instead of passing + scattered keyword arguments, users can instantiate and modify this configuration object. The crawler + code will then reference these settings to initialize the browser in a consistent, documented manner. + + Attributes: + browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". + Default: "chromium". + headless (bool): Whether to run the browser in headless mode (no visible GUI). + Default: True. + browser_mode (str): Determines how the browser should be initialized: + "builtin" - use the builtin CDP browser running in background + "dedicated" - create a new dedicated browser instance each time + "cdp" - use explicit CDP settings provided in cdp_url + "docker" - run browser in Docker container with isolation + Default: "dedicated" + use_managed_browser (bool): Launch the browser using a managed approach (e.g., via CDP), allowing + advanced manipulation. Default: False. + cdp_url (str): URL for the Chrome DevTools Protocol (CDP) endpoint. Default: "ws://localhost:9222/devtools/browser/". + browser_context_id (str or None): Pre-existing CDP browser context ID to use. When provided along with + cdp_url, the crawler will reuse this context instead of creating a new one. + Useful for cloud browser services that pre-create isolated contexts. + Default: None. + target_id (str or None): Pre-existing CDP target ID (page) to use. When provided along with + browser_context_id, the crawler will reuse this target instead of creating + a new page. Default: None. + cdp_cleanup_on_close (bool): When True and using cdp_url, the close() method will still clean up + the local Playwright client resources. Useful for cloud/server scenarios + where you don't own the remote browser but need to prevent memory leaks + from accumulated Playwright instances. Default: False. + cdp_close_delay (float): Seconds to wait after disconnecting a CDP WebSocket before stopping the + Playwright subprocess. Gives the connection time to fully release. Set to + 0 to skip the delay entirely. Only applies when cdp_cleanup_on_close=True. + Default: 1.0. + cache_cdp_connection (bool): When True and using cdp_url, the Playwright subprocess and CDP WebSocket + are cached at the class level and shared across multiple BrowserManager + instances connecting to the same cdp_url. Reference-counted; the connection + is only closed when the last user releases it. Eliminates the overhead of + repeated Playwright/CDP setup and teardown. Default: False. + create_isolated_context (bool): When True and using cdp_url, forces creation of a new browser context + instead of reusing the default context. Essential for concurrent crawls + on the same browser to prevent navigation conflicts. Default: False. + debugging_port (int): Port for the browser debugging protocol. Default: 9222. + use_persistent_context (bool): Use a persistent browser context (like a persistent profile). + Automatically sets use_managed_browser=True. Default: False. + user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a + temporary directory may be used. Default: None. + chrome_channel (str): The Chrome channel to launch (e.g., "chrome", "msedge"). Only applies if browser_type + is "chromium". Default: "chromium". + channel (str): The channel to launch (e.g., "chromium", "chrome", "msedge"). Only applies if browser_type + is "chromium". Default: "chromium". + proxy (Optional[str]): Proxy server URL (e.g., "http://username:password@proxy:port"). If None, no proxy is used. + Default: None. + proxy_config (ProxyConfig or dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}. + If None, no additional proxy config. Default: None. + viewport_width (int): Default viewport width for pages. Default: 1080. + viewport_height (int): Default viewport height for pages. Default: 600. + viewport (dict): Default viewport dimensions for pages. If set, overrides viewport_width and viewport_height. + Default: None. + device_scale_factor (float): The device pixel ratio used for rendering pages. Controls how many + physical pixels map to one CSS pixel, allowing simulation of HiDPI + or Retina displays. For example, a viewport of 1920x1080 with a + device_scale_factor of 2.0 produces screenshots at 3840x2160 resolution. + Increasing this value improves screenshot quality but may increase + memory usage and rendering time. + Default: 1.0. + verbose (bool): Enable verbose logging. + Default: True. + accept_downloads (bool): Whether to allow file downloads. If True, requires a downloads_path. + Default: False. + downloads_path (str or None): Directory to store downloaded files. If None and accept_downloads is True, + a default path will be created. Default: None. + storage_state (str or dict or None): An in-memory storage state (cookies, localStorage). + Default: None. + ignore_https_errors (bool): Ignore HTTPS certificate errors. Default: True. + java_script_enabled (bool): Enable JavaScript execution in pages. Default: True. + cookies (list): List of cookies to add to the browser context. Each cookie is a dict with fields like + {"name": "...", "value": "...", "url": "..."}. + Default: []. + headers (dict): Extra HTTP headers to apply to all requests in this context. + Default: {}. + user_agent (str): Custom User-Agent string to use. Default: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36". + user_agent_mode (str or None): Mode for generating the user agent (e.g., "random"). If None, use the provided + user_agent as-is. Default: None. + user_agent_generator_config (dict or None): Configuration for user agent generation if user_agent_mode is set. + Default: None. + text_mode (bool): If True, disables images and other rich content for potentially faster load times. + Default: False. + light_mode (bool): Disables certain background features for performance gains. Default: False. + extra_args (list): Additional command-line arguments passed to the browser. + Default: []. + enable_stealth (bool): If True, applies playwright-stealth to bypass basic bot detection. + Cannot be used with use_undetected browser mode. Default: False. + memory_saving_mode (bool): If True, adds aggressive cache discard and V8 heap cap flags + to reduce Chromium memory growth. Recommended for high-volume + crawling (1000+ pages). May slightly reduce performance due to + cache eviction. Default: False. + max_pages_before_recycle (int): Number of pages to crawl before recycling the browser + process to reclaim leaked memory. 0 = disabled. + Recommended: 500-1000 for long-running crawlers. + Default: 0. + avoid_ads (bool): If True, blocks ad-related and tracker network requests at the + browser context level using a curated blocklist of top ad/tracker + domains. Default: False. + avoid_css (bool): If True, blocks loading of CSS files (css, less, scss, sass) to + reduce resource usage and speed up crawling. Default: False. + """ + + def __init__( + self, + browser_type: str = "chromium", + headless: bool = True, + browser_mode: str = "dedicated", + use_managed_browser: bool = False, + cdp_url: str = None, + browser_context_id: str = None, + target_id: str = None, + cdp_cleanup_on_close: bool = False, + cdp_close_delay: float = 1.0, + cache_cdp_connection: bool = False, + create_isolated_context: bool = False, + use_persistent_context: bool = False, + user_data_dir: str = None, + chrome_channel: str = "chromium", + channel: str = "chromium", + proxy: str = None, + proxy_config: Union[ProxyConfig, dict, None] = None, + viewport_width: int = 1080, + viewport_height: int = 600, + viewport: dict = None, + device_scale_factor: float = 1.0, + accept_downloads: bool = False, + downloads_path: str = None, + storage_state: Union[str, dict, None] = None, + ignore_https_errors: bool = True, + java_script_enabled: bool = True, + sleep_on_close: bool = False, + verbose: bool = True, + cookies: list = None, + headers: dict = None, + user_agent: str = ( + # "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) AppleWebKit/537.36 " + # "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + # "(KHTML, like Gecko) Chrome/116.0.5845.187 Safari/604.1 Edg/117.0.2045.47" + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36" + ), + user_agent_mode: str = "", + user_agent_generator_config: dict = {}, + text_mode: bool = False, + light_mode: bool = False, + extra_args: list = None, + debugging_port: int = 9222, + host: str = "localhost", + enable_stealth: bool = False, + avoid_ads: bool = False, + avoid_css: bool = False, + init_scripts: List[str] = None, + memory_saving_mode: bool = False, + max_pages_before_recycle: int = 0, + ): + + self.browser_type = browser_type + self.headless = headless + self.browser_mode = browser_mode + self.use_managed_browser = use_managed_browser + self.cdp_url = cdp_url + self.browser_context_id = browser_context_id + self.target_id = target_id + self.cdp_cleanup_on_close = cdp_cleanup_on_close + self.cdp_close_delay = cdp_close_delay + self.cache_cdp_connection = cache_cdp_connection + self.create_isolated_context = create_isolated_context + self.use_persistent_context = use_persistent_context + self.user_data_dir = user_data_dir + self.chrome_channel = chrome_channel or self.browser_type or "chromium" + self.channel = channel or self.browser_type or "chromium" + if self.browser_type in ["firefox", "webkit"]: + self.channel = "" + self.chrome_channel = "" + if proxy: + warnings.warn("The 'proxy' parameter is deprecated and will be removed in a future release. Use 'proxy_config' instead.", UserWarning) + self.proxy = proxy + self.proxy_config = proxy_config + if isinstance(self.proxy_config, dict): + self.proxy_config = ProxyConfig.from_dict(self.proxy_config) + if isinstance(self.proxy_config, str): + self.proxy_config = ProxyConfig.from_string(self.proxy_config) + + if self.proxy and self.proxy_config: + warnings.warn("Both 'proxy' and 'proxy_config' are provided. 'proxy_config' will take precedence.", UserWarning) + self.proxy = None + elif self.proxy: + # Convert proxy string to ProxyConfig if proxy_config is not provided + self.proxy_config = ProxyConfig.from_string(self.proxy) + self.proxy = None + + self.viewport_width = viewport_width + self.viewport_height = viewport_height + self.viewport = viewport + if self.viewport is not None: + self.viewport_width = self.viewport.get("width", 1080) + self.viewport_height = self.viewport.get("height", 600) + self.device_scale_factor = device_scale_factor + self.accept_downloads = accept_downloads + self.downloads_path = downloads_path + self.storage_state = storage_state + self.ignore_https_errors = ignore_https_errors + self.java_script_enabled = java_script_enabled + self.cookies = cookies if cookies is not None else [] + self.headers = headers if headers is not None else {} + self.user_agent = user_agent + self.user_agent_mode = user_agent_mode + self.user_agent_generator_config = user_agent_generator_config + self.text_mode = text_mode + self.light_mode = light_mode + self.extra_args = extra_args if extra_args is not None else [] + self.sleep_on_close = sleep_on_close + self.verbose = verbose + self.debugging_port = debugging_port + self.host = host + self.enable_stealth = enable_stealth + self.avoid_ads = avoid_ads + self.avoid_css = avoid_css + self.init_scripts = init_scripts if init_scripts is not None else [] + self.memory_saving_mode = memory_saving_mode + self.max_pages_before_recycle = max_pages_before_recycle + + fa_user_agenr_generator = ValidUAGenerator() + if self.user_agent_mode == "random": + self.user_agent = fa_user_agenr_generator.generate( + **(self.user_agent_generator_config or {}) + ) + else: + pass + + self.browser_hint = UAGen.generate_client_hints(self.user_agent) + self.headers.setdefault("sec-ch-ua", self.browser_hint) + + # Set appropriate browser management flags based on browser_mode + if self.browser_mode == "builtin": + # Builtin mode uses managed browser connecting to builtin CDP endpoint + self.use_managed_browser = True + # cdp_url will be set later by browser_manager + elif self.browser_mode == "docker": + # Docker mode uses managed browser with CDP to connect to browser in container + self.use_managed_browser = True + # cdp_url will be set later by docker browser strategy + elif self.browser_mode == "custom" and self.cdp_url: + # Custom mode with explicit CDP URL + self.use_managed_browser = True + elif self.browser_mode == "dedicated": + # Dedicated mode uses a new browser instance each time + pass + + # If persistent context is requested, ensure managed browser is enabled + if self.use_persistent_context: + self.use_managed_browser = True + + # Validate stealth configuration + if self.enable_stealth and self.use_managed_browser and self.browser_mode == "builtin": + raise ValueError( + "enable_stealth cannot be used with browser_mode='builtin'. " + "Stealth mode requires a dedicated browser instance." + ) + + @staticmethod + def from_kwargs(kwargs: dict) -> "BrowserConfig": + # Auto-deserialize any dict values that use the {"type": ..., "params": ...} + # serialization format (e.g. from JSON API requests or dump()/load() roundtrips). + kwargs = { + k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v + for k, v in kwargs.items() + } + # Only pass keys present in kwargs so that __init__ defaults (and + # set_defaults() overrides) are respected for missing keys. + valid = inspect.signature(BrowserConfig.__init__).parameters.keys() - {"self"} + return BrowserConfig(**{k: v for k, v in kwargs.items() if k in valid}) + + def to_dict(self): + result = { + "browser_type": self.browser_type, + "headless": self.headless, + "browser_mode": self.browser_mode, + "use_managed_browser": self.use_managed_browser, + "cdp_url": self.cdp_url, + "browser_context_id": self.browser_context_id, + "target_id": self.target_id, + "cdp_cleanup_on_close": self.cdp_cleanup_on_close, + "create_isolated_context": self.create_isolated_context, + "use_persistent_context": self.use_persistent_context, + "user_data_dir": self.user_data_dir, + "chrome_channel": self.chrome_channel, + "channel": self.channel, + "proxy": self.proxy, + "proxy_config": self.proxy_config.to_dict() if hasattr(self.proxy_config, 'to_dict') else self.proxy_config, + "viewport_width": self.viewport_width, + "viewport_height": self.viewport_height, + "device_scale_factor": self.device_scale_factor, + "accept_downloads": self.accept_downloads, + "downloads_path": self.downloads_path, + "storage_state": self.storage_state, + "ignore_https_errors": self.ignore_https_errors, + "java_script_enabled": self.java_script_enabled, + "cookies": self.cookies, + "headers": self.headers, + "user_agent": self.user_agent, + "user_agent_mode": self.user_agent_mode, + "user_agent_generator_config": self.user_agent_generator_config, + "text_mode": self.text_mode, + "light_mode": self.light_mode, + "extra_args": self.extra_args, + "sleep_on_close": self.sleep_on_close, + "verbose": self.verbose, + "debugging_port": self.debugging_port, + "host": self.host, + "enable_stealth": self.enable_stealth, + "avoid_ads": self.avoid_ads, + "avoid_css": self.avoid_css, + "init_scripts": self.init_scripts, + "memory_saving_mode": self.memory_saving_mode, + "max_pages_before_recycle": self.max_pages_before_recycle, + } + + + return result + + def clone(self, **kwargs): + """Create a copy of this configuration with updated values. + + Args: + **kwargs: Key-value pairs of configuration options to update + + Returns: + BrowserConfig: A new instance with the specified updates + """ + config_dict = self.to_dict() + config_dict.update(kwargs) + return BrowserConfig.from_kwargs(config_dict) + + # Create a funciton returns dict of the object + def dump(self) -> dict: + # Serialize the object to a dictionary + return to_serializable_dict(self) + + @staticmethod + def load(data: dict, provenance: "Provenance" = None) -> "BrowserConfig": + # Deserialize the object from a dictionary + if provenance is None: + provenance = Provenance.TRUSTED + config = from_serializable_dict(data, provenance) + if isinstance(config, BrowserConfig): + return config + # Plain-dict path: enforce the untrusted gate on top-level fields too, + # so a body that sends bare kwargs (no {type,params}) cannot bypass it. + if provenance == Provenance.UNTRUSTED and isinstance(config, dict): + config = _enforce_untrusted("BrowserConfig", config) + return BrowserConfig.from_kwargs(config) + + def set_nstproxy( + self, + token: str, + channel_id: str, + country: str = "ANY", + state: str = "", + city: str = "", + protocol: str = "http", + session_duration: int = 10, + ): + """ + Fetch a proxy from NSTProxy API and automatically assign it to proxy_config. + + Get your NSTProxy token from: https://app.nstproxy.com/profile + + Args: + token (str): NSTProxy API token. + channel_id (str): NSTProxy channel ID. + country (str, optional): Country code (default: "ANY"). + state (str, optional): State code (default: ""). + city (str, optional): City name (default: ""). + protocol (str, optional): Proxy protocol ("http" or "socks5"). Defaults to "http". + session_duration (int, optional): Session duration in minutes (0 = rotate each request). Defaults to 10. + + Raises: + ValueError: If the API response format is invalid. + PermissionError: If the API returns an error message. + """ + + # --- Validate input early --- + if not token or not channel_id: + raise ValueError("[NSTProxy] token and channel_id are required") + + if protocol not in ("http", "socks5"): + raise ValueError(f"[NSTProxy] Invalid protocol: {protocol}") + + # --- Build NSTProxy API URL --- + params = { + "fType": 2, + "count": 1, + "channelId": channel_id, + "country": country, + "protocol": protocol, + "sessionDuration": session_duration, + "token": token, + } + if state: + params["state"] = state + if city: + params["city"] = city + + url = "https://api.nstproxy.com/api/v1/generate/apiproxies" + + try: + response = requests.get(url, params=params, timeout=10) + response.raise_for_status() + + data = response.json() + + # --- Handle API error response --- + if isinstance(data, dict) and data.get("err"): + raise PermissionError(f"[NSTProxy] API Error: {data.get('msg', 'Unknown error')}") + + if not isinstance(data, list) or not data: + raise ValueError("[NSTProxy] Invalid API response — expected a non-empty list") + + proxy_info = data[0] + + # --- Apply proxy config --- + self.proxy_config = ProxyConfig( + server=f"{protocol}://{proxy_info['ip']}:{proxy_info['port']}", + username=proxy_info["username"], + password=proxy_info["password"], + ) + + except Exception as e: + print(f"[NSTProxy] ❌ Failed to set proxy: {e}") + raise + +class VirtualScrollConfig: + """Configuration for virtual scroll handling. + + This config enables capturing content from pages with virtualized scrolling + (like Twitter, Instagram feeds) where DOM elements are recycled as user scrolls. + """ + + def __init__( + self, + container_selector: str, + scroll_count: int = 10, + scroll_by: Union[str, int] = "container_height", + wait_after_scroll: float = 0.5, + ): + """ + Initialize virtual scroll configuration. + + Args: + container_selector: CSS selector for the scrollable container + scroll_count: Maximum number of scrolls to perform + scroll_by: Amount to scroll - can be: + - "container_height": scroll by container's height + - "page_height": scroll by viewport height + - int: fixed pixel amount + wait_after_scroll: Seconds to wait after each scroll for content to load + """ + self.container_selector = container_selector + self.scroll_count = scroll_count + self.scroll_by = scroll_by + self.wait_after_scroll = wait_after_scroll + + def to_dict(self) -> dict: + """Convert to dictionary for serialization.""" + return { + "container_selector": self.container_selector, + "scroll_count": self.scroll_count, + "scroll_by": self.scroll_by, + "wait_after_scroll": self.wait_after_scroll, + } + + @classmethod + def from_dict(cls, data: dict) -> "VirtualScrollConfig": + """Create instance from dictionary.""" + return cls(**data) + +class LinkPreviewConfig: + """Configuration for link head extraction and scoring.""" + + def __init__( + self, + include_internal: bool = True, + include_external: bool = False, + include_patterns: Optional[List[str]] = None, + exclude_patterns: Optional[List[str]] = None, + concurrency: int = 10, + timeout: int = 5, + max_links: int = 100, + query: Optional[str] = None, + score_threshold: Optional[float] = None, + verbose: bool = False + ): + """ + Initialize link extraction configuration. + + Args: + include_internal: Whether to include same-domain links + include_external: Whether to include different-domain links + include_patterns: List of glob patterns to include (e.g., ["*/docs/*", "*/api/*"]) + exclude_patterns: List of glob patterns to exclude (e.g., ["*/login*", "*/admin*"]) + concurrency: Number of links to process simultaneously + timeout: Timeout in seconds for each link's head extraction + max_links: Maximum number of links to process (prevents overload) + query: Query string for BM25 contextual scoring (optional) + score_threshold: Minimum relevance score to include links (0.0-1.0, optional) + verbose: Show detailed progress during extraction + """ + self.include_internal = include_internal + self.include_external = include_external + self.include_patterns = include_patterns + self.exclude_patterns = exclude_patterns + self.concurrency = concurrency + self.timeout = timeout + self.max_links = max_links + self.query = query + self.score_threshold = score_threshold + self.verbose = verbose + + # Validation + if concurrency <= 0: + raise ValueError("concurrency must be positive") + if timeout <= 0: + raise ValueError("timeout must be positive") + if max_links <= 0: + raise ValueError("max_links must be positive") + if score_threshold is not None and not (0.0 <= score_threshold <= 1.0): + raise ValueError("score_threshold must be between 0.0 and 1.0") + if not include_internal and not include_external: + raise ValueError("At least one of include_internal or include_external must be True") + + @staticmethod + def from_dict(config_dict: Dict[str, Any]) -> "LinkPreviewConfig": + """Create LinkPreviewConfig from dictionary (for backward compatibility).""" + if not config_dict: + return None + + return LinkPreviewConfig( + include_internal=config_dict.get("include_internal", True), + include_external=config_dict.get("include_external", False), + include_patterns=config_dict.get("include_patterns"), + exclude_patterns=config_dict.get("exclude_patterns"), + concurrency=config_dict.get("concurrency", 10), + timeout=config_dict.get("timeout", 5), + max_links=config_dict.get("max_links", 100), + query=config_dict.get("query"), + score_threshold=config_dict.get("score_threshold"), + verbose=config_dict.get("verbose", False) + ) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary format.""" + return { + "include_internal": self.include_internal, + "include_external": self.include_external, + "include_patterns": self.include_patterns, + "exclude_patterns": self.exclude_patterns, + "concurrency": self.concurrency, + "timeout": self.timeout, + "max_links": self.max_links, + "query": self.query, + "score_threshold": self.score_threshold, + "verbose": self.verbose + } + + def clone(self, **kwargs) -> "LinkPreviewConfig": + """Create a copy with updated values.""" + config_dict = self.to_dict() + config_dict.update(kwargs) + return LinkPreviewConfig.from_dict(config_dict) + + +class HTTPCrawlerConfig: + """HTTP-specific crawler configuration""" + + method: str = "GET" + headers: Optional[Dict[str, str]] = None + data: Optional[Dict[str, Any]] = None + json: Optional[Dict[str, Any]] = None + follow_redirects: bool = True + verify_ssl: bool = True + downloads_path: Optional[str] = None + + def __init__( + self, + method: str = "GET", + headers: Optional[Dict[str, str]] = None, + data: Optional[Dict[str, Any]] = None, + json: Optional[Dict[str, Any]] = None, + follow_redirects: bool = True, + verify_ssl: bool = True, + downloads_path: Optional[str] = None, + ): + self.method = method + self.headers = headers + self.data = data + self.json = json + self.follow_redirects = follow_redirects + self.verify_ssl = verify_ssl + self.downloads_path = downloads_path + + @staticmethod + def from_kwargs(kwargs: dict) -> "HTTPCrawlerConfig": + return HTTPCrawlerConfig( + method=kwargs.get("method", "GET"), + headers=kwargs.get("headers"), + data=kwargs.get("data"), + json=kwargs.get("json"), + follow_redirects=kwargs.get("follow_redirects", True), + verify_ssl=kwargs.get("verify_ssl", True), + downloads_path=kwargs.get("downloads_path"), + ) + + def to_dict(self): + return { + "method": self.method, + "headers": self.headers, + "data": self.data, + "json": self.json, + "follow_redirects": self.follow_redirects, + "verify_ssl": self.verify_ssl, + "downloads_path": self.downloads_path, + } + + def clone(self, **kwargs): + """Create a copy of this configuration with updated values. + + Args: + **kwargs: Key-value pairs of configuration options to update + + Returns: + HTTPCrawlerConfig: A new instance with the specified updates + """ + config_dict = self.to_dict() + config_dict.update(kwargs) + return HTTPCrawlerConfig.from_kwargs(config_dict) + + def dump(self) -> dict: + return to_serializable_dict(self) + + @staticmethod + def load(data: dict, provenance: "Provenance" = None) -> "HTTPCrawlerConfig": + if provenance is None: + provenance = Provenance.TRUSTED + config = from_serializable_dict(data, provenance) + if isinstance(config, HTTPCrawlerConfig): + return config + if provenance == Provenance.UNTRUSTED and isinstance(config, dict): + config = _enforce_untrusted("HTTPCrawlerConfig", config) + return HTTPCrawlerConfig.from_kwargs(config) + +@_with_defaults +class CrawlerRunConfig(): + + """ + Configuration class for controlling how the crawler runs each crawl operation. + This includes parameters for content extraction, page manipulation, waiting conditions, + caching, and other runtime behaviors. + + This centralizes parameters that were previously scattered as kwargs to `arun()` and related methods. + By using this class, you have a single place to understand and adjust the crawling options. + + Attributes: + # Deep Crawl Parameters + deep_crawl_strategy (DeepCrawlStrategy or None): Strategy to use for deep crawling. + + # Content Processing Parameters + word_count_threshold (int): Minimum word count threshold before processing content. + Default: MIN_WORD_THRESHOLD (typically 200). + extraction_strategy (ExtractionStrategy or None): Strategy to extract structured data from crawled pages. + Default: None (NoExtractionStrategy is used if None). + chunking_strategy (ChunkingStrategy): Strategy to chunk content before extraction. + Default: RegexChunking(). + markdown_generator (MarkdownGenerationStrategy): Strategy for generating markdown. + Default: None. + only_text (bool): If True, attempt to extract text-only content where applicable. + Default: False. + css_selector (str or None): CSS selector to extract a specific portion of the page. + Default: None. + + target_elements (list of str or None): List of CSS selectors for specific elements for Markdown generation + and structured data extraction. When you set this, only the contents + of these elements are processed for extraction and Markdown generation. + If you do not set any value, the entire page is processed. + The difference between this and css_selector is that this will shrink + the initial raw HTML to the selected element, while this will only affect + the extraction and Markdown generation. + Default: None + excluded_tags (list of str or None): List of HTML tags to exclude from processing. + Default: None. + excluded_selector (str or None): CSS selector to exclude from processing. + Default: None. + keep_data_attributes (bool): If True, retain `data-*` attributes while removing unwanted attributes. + Default: False. + keep_attrs (list of str): List of HTML attributes to keep during processing. + Default: []. + remove_forms (bool): If True, remove all `<form>` elements from the HTML. + Default: False. + prettiify (bool): If True, apply `fast_format_html` to produce prettified HTML output. + Default: False. + parser_type (str): Type of parser to use for HTML parsing. + Default: "lxml". + scraping_strategy (ContentScrapingStrategy): Scraping strategy to use. + Default: LXMLWebScrapingStrategy. + proxy_config (ProxyConfig or dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}. + If None, no additional proxy config. Default: None. + + # Sticky Proxy Session Parameters + proxy_session_id (str or None): When set, maintains the same proxy for all requests sharing this session ID. + The proxy is acquired on first request and reused for subsequent requests. + Session expires when explicitly released or crawler context is closed. + Default: None. + proxy_session_ttl (int or None): Time-to-live for sticky session in seconds. + After TTL expires, a new proxy is acquired on next request. + Default: None (session lasts until explicitly released or crawler closes). + proxy_session_auto_release (bool): If True, automatically release the proxy session after a batch operation. + Useful for arun_many() to clean up sessions automatically. + Default: False. + + # Browser Location and Identity Parameters + locale (str or None): Locale to use for the browser context (e.g., "en-US"). + Default: None. + timezone_id (str or None): Timezone identifier to use for the browser context (e.g., "America/New_York"). + Default: None. + geolocation (GeolocationConfig or None): Geolocation configuration for the browser. + Default: None. + + # SSL Parameters + fetch_ssl_certificate: bool = False, + # Caching Parameters + cache_mode (CacheMode or None): Defines how caching is handled. + If None, defaults to CacheMode.ENABLED internally. + Default: CacheMode.BYPASS. + session_id (str or None): Optional session ID to persist the browser context and the created + page instance. If the ID already exists, the crawler does not + create a new page and uses the current page to preserve the state. + bypass_cache (bool): Legacy parameter, if True acts like CacheMode.BYPASS. + Default: False. + disable_cache (bool): Legacy parameter, if True acts like CacheMode.DISABLED. + Default: False. + no_cache_read (bool): Legacy parameter, if True acts like CacheMode.WRITE_ONLY. + Default: False. + no_cache_write (bool): Legacy parameter, if True acts like CacheMode.READ_ONLY. + Default: False. + shared_data (dict or None): Shared data to be passed between hooks. + Default: None. + + # Cache Validation Parameters (Smart Cache) + check_cache_freshness (bool): If True, validates cached content freshness using HTTP + conditional requests (ETag/Last-Modified) and head fingerprinting + before returning cached results. Avoids full browser crawls when + content hasn't changed. Only applies when cache_mode allows reads. + Default: False. + cache_validation_timeout (float): Timeout in seconds for cache validation HTTP requests. + Default: 10.0. + + # Page Navigation and Timing Parameters + wait_until (str): The condition to wait for when navigating, e.g. "domcontentloaded". + Default: "domcontentloaded". + page_timeout (int): Timeout in ms for page operations like navigation. + Default: 60000 (60 seconds). + wait_for (str or None): A CSS selector or JS condition to wait for before extracting content. + Default: None. + wait_for_timeout (int or None): Specific timeout in ms for the wait_for condition. + If None, uses page_timeout instead. + Default: None. + wait_for_images (bool): If True, wait for images to load before extracting content. + Default: False. + delay_before_return_html (float): Delay in seconds before retrieving final HTML. + Default: 0.1. + mean_delay (float): Mean base delay between requests when calling arun_many. + Default: 0.1. + max_range (float): Max random additional delay range for requests in arun_many. + Default: 0.3. + semaphore_count (int): Number of concurrent operations allowed. + Default: 5. + + # Page Interaction Parameters + js_code (str or list of str or None): JavaScript code/snippets to run on the page + after wait_for and delay_before_return_html. + Default: None. + js_code_before_wait (str or list of str or None): JavaScript to run BEFORE wait_for. + Use for triggering loading that wait_for then checks. + Default: None. + js_only (bool): If True, indicates subsequent calls are JS-driven updates, not full page loads. + Default: False. + ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding. + Default: True. + scan_full_page (bool): If True, scroll through the entire page to load all content. + Default: False. + scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True. + Default: 0.2. + max_scroll_steps (Optional[int]): Maximum number of scroll steps to perform during full page scan. + If None, scrolls until the entire page is loaded. Default: None. + process_iframes (bool): If True, attempts to process and inline iframe content. + Default: False. + flatten_shadow_dom (bool): If True, flatten shadow DOM content into the light DOM + before HTML capture so page.content() includes it. + Also injects an init script to force-open closed shadow roots. + Default: False. + remove_overlay_elements (bool): If True, remove overlays/popups before extracting HTML. + Default: False. + remove_consent_popups (bool): If True, remove GDPR/cookie consent popups (IAB TCF/CMP) + before extracting HTML. Targets known CMP providers like + OneTrust, Cookiebot, TrustArc, Quantcast, Didomi, etc. + Default: False. + simulate_user (bool): If True, simulate user interactions (mouse moves, clicks) for anti-bot measures. + Default: False. + override_navigator (bool): If True, overrides navigator properties for more human-like behavior. + Default: False. + magic (bool): If True, attempts automatic handling of overlays/popups. + Default: False. + adjust_viewport_to_content (bool): If True, adjust viewport according to the page content dimensions. + Default: False. + + # Media Handling Parameters + screenshot (bool): Whether to take a screenshot after crawling. + Default: False. + screenshot_wait_for (float or None): Additional wait time before taking a screenshot. + Default: None. + screenshot_height_threshold (int): Threshold for page height to decide screenshot strategy. + Default: SCREENSHOT_HEIGHT_TRESHOLD (from config, e.g. 20000). + force_viewport_screenshot (bool): If True, always take viewport-only screenshots regardless of page height. + When False, uses automatic decision (viewport for short pages, full-page for long pages). + Default: False. + pdf (bool): Whether to generate a PDF of the page. + Default: False. + image_description_min_word_threshold (int): Minimum words for image description extraction. + Default: IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD (e.g., 50). + image_score_threshold (int): Minimum score threshold for processing an image. + Default: IMAGE_SCORE_THRESHOLD (e.g., 3). + exclude_external_images (bool): If True, exclude all external images from processing. + Default: False. + table_score_threshold (int): Minimum score threshold for processing a table. + Default: 7. + table_extraction (TableExtractionStrategy): Strategy to use for table extraction. + Default: DefaultTableExtraction with table_score_threshold. + + # Virtual Scroll Parameters + virtual_scroll_config (VirtualScrollConfig or dict or None): Configuration for handling virtual scroll containers. + Used for capturing content from pages with virtualized + scrolling (e.g., Twitter, Instagram feeds). + Default: None. + + # Link and Domain Handling Parameters + exclude_social_media_domains (list of str): List of domains to exclude for social media links. + Default: SOCIAL_MEDIA_DOMAINS (from config). + exclude_external_links (bool): If True, exclude all external links from the results. + Default: False. + exclude_internal_links (bool): If True, exclude internal links from the results. + Default: False. + exclude_social_media_links (bool): If True, exclude links pointing to social media domains. + Default: False. + exclude_domains (list of str): List of specific domains to exclude from results. + Default: []. + exclude_internal_links (bool): If True, exclude internal links from the results. + Default: False. + score_links (bool): If True, calculate intrinsic quality scores for all links using URL structure, + text quality, and contextual relevance metrics. Separate from link_preview_config. + Default: False. + + # Debugging and Logging Parameters + verbose (bool): Enable verbose logging. + Default: True. + log_console (bool): If True, log console messages from the page. + Default: False. + + # HTTP Crwler Strategy Parameters + method (str): HTTP method to use for the request, when using AsyncHTTPCrwalerStrategy. + Default: "GET". + data (dict): Data to send in the request body, when using AsyncHTTPCrwalerStrategy. + Default: None. + json (dict): JSON data to send in the request body, when using AsyncHTTPCrwalerStrategy. + + # Connection Parameters + stream (bool): If True, enables streaming of crawled URLs as they are processed when used with arun_many. + Default: False. + process_in_browser (bool): If True, forces raw:/file:// URLs to be processed through the browser + pipeline (enabling js_code, wait_for, scrolling, etc.). When False (default), + raw:/file:// URLs use a fast path that returns HTML directly without browser + interaction. This is automatically enabled when browser-requiring parameters + are detected (js_code, wait_for, screenshot, pdf, etc.). + Default: False. + + check_robots_txt (bool): Whether to check robots.txt rules before crawling. Default: False + Default: False. + user_agent (str): Custom User-Agent string to use. + Default: None. + user_agent_mode (str or None): Mode for generating the user agent (e.g., "random"). If None, use the provided user_agent as-is. + Default: None. + user_agent_generator_config (dict or None): Configuration for user agent generation if user_agent_mode is set. + Default: None. + + # Experimental Parameters + experimental (dict): Dictionary containing experimental parameters that are in beta phase. + This allows passing temporary features that are not yet fully integrated + into the main parameter set. + Default: None. + + url: str = None # This is not a compulsory parameter + """ + _UNWANTED_PROPS = { + 'disable_cache' : 'Instead, use cache_mode=CacheMode.DISABLED', + 'bypass_cache' : 'Instead, use cache_mode=CacheMode.BYPASS', + 'no_cache_read' : 'Instead, use cache_mode=CacheMode.WRITE_ONLY', + 'no_cache_write' : 'Instead, use cache_mode=CacheMode.READ_ONLY', + } + + def __init__( + self, + # Content Processing Parameters + word_count_threshold: int = MIN_WORD_THRESHOLD, + extraction_strategy: ExtractionStrategy = None, + chunking_strategy: ChunkingStrategy = RegexChunking(), + markdown_generator: MarkdownGenerationStrategy = DefaultMarkdownGenerator(), + only_text: bool = False, + css_selector: str = None, + target_elements: List[str] = None, + excluded_tags: list = None, + excluded_selector: str = None, + keep_data_attributes: bool = False, + keep_attrs: list = None, + remove_forms: bool = False, + prettiify: bool = False, + parser_type: str = "lxml", + scraping_strategy: ContentScrapingStrategy = None, + proxy_config: Union["ProxyConfig", List["ProxyConfig"], dict, str, None] = None, + proxy_rotation_strategy: Optional[ProxyRotationStrategy] = None, + # Sticky Proxy Session Parameters + proxy_session_id: Optional[str] = None, + proxy_session_ttl: Optional[int] = None, + proxy_session_auto_release: bool = False, + # Browser Location and Identity Parameters + locale: Optional[str] = None, + timezone_id: Optional[str] = None, + geolocation: Optional[GeolocationConfig] = None, + # SSL Parameters + fetch_ssl_certificate: bool = False, + # Caching Parameters + cache_mode: CacheMode = CacheMode.BYPASS, + session_id: str = None, + bypass_cache: bool = False, + disable_cache: bool = False, + no_cache_read: bool = False, + no_cache_write: bool = False, + shared_data: dict = None, + # Cache Validation Parameters (Smart Cache) + check_cache_freshness: bool = False, + cache_validation_timeout: float = 10.0, + # Page Navigation and Timing Parameters + wait_until: str = "domcontentloaded", + page_timeout: int = PAGE_TIMEOUT, + wait_for: str = None, + wait_for_timeout: int = None, + wait_for_images: bool = False, + delay_before_return_html: float = 0.1, + mean_delay: float = 0.1, + max_range: float = 0.3, + semaphore_count: int = 5, + # Page Interaction Parameters + js_code: Union[str, List[str]] = None, + js_code_before_wait: Union[str, List[str]] = None, + c4a_script: Union[str, List[str]] = None, + js_only: bool = False, + ignore_body_visibility: bool = True, + scan_full_page: bool = False, + scroll_delay: float = 0.2, + max_scroll_steps: Optional[int] = None, + process_iframes: bool = False, + flatten_shadow_dom: bool = False, + remove_overlay_elements: bool = False, + remove_consent_popups: bool = False, + simulate_user: bool = False, + override_navigator: bool = False, + magic: bool = False, + adjust_viewport_to_content: bool = False, + # Media Handling Parameters + screenshot: bool = False, + screenshot_wait_for: float = None, + screenshot_height_threshold: int = SCREENSHOT_HEIGHT_TRESHOLD, + force_viewport_screenshot: bool = False, + pdf: bool = False, + capture_mhtml: bool = False, + image_description_min_word_threshold: int = IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, + image_score_threshold: int = IMAGE_SCORE_THRESHOLD, + table_score_threshold: int = 7, + table_extraction: TableExtractionStrategy = None, + exclude_external_images: bool = False, + exclude_all_images: bool = False, + # Link and Domain Handling Parameters + exclude_social_media_domains: list = None, + exclude_external_links: bool = False, + exclude_social_media_links: bool = False, + exclude_domains: list = None, + exclude_internal_links: bool = False, + score_links: bool = False, + preserve_https_for_internal_links: bool = False, + # Debugging and Logging Parameters + verbose: bool = True, + log_console: bool = False, + # Network and Console Capturing Parameters + capture_network_requests: bool = False, + capture_console_messages: bool = False, + # Connection Parameters + method: str = "GET", + stream: bool = False, + prefetch: bool = False, # When True, return only HTML + links (skip heavy processing) + process_in_browser: bool = False, # Force browser processing for raw:/file:// URLs + url: str = None, + base_url: str = None, # Base URL for markdown link resolution (used with raw: HTML) + check_robots_txt: bool = False, + user_agent: str = None, + user_agent_mode: str = None, + user_agent_generator_config: dict = {}, + # Deep Crawl Parameters + deep_crawl_strategy: Optional[DeepCrawlStrategy] = None, + # Link Extraction Parameters + link_preview_config: Union[LinkPreviewConfig, Dict[str, Any]] = None, + # Virtual Scroll Parameters + virtual_scroll_config: Union[VirtualScrollConfig, Dict[str, Any]] = None, + # URL Matching Parameters + url_matcher: Optional[UrlMatcher] = None, + match_mode: MatchMode = MatchMode.OR, + # Experimental Parameters + experimental: Dict[str, Any] = None, + # Anti-Bot Retry Parameters + max_retries: int = 0, + fallback_fetch_function: Optional[Callable[[str], Awaitable[str]]] = None, + ): + # TODO: Planning to set properties dynamically based on the __init__ signature + self.url = url + self.base_url = base_url # Base URL for markdown link resolution + + # Content Processing Parameters + self.word_count_threshold = word_count_threshold + self.extraction_strategy = extraction_strategy + self.chunking_strategy = chunking_strategy + self.markdown_generator = markdown_generator + self.only_text = only_text + self.css_selector = css_selector + self.target_elements = target_elements or [] + self.excluded_tags = excluded_tags or [] + self.excluded_selector = excluded_selector or "" + self.keep_data_attributes = keep_data_attributes + self.keep_attrs = keep_attrs or [] + self.remove_forms = remove_forms + self.prettiify = prettiify + self.parser_type = parser_type + self.scraping_strategy = scraping_strategy or LXMLWebScrapingStrategy() + self.proxy_config = proxy_config # runs through property setter + + self.proxy_rotation_strategy = proxy_rotation_strategy + + # Sticky Proxy Session Parameters + self.proxy_session_id = proxy_session_id + self.proxy_session_ttl = proxy_session_ttl + self.proxy_session_auto_release = proxy_session_auto_release + + # Browser Location and Identity Parameters + self.locale = locale + self.timezone_id = timezone_id + self.geolocation = geolocation + + # SSL Parameters + self.fetch_ssl_certificate = fetch_ssl_certificate + + # Caching Parameters + self.cache_mode = cache_mode + self.session_id = session_id + self.bypass_cache = bypass_cache + self.disable_cache = disable_cache + self.no_cache_read = no_cache_read + self.no_cache_write = no_cache_write + self.shared_data = shared_data + # Cache Validation (Smart Cache) + self.check_cache_freshness = check_cache_freshness + self.cache_validation_timeout = cache_validation_timeout + + # Page Navigation and Timing Parameters + self.wait_until = wait_until + self.page_timeout = page_timeout + self.wait_for = wait_for + self.wait_for_timeout = wait_for_timeout + self.wait_for_images = wait_for_images + self.delay_before_return_html = delay_before_return_html + self.mean_delay = mean_delay + self.max_range = max_range + self.semaphore_count = semaphore_count + + # Page Interaction Parameters + self.js_code = js_code + self.js_code_before_wait = js_code_before_wait + self.c4a_script = c4a_script + self.js_only = js_only + self.ignore_body_visibility = ignore_body_visibility + self.scan_full_page = scan_full_page + self.scroll_delay = scroll_delay + self.max_scroll_steps = max_scroll_steps + self.process_iframes = process_iframes + self.flatten_shadow_dom = flatten_shadow_dom + self.remove_overlay_elements = remove_overlay_elements + self.remove_consent_popups = remove_consent_popups + self.simulate_user = simulate_user + self.override_navigator = override_navigator + self.magic = magic + self.adjust_viewport_to_content = adjust_viewport_to_content + + # Media Handling Parameters + self.screenshot = screenshot + self.screenshot_wait_for = screenshot_wait_for + self.screenshot_height_threshold = screenshot_height_threshold + self.force_viewport_screenshot = force_viewport_screenshot + self.pdf = pdf + self.capture_mhtml = capture_mhtml + self.image_description_min_word_threshold = image_description_min_word_threshold + self.image_score_threshold = image_score_threshold + self.exclude_external_images = exclude_external_images + self.exclude_all_images = exclude_all_images + self.table_score_threshold = table_score_threshold + + # Table extraction strategy (default to DefaultTableExtraction if not specified) + if table_extraction is None: + self.table_extraction = DefaultTableExtraction(table_score_threshold=table_score_threshold) + else: + self.table_extraction = table_extraction + + # Link and Domain Handling Parameters + self.exclude_social_media_domains = ( + exclude_social_media_domains or SOCIAL_MEDIA_DOMAINS + ) + self.exclude_external_links = exclude_external_links + self.exclude_social_media_links = exclude_social_media_links + self.exclude_domains = exclude_domains or [] + self.exclude_internal_links = exclude_internal_links + self.score_links = score_links + self.preserve_https_for_internal_links = preserve_https_for_internal_links + + # Debugging and Logging Parameters + self.verbose = verbose + self.log_console = log_console + + # Network and Console Capturing Parameters + self.capture_network_requests = capture_network_requests + self.capture_console_messages = capture_console_messages + + # Connection Parameters + self.stream = stream + self.prefetch = prefetch # Prefetch mode: return only HTML + links + self.process_in_browser = process_in_browser # Force browser processing for raw:/file:// URLs + self.method = method + + # Robots.txt Handling Parameters + self.check_robots_txt = check_robots_txt + + # User Agent Parameters + self.user_agent = user_agent + self.user_agent_mode = user_agent_mode + self.user_agent_generator_config = user_agent_generator_config + + # Validate type of extraction strategy and chunking strategy if they are provided + if self.extraction_strategy is not None and not isinstance( + self.extraction_strategy, ExtractionStrategy + ): + raise ValueError( + "extraction_strategy must be an instance of ExtractionStrategy" + ) + if self.chunking_strategy is not None and not isinstance( + self.chunking_strategy, ChunkingStrategy + ): + raise ValueError( + "chunking_strategy must be an instance of ChunkingStrategy" + ) + if self.markdown_generator is not None and not isinstance( + self.markdown_generator, MarkdownGenerationStrategy + ): + hint = "" + if isinstance(self.markdown_generator, dict): + hint = ( + ' The JSON format must be {"type": "<ClassName>", "params": {...}}.' + ' Note: "params" is required — "options" or other keys are not recognized.' + ) + raise ValueError( + "markdown_generator must be an instance of MarkdownGenerationStrategy, " + f"got {type(self.markdown_generator).__name__}.{hint}" + ) + + # Set default chunking strategy if None + if self.chunking_strategy is None: + self.chunking_strategy = RegexChunking() + + # Deep Crawl Parameters + self.deep_crawl_strategy = deep_crawl_strategy + + # Link Extraction Parameters + if link_preview_config is None: + self.link_preview_config = None + elif isinstance(link_preview_config, LinkPreviewConfig): + self.link_preview_config = link_preview_config + elif isinstance(link_preview_config, dict): + # Convert dict to config object for backward compatibility + self.link_preview_config = LinkPreviewConfig.from_dict(link_preview_config) + else: + raise ValueError("link_preview_config must be LinkPreviewConfig object or dict") + + # Virtual Scroll Parameters + if virtual_scroll_config is None: + self.virtual_scroll_config = None + elif isinstance(virtual_scroll_config, VirtualScrollConfig): + self.virtual_scroll_config = virtual_scroll_config + elif isinstance(virtual_scroll_config, dict): + # Convert dict to config object for backward compatibility + self.virtual_scroll_config = VirtualScrollConfig.from_dict(virtual_scroll_config) + else: + raise ValueError("virtual_scroll_config must be VirtualScrollConfig object or dict") + + # URL Matching Parameters + self.url_matcher = url_matcher + self.match_mode = match_mode + + # Experimental Parameters + self.experimental = experimental or {} + + # Anti-Bot Retry Parameters + self.max_retries = max_retries + self.fallback_fetch_function = fallback_fetch_function + + # Compile C4A scripts if provided + if self.c4a_script and not self.js_code: + self._compile_c4a_script() + + + @staticmethod + def _normalize_proxy_config(value): + """Normalize proxy_config to ProxyConfig, list of ProxyConfig/None, or None.""" + if isinstance(value, list): + normalized = [] + for p in value: + if p is None or p == "direct": + normalized.append(None) + elif isinstance(p, dict): + normalized.append(ProxyConfig.from_dict(p)) + elif isinstance(p, str): + normalized.append(ProxyConfig.from_string(p)) + else: + normalized.append(p) + return normalized + elif isinstance(value, dict): + return ProxyConfig.from_dict(value) + elif isinstance(value, str): + if value == "direct": + return None + return ProxyConfig.from_string(value) + return value # ProxyConfig or None + + @property + def proxy_config(self): + return self._proxy_config + + @proxy_config.setter + def proxy_config(self, value): + self._proxy_config = CrawlerRunConfig._normalize_proxy_config(value) + + def _get_proxy_list(self) -> list: + """Normalize proxy_config to a list for the retry loop.""" + if self.proxy_config is None: + return [None] + if isinstance(self.proxy_config, list): + return self.proxy_config if self.proxy_config else [None] + return [self.proxy_config] + + def _compile_c4a_script(self): + """Compile C4A script to JavaScript""" + try: + # Try importing the compiler + try: + from .script import compile + except ImportError: + from crawl4ai.script import compile + + # Handle both string and list inputs + if isinstance(self.c4a_script, str): + scripts = [self.c4a_script] + else: + scripts = self.c4a_script + + # Compile each script + compiled_js = [] + for i, script in enumerate(scripts): + result = compile(script) + + if result.success: + compiled_js.extend(result.js_code) + else: + # Format error message following existing patterns + error = result.first_error + error_msg = ( + f"C4A Script compilation error (script {i+1}):\n" + f" Line {error.line}, Column {error.column}: {error.message}\n" + f" Code: {error.source_line}" + ) + if error.suggestions: + error_msg += f"\n Suggestion: {error.suggestions[0].message}" + + raise ValueError(error_msg) + + self.js_code = compiled_js + + except ImportError: + raise ValueError( + "C4A script compiler not available. " + "Please ensure crawl4ai.script module is properly installed." + ) + except Exception as e: + # Re-raise with context + if "compilation error" not in str(e).lower(): + raise ValueError(f"Failed to compile C4A script: {str(e)}") + raise + + def is_match(self, url: str) -> bool: + """Check if this config matches the given URL. + + Args: + url: The URL to check against this config's matcher + + Returns: + bool: True if this config should be used for the URL or if no matcher is set. + """ + if self.url_matcher is None: + return True + + if callable(self.url_matcher): + # Single function matcher + return self.url_matcher(url) + + elif isinstance(self.url_matcher, str): + # Single pattern string + from fnmatch import fnmatch + return fnmatch(url, self.url_matcher) + + elif isinstance(self.url_matcher, list): + # List of mixed matchers + if not self.url_matcher: # Empty list + return False + + results = [] + for matcher in self.url_matcher: + if callable(matcher): + results.append(matcher(url)) + elif isinstance(matcher, str): + from fnmatch import fnmatch + results.append(fnmatch(url, matcher)) + else: + # Skip invalid matchers + continue + + # Apply match mode logic + if self.match_mode == MatchMode.OR: + return any(results) if results else False + else: # AND mode + return all(results) if results else False + + return False + + + def __getattr__(self, name): + """Handle attribute access.""" + if name in self._UNWANTED_PROPS: + raise AttributeError(f"Getting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}") + raise AttributeError(f"'{self.__class__.__name__}' has no attribute '{name}'") + + def __setattr__(self, name, value): + """Handle attribute setting.""" + # TODO: Planning to set properties dynamically based on the __init__ signature + sig = inspect.signature(self.__init__) + all_params = sig.parameters # Dictionary of parameter names and their details + + if name in self._UNWANTED_PROPS and value is not all_params[name].default: + raise AttributeError(f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}") + + super().__setattr__(name, value) + + @staticmethod + def from_kwargs(kwargs: dict) -> "CrawlerRunConfig": + # Auto-deserialize any dict values that use the {"type": ..., "params": ...} + # serialization format (e.g. from JSON API requests or dump()/load() roundtrips). + # This covers markdown_generator, extraction_strategy, content_filter, etc. + kwargs = { + k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v + for k, v in kwargs.items() + } + # Only pass keys present in kwargs so that __init__ defaults (and + # set_defaults() overrides) are respected for missing keys. + valid = inspect.signature(CrawlerRunConfig.__init__).parameters.keys() - {"self"} + return CrawlerRunConfig(**{k: v for k, v in kwargs.items() if k in valid}) + + # Create a funciton returns dict of the object + def dump(self) -> dict: + # Serialize the object to a dictionary + return to_serializable_dict(self) + + @staticmethod + def load(data: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig": + # Deserialize the object from a dictionary + if provenance is None: + provenance = Provenance.TRUSTED + config = from_serializable_dict(data, provenance) + if isinstance(config, CrawlerRunConfig): + return config + if provenance == Provenance.UNTRUSTED and isinstance(config, dict): + config = _enforce_untrusted("CrawlerRunConfig", config) + return CrawlerRunConfig.from_kwargs(config) + + def to_dict(self): + return { + "word_count_threshold": self.word_count_threshold, + "extraction_strategy": self.extraction_strategy, + "chunking_strategy": self.chunking_strategy, + "markdown_generator": self.markdown_generator, + "only_text": self.only_text, + "css_selector": self.css_selector, + "target_elements": self.target_elements, + "excluded_tags": self.excluded_tags, + "excluded_selector": self.excluded_selector, + "keep_data_attributes": self.keep_data_attributes, + "keep_attrs": self.keep_attrs, + "remove_forms": self.remove_forms, + "prettiify": self.prettiify, + "parser_type": self.parser_type, + "scraping_strategy": self.scraping_strategy, + "proxy_config": ( + [p.to_dict() if hasattr(p, 'to_dict') else p for p in self.proxy_config] + if isinstance(self.proxy_config, list) + else (self.proxy_config.to_dict() if hasattr(self.proxy_config, 'to_dict') else self.proxy_config) + ), + "proxy_rotation_strategy": self.proxy_rotation_strategy, + "proxy_session_id": self.proxy_session_id, + "proxy_session_ttl": self.proxy_session_ttl, + "proxy_session_auto_release": self.proxy_session_auto_release, + "locale": self.locale, + "timezone_id": self.timezone_id, + "geolocation": self.geolocation, + "fetch_ssl_certificate": self.fetch_ssl_certificate, + "cache_mode": self.cache_mode, + "session_id": self.session_id, + "bypass_cache": self.bypass_cache, + "disable_cache": self.disable_cache, + "no_cache_read": self.no_cache_read, + "no_cache_write": self.no_cache_write, + "shared_data": self.shared_data, + "wait_until": self.wait_until, + "page_timeout": self.page_timeout, + "wait_for": self.wait_for, + "wait_for_timeout": self.wait_for_timeout, + "wait_for_images": self.wait_for_images, + "delay_before_return_html": self.delay_before_return_html, + "mean_delay": self.mean_delay, + "max_range": self.max_range, + "semaphore_count": self.semaphore_count, + "js_code": self.js_code, + "js_code_before_wait": self.js_code_before_wait, + "js_only": self.js_only, + "ignore_body_visibility": self.ignore_body_visibility, + "scan_full_page": self.scan_full_page, + "scroll_delay": self.scroll_delay, + "max_scroll_steps": self.max_scroll_steps, + "process_iframes": self.process_iframes, + "flatten_shadow_dom": self.flatten_shadow_dom, + "remove_overlay_elements": self.remove_overlay_elements, + "remove_consent_popups": self.remove_consent_popups, + "simulate_user": self.simulate_user, + "override_navigator": self.override_navigator, + "magic": self.magic, + "adjust_viewport_to_content": self.adjust_viewport_to_content, + "screenshot": self.screenshot, + "screenshot_wait_for": self.screenshot_wait_for, + "screenshot_height_threshold": self.screenshot_height_threshold, + "pdf": self.pdf, + "capture_mhtml": self.capture_mhtml, + "image_description_min_word_threshold": self.image_description_min_word_threshold, + "image_score_threshold": self.image_score_threshold, + "table_score_threshold": self.table_score_threshold, + "table_extraction": self.table_extraction, + "exclude_all_images": self.exclude_all_images, + "exclude_external_images": self.exclude_external_images, + "exclude_social_media_domains": self.exclude_social_media_domains, + "exclude_external_links": self.exclude_external_links, + "exclude_social_media_links": self.exclude_social_media_links, + "exclude_domains": self.exclude_domains, + "exclude_internal_links": self.exclude_internal_links, + "score_links": self.score_links, + "preserve_https_for_internal_links": self.preserve_https_for_internal_links, + "verbose": self.verbose, + "log_console": self.log_console, + "capture_network_requests": self.capture_network_requests, + "capture_console_messages": self.capture_console_messages, + "method": self.method, + "stream": self.stream, + "prefetch": self.prefetch, + "process_in_browser": self.process_in_browser, + "check_robots_txt": self.check_robots_txt, + "user_agent": self.user_agent, + "user_agent_mode": self.user_agent_mode, + "user_agent_generator_config": self.user_agent_generator_config, + "deep_crawl_strategy": self.deep_crawl_strategy, + "link_preview_config": self.link_preview_config.to_dict() if self.link_preview_config else None, + "url": self.url, + "url_matcher": self.url_matcher, + "match_mode": self.match_mode, + "experimental": self.experimental, + "max_retries": self.max_retries, + } + + def clone(self, **kwargs): + """Create a copy of this configuration with updated values. + + Args: + **kwargs: Key-value pairs of configuration options to update + + Returns: + CrawlerRunConfig: A new instance with the specified updates + + Example: + ```python + # Create a new config with streaming enabled + stream_config = config.clone(stream=True) + + # Create a new config with multiple updates + new_config = config.clone( + stream=True, + cache_mode=CacheMode.BYPASS, + verbose=True + ) + ``` + """ + config_dict = self.to_dict() + config_dict.update(kwargs) + return CrawlerRunConfig.from_kwargs(config_dict) + +class LLMConfig: + def __init__( + self, + provider: str = DEFAULT_PROVIDER, + api_token: Optional[str] = None, + base_url: Optional[str] = None, + temperature: Optional[float] = None, + max_tokens: Optional[int] = None, + top_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + stop: Optional[List[str]] = None, + n: Optional[int] = None, + backoff_base_delay: Optional[int] = None, + backoff_max_attempts: Optional[int] = None, + backoff_exponential_factor: Optional[int] = None, + provenance: "Provenance" = None, + ): + """Configuaration class for LLM provider and API token.""" + if provenance is None: + provenance = Provenance.TRUSTED + # Defense in depth: untrusted callers can already not reach here (the + # type gate forbids constructing LLMConfig from a request body), but if + # they ever do, never resolve env vars or read provider keys from the + # environment - that is the credential-exfil gadget. + if provenance == Provenance.UNTRUSTED: + if api_token and api_token.startswith("env:"): + raise UntrustedConfigError( + "LLMConfig.api_token may not reference an environment variable " + "from an untrusted request" + ) + self.provider = provider + self.api_token = api_token # never os.getenv + self.base_url = base_url + self.temperature = temperature + self.max_tokens = max_tokens + self.top_p = top_p + self.frequency_penalty = frequency_penalty + self.presence_penalty = presence_penalty + self.stop = stop + self.n = n + self.backoff_base_delay = backoff_base_delay if backoff_base_delay is not None else 2 + self.backoff_max_attempts = backoff_max_attempts if backoff_max_attempts is not None else 3 + self.backoff_exponential_factor = backoff_exponential_factor if backoff_exponential_factor is not None else 2 + return + self.provider = provider + if api_token and not api_token.startswith("env:"): + self.api_token = api_token + elif api_token and api_token.startswith("env:"): + self.api_token = os.getenv(api_token[4:]) + else: + # Check if given provider starts with any of key in PROVIDER_MODELS_PREFIXES + # If not, check if it is in PROVIDER_MODELS + prefixes = PROVIDER_MODELS_PREFIXES.keys() + if any(provider.startswith(prefix) for prefix in prefixes): + selected_prefix = next( + (prefix for prefix in prefixes if provider.startswith(prefix)), + None, + ) + self.api_token = PROVIDER_MODELS_PREFIXES.get(selected_prefix) + else: + self.provider = DEFAULT_PROVIDER + self.api_token = os.getenv(DEFAULT_PROVIDER_API_KEY) + self.base_url = base_url + self.temperature = temperature + self.max_tokens = max_tokens + self.top_p = top_p + self.frequency_penalty = frequency_penalty + self.presence_penalty = presence_penalty + self.stop = stop + self.n = n + self.backoff_base_delay = backoff_base_delay if backoff_base_delay is not None else 2 + self.backoff_max_attempts = backoff_max_attempts if backoff_max_attempts is not None else 3 + self.backoff_exponential_factor = backoff_exponential_factor if backoff_exponential_factor is not None else 2 + + @staticmethod + def from_kwargs(kwargs: dict) -> "LLMConfig": + return LLMConfig( + provider=kwargs.get("provider", DEFAULT_PROVIDER), + api_token=kwargs.get("api_token"), + base_url=kwargs.get("base_url"), + temperature=kwargs.get("temperature"), + max_tokens=kwargs.get("max_tokens"), + top_p=kwargs.get("top_p"), + frequency_penalty=kwargs.get("frequency_penalty"), + presence_penalty=kwargs.get("presence_penalty"), + stop=kwargs.get("stop"), + n=kwargs.get("n"), + backoff_base_delay=kwargs.get("backoff_base_delay"), + backoff_max_attempts=kwargs.get("backoff_max_attempts"), + backoff_exponential_factor=kwargs.get("backoff_exponential_factor") + ) + + def to_dict(self): + return { + "provider": self.provider, + "api_token": self.api_token, + "base_url": self.base_url, + "temperature": self.temperature, + "max_tokens": self.max_tokens, + "top_p": self.top_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + "stop": self.stop, + "n": self.n, + "backoff_base_delay": self.backoff_base_delay, + "backoff_max_attempts": self.backoff_max_attempts, + "backoff_exponential_factor": self.backoff_exponential_factor + } + + def clone(self, **kwargs): + """Create a copy of this configuration with updated values. + + Args: + **kwargs: Key-value pairs of configuration options to update + + Returns: + llm_config: A new instance with the specified updates + """ + config_dict = self.to_dict() + config_dict.update(kwargs) + return LLMConfig.from_kwargs(config_dict) + +class SeedingConfig: + """ + Configuration class for URL discovery and pre-validation via AsyncUrlSeeder. + """ + def __init__( + self, + source: str = "sitemap+cc", + pattern: Optional[str] = "*", + live_check: bool = False, + extract_head: bool = False, + max_urls: int = -1, + concurrency: int = 1000, + hits_per_sec: int = 5, + force: bool = False, + base_directory: Optional[str] = None, + llm_config: Optional[LLMConfig] = None, + verbose: Optional[bool] = None, + query: Optional[str] = None, + score_threshold: Optional[float] = None, + scoring_method: str = "bm25", + filter_nonsense_urls: bool = True, + cache_ttl_hours: int = 24, + validate_sitemap_lastmod: bool = True, + ): + """ + Initialize URL seeding configuration. + + Args: + source: Discovery source(s) to use. Options: "sitemap", "cc" (Common Crawl), + or "sitemap+cc" (both). Default: "sitemap+cc" + pattern: URL pattern to filter discovered URLs (e.g., "*example.com/blog/*"). + Supports glob-style wildcards. Default: "*" (all URLs) + live_check: Whether to perform HEAD requests to verify URL liveness. + Default: False + extract_head: Whether to fetch and parse <head> section for metadata extraction. + Required for BM25 relevance scoring. Default: False + max_urls: Maximum number of URLs to discover. Use -1 for no limit. + Default: -1 + concurrency: Maximum concurrent requests for live checks/head extraction. + Default: 1000 + hits_per_sec: Rate limit in requests per second to avoid overwhelming servers. + Default: 5 + force: If True, bypasses the AsyncUrlSeeder's internal .jsonl cache and + re-fetches URLs. Default: False + base_directory: Base directory for UrlSeeder's cache files (.jsonl). + If None, uses default ~/.crawl4ai/. Default: None + llm_config: LLM configuration for future features (e.g., semantic scoring). + Currently unused. Default: None + verbose: Override crawler's general verbose setting for seeding operations. + Default: None (inherits from crawler) + query: Search query for BM25 relevance scoring (e.g., "python tutorials"). + Requires extract_head=True. Default: None + score_threshold: Minimum relevance score (0.0-1.0) to include URL. + Only applies when query is provided. Default: None + scoring_method: Scoring algorithm to use. Currently only "bm25" is supported. + Future: "semantic". Default: "bm25" + filter_nonsense_urls: Filter out utility URLs like robots.txt, sitemap.xml, + ads.txt, favicon.ico, etc. Default: True + cache_ttl_hours: Hours before sitemap cache expires. Set to 0 to disable TTL + (only lastmod validation). Default: 24 + validate_sitemap_lastmod: If True, compares sitemap's <lastmod> with cache + timestamp and refetches if sitemap is newer. Default: True + """ + self.source = source + self.pattern = pattern + self.live_check = live_check + self.extract_head = extract_head + self.max_urls = max_urls + self.concurrency = concurrency + self.hits_per_sec = hits_per_sec + self.force = force + self.base_directory = base_directory + self.llm_config = llm_config + self.verbose = verbose + self.query = query + self.score_threshold = score_threshold + self.scoring_method = scoring_method + self.filter_nonsense_urls = filter_nonsense_urls + self.cache_ttl_hours = cache_ttl_hours + self.validate_sitemap_lastmod = validate_sitemap_lastmod + + # Add to_dict, from_kwargs, and clone methods for consistency + def to_dict(self) -> Dict[str, Any]: + return {k: v for k, v in self.__dict__.items() if k != 'llm_config' or v is not None} + + @staticmethod + def from_kwargs(kwargs: Dict[str, Any]) -> 'SeedingConfig': + return SeedingConfig(**kwargs) + + def clone(self, **kwargs: Any) -> 'SeedingConfig': + config_dict = self.to_dict() + config_dict.update(kwargs) + return SeedingConfig.from_kwargs(config_dict) + + +class DomainMapperConfig: + """ + Configuration for comprehensive domain URL discovery via DomainMapper. + + Discovers all URLs under a domain using multiple sources (sitemap, Common Crawl, + Wayback Machine, Certificate Transparency, path probing, robots.txt mining, + RSS/Atom feeds, homepage link extraction) without deep crawling. + """ + def __init__( + self, + source: str = "sitemap+cc+crt+probe", + max_urls: int = -1, + concurrency: int = 50, + hits_per_sec: int = 10, + force: bool = False, + extract_head: bool = True, + filter_nonsense_urls: bool = True, + soft_404_detection: bool = True, + query: Optional[str] = None, + score_threshold: Optional[float] = None, + scoring_method: str = "bm25", + probe_paths: Optional[List[str]] = None, + common_subdomains: Optional[List[str]] = None, + include_subdomains: bool = True, + source_timeout: float = 30.0, + use_browser_for_homepage: bool = False, + verbose: Optional[bool] = None, + cache_ttl_hours: int = 24, + base_directory: Optional[str] = None, + dns_timeout: float = 3.0, + http_timeout: float = 10.0, + ): + """ + Args: + source: Discovery sources joined by '+'. Valid: sitemap, cc, wayback, + crt, probe, robots, feed, homepage. Default: "sitemap+cc+crt+probe" + max_urls: Maximum URLs to return. -1 for unlimited. + concurrency: Max concurrent requests across all hosts. + hits_per_sec: Rate limit in requests/second. + force: Bypass all caches. + extract_head: Fetch and parse <head> metadata for each URL. + filter_nonsense_urls: Filter utility URLs (robots.txt, favicon, etc.). + soft_404_detection: Fingerprint and filter soft-404 pages (SPAs). + query: BM25 relevance query. Requires extract_head=True. + score_threshold: Minimum relevance score (0.0-1.0) to include URL. + scoring_method: Scoring algorithm. Currently only "bm25". + probe_paths: Extra paths to probe on each host (added to defaults). + common_subdomains: Extra subdomain prefixes to guess (added to defaults). + include_subdomains: Discover and scan subdomains. When False, only scans + the exact domain provided (skips crt.sh, DNS guessing, + Wayback/CC host discovery). Default True. + source_timeout: Max seconds per discovery source. Sources that exceed + this are killed and skipped. Prevents slow sources from + blocking fast results. Default 30.0. + use_browser_for_homepage: Use Playwright for JS-rendered homepages. + verbose: Override logger verbose setting. + cache_ttl_hours: Hours before cached results expire. + base_directory: Base directory for cache files. + dns_timeout: Timeout in seconds for DNS resolution. + http_timeout: Timeout in seconds for HTTP requests. + """ + self.source = source + self.max_urls = max_urls + self.concurrency = concurrency + self.hits_per_sec = hits_per_sec + self.force = force + self.extract_head = extract_head + self.filter_nonsense_urls = filter_nonsense_urls + self.soft_404_detection = soft_404_detection + self.query = query + self.score_threshold = score_threshold + self.scoring_method = scoring_method + self.probe_paths = probe_paths + self.common_subdomains = common_subdomains + self.include_subdomains = include_subdomains + self.source_timeout = source_timeout + self.use_browser_for_homepage = use_browser_for_homepage + self.verbose = verbose + self.cache_ttl_hours = cache_ttl_hours + self.base_directory = base_directory + self.dns_timeout = dns_timeout + self.http_timeout = http_timeout + + def to_dict(self) -> Dict[str, Any]: + return {k: v for k, v in self.__dict__.items()} + + @staticmethod + def from_kwargs(kwargs: Dict[str, Any]) -> 'DomainMapperConfig': + return DomainMapperConfig(**kwargs) + + def clone(self, **kwargs: Any) -> 'DomainMapperConfig': + config_dict = self.to_dict() + config_dict.update(kwargs) + return DomainMapperConfig.from_kwargs(config_dict) diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py new file mode 100644 index 0000000..265c376 --- /dev/null +++ b/crawl4ai/async_crawler_strategy.py @@ -0,0 +1,2830 @@ +from __future__ import annotations + +import asyncio +import base64 +import time +from abc import ABC, abstractmethod +from typing import Callable, Dict, Any, List, Union +from typing import Optional, AsyncGenerator, Final +import os +from playwright.async_api import Page, Error +from playwright.async_api import TimeoutError as PlaywrightTimeoutError +from io import BytesIO +from PIL import Image, ImageDraw, ImageFont +import hashlib +import random +import uuid +from .js_snippet import load_js_script +from .models import AsyncCrawlResponse +from .config import SCREENSHOT_HEIGHT_TRESHOLD +from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig +from .async_logger import AsyncLogger +from .ssl_certificate import SSLCertificate +from .user_agent_generator import ValidUAGenerator, UAGen +from .browser_manager import BrowserManager +from .browser_adapter import BrowserAdapter, PlaywrightAdapter, UndetectedAdapter + +import aiofiles +import aiohttp +import chardet +from aiohttp.client import ClientTimeout +from urllib.parse import urlparse +from types import MappingProxyType +import contextlib +from functools import partial + +class AsyncCrawlerStrategy(ABC): + """ + Abstract base class for crawler strategies. + Subclasses must implement the crawl method. + """ + + @abstractmethod + async def crawl(self, url: str, **kwargs) -> AsyncCrawlResponse: + pass # 4 + 3 + +class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy): + """ + Crawler strategy using Playwright. + + Attributes: + browser_config (BrowserConfig): Configuration object containing browser settings. + logger (AsyncLogger): Logger instance for recording events and errors. + _downloaded_files (List[str]): List of downloaded file paths. + hooks (Dict[str, Callable]): Dictionary of hooks for custom behavior. + browser_manager (BrowserManager): Manager for browser creation and management. + + Methods: + __init__(self, browser_config=None, logger=None, **kwargs): + Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration. + __aenter__(self): + Start the browser and initialize the browser manager. + __aexit__(self, exc_type, exc_val, exc_tb): + Close the browser and clean up resources. + start(self): + Start the browser and initialize the browser manager. + close(self): + Close the browser and clean up resources. + kill_session(self, session_id): + Kill a browser session and clean up resources. + crawl(self, url, **kwargs): + Run the crawler for a single URL. + + """ + + def __init__( + self, browser_config: BrowserConfig = None, logger: AsyncLogger = None, browser_adapter: BrowserAdapter = None, **kwargs + ): + """ + Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration. + + Args: + browser_config (BrowserConfig): Configuration object containing browser settings. + If None, will be created from kwargs for backwards compatibility. + logger: Logger instance for recording events and errors. + browser_adapter (BrowserAdapter): Browser adapter for handling browser-specific operations. + If None, defaults to PlaywrightAdapter. + **kwargs: Additional arguments for backwards compatibility and extending functionality. + """ + # Initialize browser config, either from provided object or kwargs + self.browser_config = browser_config or BrowserConfig.from_kwargs(kwargs) + # Initialize with default logger if none provided to prevent NoneType errors + self.logger = logger if logger is not None else AsyncLogger(verbose=False) + + # Initialize browser adapter + self.adapter = browser_adapter or PlaywrightAdapter() + + # Initialize session management + self._downloaded_files = [] + + # Initialize hooks system + self.hooks = { + "on_browser_created": None, + "on_page_context_created": None, + "on_user_agent_updated": None, + "on_execution_started": None, + "on_execution_ended": None, + "before_goto": None, + "after_goto": None, + "before_return_html": None, + "before_retrieve_html": None, + } + + # Initialize browser manager with config + self.browser_manager = BrowserManager( + browser_config=self.browser_config, + logger=self.logger, + use_undetected=isinstance(self.adapter, UndetectedAdapter) + ) + + async def __aenter__(self): + await self.start() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + async def start(self): + """ + Start the browser and initialize the browser manager. + """ + await self.browser_manager.start() + await self.execute_hook( + "on_browser_created", + self.browser_manager.browser, + context=self.browser_manager.default_context, + ) + + async def close(self): + """ + Close the browser and clean up resources. + """ + await self.browser_manager.close() + # Explicitly reset the static Playwright instance (skip if using cached CDP) + if not self.browser_manager._using_cached_cdp: + BrowserManager._playwright_instance = None + + async def kill_session(self, session_id: str): + """ + Kill a browser session and clean up resources. + + Args: + session_id (str): The ID of the session to kill. + + Returns: + None + """ + # Log a warning message and no need kill session, in new version auto kill session + self.logger.warning( + message="Session auto-kill is enabled in the new version. No need to manually kill sessions.", + tag="WARNING", + ) + await self.browser_manager.kill_session(session_id) + + def set_hook(self, hook_type: str, hook: Callable): + """ + Set a hook function for a specific hook type. Following are list of hook types: + - on_browser_created: Called when a new browser instance is created. + - on_page_context_created: Called when a new page context is created. + - on_user_agent_updated: Called when the user agent is updated. + - on_execution_started: Called when the execution starts. + - before_goto: Called before a goto operation. + - after_goto: Called after a goto operation. + - before_return_html: Called before returning HTML content. + - before_retrieve_html: Called before retrieving HTML content. + + All hooks except on_browser_created accepts a context and a page as arguments and **kwargs. However, on_browser_created accepts a browser and a context as arguments and **kwargs. + + Args: + hook_type (str): The type of the hook. + hook (Callable): The hook function to set. + + Returns: + None + """ + if hook_type in self.hooks: + self.hooks[hook_type] = hook + else: + raise ValueError(f"Invalid hook type: {hook_type}") + + async def execute_hook(self, hook_type: str, *args, **kwargs): + """ + Execute a hook function for a specific hook type. + + Args: + hook_type (str): The type of the hook. + *args: Variable length positional arguments. + **kwargs: Keyword arguments. + + Returns: + The return value of the hook function, if any. + """ + hook = self.hooks.get(hook_type) + if hook: + if asyncio.iscoroutinefunction(hook): + return await hook(*args, **kwargs) + else: + return hook(*args, **kwargs) + return args[0] if args else None + + def update_user_agent(self, user_agent: str): + """ + Update the user agent for the browser. + + Args: + user_agent (str): The new user agent string. + + Returns: + None + """ + self.user_agent = user_agent + + def set_custom_headers(self, headers: Dict[str, str]): + """ + Set custom headers for the browser. + + Args: + headers (Dict[str, str]): A dictionary of headers to set. + + Returns: + None + """ + self.headers = headers + + async def smart_wait(self, page: Page, wait_for: str, timeout: float = 30000): + """ + Wait for a condition in a smart way. This functions works as below: + + 1. If wait_for starts with 'js:', it assumes it's a JavaScript function and waits for it to return true. + 2. If wait_for starts with 'css:', it assumes it's a CSS selector and waits for it to be present. + 3. Otherwise, it tries to evaluate wait_for as a JavaScript function and waits for it to return true. + 4. If it's not a JavaScript function, it assumes it's a CSS selector and waits for it to be present. + + This is a more advanced version of the wait_for parameter in CrawlerStrategy.crawl(). + Args: + page: Playwright page object + wait_for (str): The condition to wait for. Can be a CSS selector, a JavaScript function, or explicitly prefixed with 'js:' or 'css:'. + timeout (float): Maximum time to wait in milliseconds + + Returns: + None + """ + wait_for = wait_for.strip() + + if wait_for.startswith("js:"): + # Explicitly specified JavaScript + js_code = wait_for[3:].strip() + return await self.csp_compliant_wait(page, js_code, timeout) + elif wait_for.startswith("css:"): + # Explicitly specified CSS selector + css_selector = wait_for[4:].strip() + try: + await page.wait_for_selector(css_selector, timeout=timeout) + except Error as e: + if "Timeout" in str(e): + raise TimeoutError( + f"Timeout after {timeout}ms waiting for selector '{css_selector}'" + ) + else: + raise ValueError(f"Invalid CSS selector: '{css_selector}'") + else: + # Auto-detect based on content + if wait_for.startswith("()") or wait_for.startswith("function"): + # It's likely a JavaScript function + return await self.csp_compliant_wait(page, wait_for, timeout) + else: + # Assume it's a CSS selector first + try: + await page.wait_for_selector(wait_for, timeout=timeout) + except Error as e: + if "Timeout" in str(e): + raise TimeoutError( + f"Timeout after {timeout}ms waiting for selector '{wait_for}'" + ) + else: + # If it's not a timeout error, it might be an invalid selector + # Let's try to evaluate it as a JavaScript function as a fallback + try: + return await self.csp_compliant_wait( + page, f"() => {{{wait_for}}}", timeout + ) + except Error: + raise ValueError( + f"Invalid wait_for parameter: '{wait_for}'. " + "It should be either a valid CSS selector, a JavaScript function, " + "or explicitly prefixed with 'js:' or 'css:'." + ) + + async def csp_compliant_wait( + self, page: Page, user_wait_function: str, timeout: float = 30000 + ): + """ + Wait for a condition in a CSP-compliant way. + + Args: + page: Playwright page object + user_wait_function: JavaScript function as string that returns boolean + timeout: Maximum time to wait in milliseconds + + Returns: + bool: True if condition was met, False if timed out + + Raises: + RuntimeError: If there's an error evaluating the condition + """ + wrapper_js = f""" + async () => {{ + const userFunction = {user_wait_function}; + const startTime = Date.now(); + try {{ + while (true) {{ + if (await userFunction()) {{ + return true; + }} + if (Date.now() - startTime > {timeout}) {{ + return false; // Return false instead of throwing + }} + await new Promise(resolve => setTimeout(resolve, 100)); + }} + }} catch (error) {{ + throw new Error(`Error evaluating condition: ${{error.message}}`); + }} + }} + """ + + try: + result = await self.adapter.evaluate(page, wrapper_js) + return result + except Exception as e: + if "Error evaluating condition" in str(e): + raise RuntimeError(f"Failed to evaluate wait condition: {str(e)}") + # For timeout or other cases, just return False + return False + + async def process_iframes(self, page): + """ + Process iframes on a page. This function will extract the content of each iframe and replace it with a div containing the extracted content. + + Args: + page: Playwright page object + + Returns: + Playwright page object + """ + # Find all iframes + iframes = await page.query_selector_all("iframe") + + for i, iframe in enumerate(iframes): + try: + # Add a unique identifier to the iframe + await iframe.evaluate(f'(element) => element.id = "iframe-{i}"') + + # Get the frame associated with this iframe + frame = await iframe.content_frame() + + if frame: + # Wait for the frame to load + await frame.wait_for_load_state( + "load", timeout=30000 + ) # 30 seconds timeout + + # Extract the content of the iframe's body + iframe_content = await frame.evaluate( + "() => document.body.innerHTML" + ) + + # Generate a unique class name for this iframe + class_name = f"extracted-iframe-content-{i}" + + # Replace the iframe with a div containing the extracted content + _iframe = iframe_content.replace("`", "\\`") + await self.adapter.evaluate(page, + f""" + () => {{ + const iframe = document.getElementById('iframe-{i}'); + const div = document.createElement('div'); + const parser = new DOMParser(); + const doc = parser.parseFromString(`{_iframe}`, 'text/html'); + while (doc.body.firstChild) {{ + div.appendChild(doc.body.firstChild); + }} + div.className = '{class_name}'; + iframe.replaceWith(div); + }} + """ + ) + else: + self.logger.warning( + message="Could not access content frame for iframe {index}", + tag="SCRAPE", + params={"index": i}, + ) + except Exception as e: + self.logger.error( + message="Error processing iframe {index}: {error}", + tag="ERROR", + params={"index": i, "error": str(e)}, + ) + + # Return the page object + return page + + async def create_session(self, **kwargs) -> str: + """ + Creates a new browser session and returns its ID. A browse session is a unique openned page can be reused for multiple crawls. + This function is asynchronous and returns a string representing the session ID. + + Args: + **kwargs: Optional keyword arguments to configure the session. + + Returns: + str: The session ID. + """ + await self.start() + + session_id = kwargs.get("session_id") or str(uuid.uuid4()) + + user_agent = kwargs.get("user_agent", self.user_agent) + # Use browser_manager to get a fresh page & context assigned to this session_id + page, context = await self.browser_manager.get_page(CrawlerRunConfig( + session_id=session_id, + user_agent=user_agent, + **kwargs, + )) + return session_id + + async def crawl( + self, url: str, config: CrawlerRunConfig, **kwargs + ) -> AsyncCrawlResponse: + """ + Crawls a given URL or processes raw HTML/local file content based on the URL prefix. + + Args: + url (str): The URL to crawl. Supported prefixes: + - 'http://' or 'https://': Web URL to crawl. + - 'file://': Local file path to process. + - 'raw://': Raw HTML content to process. + **kwargs: Additional parameters: + - 'screenshot' (bool): Whether to take a screenshot. + - ... [other existing parameters] + + Returns: + AsyncCrawlResponse: The response containing HTML, headers, status code, and optional screenshot. + """ + config = config or CrawlerRunConfig.from_kwargs(kwargs) + response_headers = {} + status_code = 200 # Default for local/raw HTML + screenshot_data = None + + if url.startswith(("http://", "https://", "view-source:")): + return await self._crawl_web(url, config) + + elif url.startswith("file://") or url.startswith("raw://") or url.startswith("raw:"): + # Check if browser processing is required for file:// or raw: URLs + needs_browser = ( + config.process_in_browser or + config.screenshot or + config.pdf or + config.capture_mhtml or + config.js_code or + config.wait_for or + config.scan_full_page or + config.remove_overlay_elements or + config.remove_consent_popups or + config.simulate_user or + config.magic or + config.process_iframes or + config.capture_console_messages or + config.capture_network_requests + ) + + if needs_browser: + # Route through _crawl_web() for full browser pipeline + # _crawl_web() will detect file:// and raw: URLs and use set_content() + return await self._crawl_web(url, config) + + # Fast path: return HTML directly without browser interaction + if url.startswith("file://"): + # Process local file + local_file_path = url[7:] # Remove 'file://' prefix + if not os.path.exists(local_file_path): + raise FileNotFoundError(f"Local file not found: {local_file_path}") + with open(local_file_path, "r", encoding="utf-8") as f: + html = f.read() + else: + # Process raw HTML content (raw:// or raw:) + html = url[6:] if url.startswith("raw://") else url[4:] + + return AsyncCrawlResponse( + html=html, + response_headers=response_headers, + status_code=status_code, + screenshot=None, + pdf_data=None, + mhtml_data=None, + get_delayed_content=None, + # For raw:/file:// URLs, use base_url if provided; don't fall back to the raw content + redirected_url=config.base_url, + ) + else: + raise ValueError( + "URL must start with 'http://', 'https://', 'file://', or 'raw:'" + ) + + async def _crawl_web( + self, url: str, config: CrawlerRunConfig + ) -> AsyncCrawlResponse: + """ + Internal method to crawl web URLs with the specified configuration. + Includes optional network and console capturing. + + Args: + url (str): The web URL to crawl + config (CrawlerRunConfig): Configuration object controlling the crawl behavior + + Returns: + AsyncCrawlResponse: The response containing HTML, headers, status code, and optional data + """ + config.url = url + response_headers = {} + execution_result = None + status_code = None + redirected_url = url + redirected_status_code = None + + # Reset downloaded files list for new crawl + self._downloaded_files = [] + + # Initialize capture lists + captured_requests = [] + captured_console = [] + + # Handle user agent with magic mode. + # For persistent contexts the UA is locked at browser launch time + # (launch_persistent_context bakes it into the protocol layer), so + # changing it here would only desync browser_config from reality. + # Users should set user_agent or user_agent_mode on BrowserConfig. + ua_changed = False + if not self.browser_config.use_persistent_context: + user_agent_to_override = config.user_agent + if user_agent_to_override: + self.browser_config.user_agent = user_agent_to_override + ua_changed = True + elif config.magic or config.user_agent_mode == "random": + self.browser_config.user_agent = ValidUAGenerator().generate( + **(config.user_agent_generator_config or {}) + ) + ua_changed = True + + # Keep sec-ch-ua in sync whenever the UA changed + if ua_changed: + self.browser_config.browser_hint = UAGen.generate_client_hints( + self.browser_config.user_agent + ) + self.browser_config.headers["sec-ch-ua"] = self.browser_config.browser_hint + + # Get page for session + page, context = await self.browser_manager.get_page(crawlerRunConfig=config) + + # When reusing a session page, abort any pending loads from the + # previous navigation to prevent timeouts on the next goto(). + if config.session_id: + try: + await page.evaluate("window.stop()") + except Exception: + pass + + try: + # Push updated UA + sec-ch-ua to the page so the server sees them + if ua_changed: + combined_headers = { + "User-Agent": self.browser_config.user_agent, + "sec-ch-ua": self.browser_config.browser_hint, + } + combined_headers.update(self.browser_config.headers) + await page.set_extra_http_headers(combined_headers) + + # await page.goto(URL) + + # Add default cookie + # await context.add_cookies( + # [{"name": "cookiesEnabled", "value": "true", "url": url}] + # ) + + # Handle navigator overrides — only inject if not already done + # at context level by setup_context(). This fallback covers + # managed-browser / persistent / CDP paths where setup_context() + # is called without a crawlerRunConfig. + if config.override_navigator or config.simulate_user or config.magic: + if not getattr(context, '_crawl4ai_nav_overrider_injected', False): + await context.add_init_script(load_js_script("navigator_overrider")) + context._crawl4ai_nav_overrider_injected = True + + # Force-open closed shadow roots — same guard against duplication + if config.flatten_shadow_dom: + if not getattr(context, '_crawl4ai_shadow_dom_injected', False): + await context.add_init_script(""" + const _origAttachShadow = Element.prototype.attachShadow; + Element.prototype.attachShadow = function(init) { + return _origAttachShadow.call(this, {...init, mode: 'open'}); + }; + """) + context._crawl4ai_shadow_dom_injected = True + + # Call hook after page creation + await self.execute_hook("on_page_context_created", page, context=context, config=config) + + # Network Request Capturing + if config.capture_network_requests: + async def handle_request_capture(request): + try: + post_data_str = None + try: + # Be cautious with large post data + post_data = request.post_data_buffer + if post_data: + # Attempt to decode, fallback to base64 or size indication + try: + post_data_str = post_data.decode('utf-8', errors='replace') + except UnicodeDecodeError: + post_data_str = f"[Binary data: {len(post_data)} bytes]" + except Exception: + post_data_str = "[Error retrieving post data]" + + captured_requests.append({ + "event_type": "request", + "url": request.url, + "method": request.method, + "headers": dict(request.headers), # Convert Header dict + "post_data": post_data_str, + "resource_type": request.resource_type, + "is_navigation_request": request.is_navigation_request(), + "timestamp": time.time() + }) + except Exception as e: + if self.logger: + self.logger.warning(f"Error capturing request details for {request.url}: {e}", tag="CAPTURE") + captured_requests.append({"event_type": "request_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()}) + + async def handle_response_capture(response): + try: + try: + # body = await response.body() + # json_body = await response.json() + text_body = await response.text() + except Exception as e: + body = None + # json_body = None + # text_body = None + captured_requests.append({ + "event_type": "response", + "url": response.url, + "status": response.status, + "status_text": response.status_text, + "headers": dict(response.headers), # Convert Header dict + "from_service_worker": response.from_service_worker, + "request_timing": response.request.timing, # Detailed timing info + "timestamp": time.time(), + "body" : { + # "raw": body, + # "json": json_body, + "text": text_body + } + }) + except Exception as e: + if self.logger: + self.logger.warning(f"Error capturing response details for {response.url}: {e}", tag="CAPTURE") + captured_requests.append({"event_type": "response_capture_error", "url": response.url, "error": str(e), "timestamp": time.time()}) + + async def handle_request_failed_capture(request): + try: + captured_requests.append({ + "event_type": "request_failed", + "url": request.url, + "method": request.method, + "resource_type": request.resource_type, + "failure_text": str(request.failure) if request.failure else "Unknown failure", + "timestamp": time.time() + }) + except Exception as e: + if self.logger: + self.logger.warning(f"Error capturing request failed details for {request.url}: {e}", tag="CAPTURE") + captured_requests.append({"event_type": "request_failed_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()}) + + page.on("request", handle_request_capture) + page.on("response", handle_response_capture) + page.on("requestfailed", handle_request_failed_capture) + + # Console Message Capturing + handle_console = None + handle_error = None + if config.capture_console_messages: + # Set up console capture using adapter + handle_console = await self.adapter.setup_console_capture(page, captured_console) + handle_error = await self.adapter.setup_error_capture(page, captured_console) + + # Set up console logging if requested + # Note: For undetected browsers, console logging won't work directly + # but captured messages can still be logged after retrieval + # Get SSL certificate information if requested and URL is HTTPS + ssl_cert = None + if config.fetch_ssl_certificate: + ssl_cert = SSLCertificate.from_url(url) + + # Set up download handling + if self.browser_config.accept_downloads: + page.on( + "download", + lambda download: asyncio.create_task( + self._handle_download(download) + ), + ) + + # Handle page navigation and content loading + if not config.js_only: + await self.execute_hook("before_goto", page, context=context, url=url, config=config) + + # Check if this is a file:// or raw: URL that needs set_content() instead of goto() + is_local_content = url.startswith("file://") or url.startswith("raw://") or url.startswith("raw:") + + if is_local_content: + # Load local content using set_content() instead of network navigation + if url.startswith("file://"): + local_file_path = url[7:] # Remove 'file://' prefix + if not os.path.exists(local_file_path): + raise FileNotFoundError(f"Local file not found: {local_file_path}") + with open(local_file_path, "r", encoding="utf-8") as f: + html_content = f.read() + else: + # raw:// or raw: + html_content = url[6:] if url.startswith("raw://") else url[4:] + + await page.set_content(html_content, wait_until=config.wait_until) + response = None + # For raw: URLs, only use base_url if provided; don't fall back to the raw HTML string + redirected_url = config.base_url + status_code = 200 + response_headers = {} + else: + # Standard web navigation with goto() + try: + # Generate a unique nonce for this request + if config.experimental.get("use_csp_nonce", False): + nonce = hashlib.sha256(os.urandom(32)).hexdigest() + + # Add CSP headers to the request + await page.set_extra_http_headers( + { + "Content-Security-Policy": f"default-src 'self'; script-src 'self' 'nonce-{nonce}' 'strict-dynamic'" + } + ) + + response = await page.goto( + url, wait_until=config.wait_until, timeout=config.page_timeout + ) + redirected_url = page.url + redirected_status_code = response.status if response else None + except Error as e: + # Allow navigation to be aborted when downloading files + # This is expected behavior for downloads in some browser engines + if 'net::ERR_ABORTED' in str(e) and self.browser_config.accept_downloads: + self.logger.info( + message=f"Navigation aborted, likely due to file download: {url}", + tag="GOTO", + params={"url": url}, + ) + response = None + else: + raise RuntimeError(f"Failed on navigating ACS-GOTO:\n{str(e)}") + + # ────────────────────────────────────────────────────────────── + # Walk the redirect chain. Playwright returns only the last + # hop, so we trace the `request.redirected_from` links until the + # first response that differs from the final one and surface its + # status-code. + # ────────────────────────────────────────────────────────────── + if response is None: + status_code = 200 + response_headers = {} + else: + first_resp = response + req = response.request + while req and req.redirected_from: + prev_req = req.redirected_from + prev_resp = await prev_req.response() + if prev_resp: # keep earliest + first_resp = prev_resp + req = prev_req + + status_code = first_resp.status + response_headers = first_resp.headers + + await self.execute_hook( + "after_goto", page, context=context, url=url, response=response, config=config + ) + + else: + status_code = 200 + response_headers = {} + + # Wait for body element and visibility + try: + await page.wait_for_selector("body", state="attached", timeout=30000) + + # Use the new check_visibility function with csp_compliant_wait + is_visible = await self.csp_compliant_wait( + page, + """() => { + const element = document.body; + if (!element) return false; + const style = window.getComputedStyle(element); + const isVisible = style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0'; + return isVisible; + }""", + timeout=30000, + ) + + if not is_visible and not config.ignore_body_visibility: + visibility_info = await self.check_visibility(page) + raise Error(f"Body element is hidden: {visibility_info}") + + except Error: + visibility_info = await self.check_visibility(page) + + if self.browser_config.verbose: + self.logger.debug( + message="Body visibility info: {info}", + tag="DEBUG", + params={"info": visibility_info}, + ) + + if not config.ignore_body_visibility: + raise Error(f"Body element is hidden: {visibility_info}") + + # try: + # await page.wait_for_selector("body", state="attached", timeout=30000) + + # await page.wait_for_function( + # """ + # () => { + # const body = document.body; + # const style = window.getComputedStyle(body); + # return style.display !== 'none' && + # style.visibility !== 'hidden' && + # style.opacity !== '0'; + # } + # """, + # timeout=30000, + # ) + # except Error as e: + # visibility_info = await page.evaluate( + # """ + # () => { + # const body = document.body; + # const style = window.getComputedStyle(body); + # return { + # display: style.display, + # visibility: style.visibility, + # opacity: style.opacity, + # hasContent: body.innerHTML.length, + # classList: Array.from(body.classList) + # } + # } + # """ + # ) + + # if self.config.verbose: + # self.logger.debug( + # message="Body visibility info: {info}", + # tag="DEBUG", + # params={"info": visibility_info}, + # ) + + # if not config.ignore_body_visibility: + # raise Error(f"Body element is hidden: {visibility_info}") + + # Handle content loading and viewport adjustment + if not self.browser_config.text_mode and ( + config.wait_for_images or config.adjust_viewport_to_content + ): + await page.wait_for_load_state("domcontentloaded") + await asyncio.sleep(0.1) + + # Check for image loading with improved error handling + images_loaded = await self.csp_compliant_wait( + page, + "() => Array.from(document.getElementsByTagName('img')).every(img => img.complete)", + timeout=1000, + ) + + if not images_loaded and self.logger: + self.logger.warning( + message="Some images failed to load within timeout", + tag="SCRAPE", + ) + + # Adjust viewport if needed + if not self.browser_config.text_mode and config.adjust_viewport_to_content: + try: + dimensions = await self.get_page_dimensions(page) + page_height = dimensions["height"] + page_width = dimensions["width"] + # page_width = await page.evaluate( + # "document.documentElement.scrollWidth" + # ) + # page_height = await page.evaluate( + # "document.documentElement.scrollHeight" + # ) + + target_width = self.browser_config.viewport_width + target_height = int(target_width * page_width / page_height * 0.95) + await page.set_viewport_size( + {"width": target_width, "height": target_height} + ) + + scale = min(target_width / page_width, target_height / page_height) + cdp = await page.context.new_cdp_session(page) + await cdp.send( + "Emulation.setDeviceMetricsOverride", + { + "width": page_width, + "height": page_height, + "deviceScaleFactor": 1, + "mobile": False, + "scale": scale, + }, + ) + await cdp.detach() + except Exception as e: + self.logger.warning( + message="Failed to adjust viewport to content: {error}", + tag="VIEWPORT", + params={"error": str(e)}, + ) + + # Handle full page scanning + if config.scan_full_page: + scan_timeout = (config.page_timeout or 30000) / 1000 # ms to seconds + try: + await asyncio.wait_for( + self._handle_full_page_scan(page, config.scroll_delay, config.max_scroll_steps), + timeout=scan_timeout, + ) + except asyncio.TimeoutError: + self.logger.warning( + message="Full page scan timed out after {timeout}s, continuing with partial scroll", + tag="PAGE_SCAN", + params={"timeout": scan_timeout}, + ) + + # --- Phase 1: Pre-wait JS and interaction --- + + # Execute js_code_before_wait (for triggering loading that wait_for checks) + if config.js_code_before_wait: + bw_result = await self.robust_execute_user_script( + page, config.js_code_before_wait + ) + if not bw_result["success"]: + self.logger.warning( + message="js_code_before_wait had issues: {error}", + tag="JS_EXEC", + params={"error": bw_result.get("error")}, + ) + + # Handle user simulation — generate mouse movement and scroll + # signals that anti-bot systems look for, without firing keyboard + # events (ArrowDown triggers JS framework navigation) or clicking + # at fixed positions (may hit buttons/links and navigate away). + if config.simulate_user or config.magic: + await page.mouse.move(random.randint(100, 300), random.randint(150, 300)) + await page.mouse.move(random.randint(300, 600), random.randint(200, 400)) + await page.mouse.wheel(0, random.randint(200, 400)) + + # --- Phase 2: Wait for page readiness --- + + if config.wait_for: + try: + timeout = config.wait_for_timeout if config.wait_for_timeout is not None else config.page_timeout + await self.smart_wait( + page, config.wait_for, timeout=timeout + ) + except Exception as e: + raise RuntimeError(f"Wait condition failed: {str(e)}") + + # Handle virtual scroll if configured (after wait_for so container exists) + if config.virtual_scroll_config: + await self._handle_virtual_scroll(page, config.virtual_scroll_config) + + # Pre-content retrieval hooks and delay + await self.execute_hook("before_retrieve_html", page, context=context, config=config) + if config.delay_before_return_html: + await asyncio.sleep(config.delay_before_return_html) + + # --- Phase 3: Post-wait JS (runs on fully-loaded page) --- + + if config.js_code: + execution_result = await self.robust_execute_user_script( + page, config.js_code + ) + + if not execution_result["success"]: + self.logger.warning( + message="User script execution had issues: {error}", + tag="JS_EXEC", + params={"error": execution_result.get("error")}, + ) + + await self.execute_hook("on_execution_started", page, context=context, config=config) + await self.execute_hook("on_execution_ended", page, context=context, config=config, result=execution_result) + + # --- Phase 4: DOM processing before HTML capture --- + + # Update image dimensions if needed + if not self.browser_config.text_mode: + update_image_dimensions_js = load_js_script("update_image_dimensions") + try: + try: + await page.wait_for_load_state("domcontentloaded", timeout=5) + except PlaywrightTimeoutError: + pass + await self.adapter.evaluate(page, update_image_dimensions_js) + except Exception as e: + self.logger.error( + message="Error updating image dimensions: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + + # Process iframes if needed + if config.process_iframes: + page = await self.process_iframes(page) + + # Handle CMP/consent popup removal (before generic overlay removal) + if config.remove_consent_popups: + await self.remove_consent_popups(page) + + # Handle overlay removal + if config.remove_overlay_elements: + await self.remove_overlay_elements(page) + + # --- Phase 5: HTML capture --- + + if config.flatten_shadow_dom: + # Use JS to serialize the full DOM including shadow roots + flatten_js = load_js_script("flatten_shadow_dom") + html = await self.adapter.evaluate(page, flatten_js) + if not html or not isinstance(html, str): + # Fallback to normal capture if JS returned nothing + self.logger.warning( + message="Shadow DOM flattening returned no content, falling back to page.content()", + tag="SCRAPE", + ) + html = await page.content() + elif config.css_selector: + try: + selectors = [s.strip() for s in config.css_selector.split(',')] + html_parts = [] + + for selector in selectors: + try: + content = await self.adapter.evaluate(page, + f"""Array.from(document.querySelectorAll("{selector}")) + .map(el => el.outerHTML) + .join('')""" + ) + html_parts.append(content) + except Error as e: + print(f"Warning: Could not get content for selector '{selector}': {str(e)}") + + html = f"<div class='crawl4ai-result'>\n" + "\n".join(html_parts) + "\n</div>" + except Error as e: + raise RuntimeError(f"Failed to extract HTML content: {str(e)}") + else: + html = await page.content() + + await self.execute_hook( + "before_return_html", page=page, html=html, context=context, config=config + ) + + # Handle PDF, MHTML and screenshot generation + start_export_time = time.perf_counter() + pdf_data = None + screenshot_data = None + mhtml_data = None + + if config.pdf: + pdf_data = await self.export_pdf(page) + + if config.capture_mhtml: + mhtml_data = await self.capture_mhtml(page) + + if config.screenshot: + if config.screenshot_wait_for: + await asyncio.sleep(config.screenshot_wait_for) + screenshot_data = await self.take_screenshot( + page, + screenshot_height_threshold=config.screenshot_height_threshold, + force_viewport_screenshot=config.force_viewport_screenshot, + scan_full_page=config.scan_full_page, + scroll_delay=config.scroll_delay + ) + + if screenshot_data or pdf_data or mhtml_data: + self.logger.info( + message="Exporting media (PDF/MHTML/screenshot) took {duration:.2f}s", + tag="EXPORT", + params={"duration": time.perf_counter() - start_export_time}, + ) + + # Define delayed content getter + async def get_delayed_content(delay: float = 5.0) -> str: + self.logger.info( + message="Waiting for {delay} seconds before retrieving content for {url}", + tag="INFO", + params={"delay": delay, "url": url}, + ) + await asyncio.sleep(delay) + return await page.content() + + # For undetected browsers, retrieve console messages before returning + if config.capture_console_messages and hasattr(self.adapter, 'retrieve_console_messages'): + final_messages = await self.adapter.retrieve_console_messages(page) + captured_console.extend(final_messages) + + ### + # This ensures we capture the current page URL at the time we return the response, + # which correctly reflects any JavaScript navigation that occurred. + # For raw:/file:// URLs, preserve the earlier redirected_url (config.base_url or None) + # instead of using page.url which would be "about:blank". + ### + is_local_content = url.startswith("file://") or url.startswith("raw://") or url.startswith("raw:") + if not is_local_content: + redirected_url = page.url # Use current page URL to capture JS redirects + + # Return complete response + return AsyncCrawlResponse( + html=html, + response_headers=response_headers, + js_execution_result=execution_result, + status_code=status_code, + screenshot=screenshot_data, + pdf_data=pdf_data, + mhtml_data=mhtml_data, + get_delayed_content=get_delayed_content, + ssl_certificate=ssl_cert, + downloaded_files=( + self._downloaded_files if self._downloaded_files else None + ), + redirected_url=redirected_url, + redirected_status_code=redirected_status_code, + # Include captured data if enabled + network_requests=captured_requests if config.capture_network_requests else None, + console_messages=captured_console if config.capture_console_messages else None, + ) + + except Exception as e: + raise e + + finally: + # Always clean up event listeners to prevent accumulation + # across reuses (even for session pages). + try: + if config.capture_network_requests: + page.remove_listener("request", handle_request_capture) + page.remove_listener("response", handle_response_capture) + page.remove_listener("requestfailed", handle_request_failed_capture) + if config.capture_console_messages: + if hasattr(self.adapter, 'retrieve_console_messages'): + final_messages = await self.adapter.retrieve_console_messages(page) + captured_console.extend(final_messages) + await self.adapter.cleanup_console_capture(page, handle_console, handle_error) + except Exception: + pass + + if not config.session_id: + # ALWAYS decrement refcount first — must succeed even if + # the browser crashed or the page is in a bad state. + try: + await self.browser_manager.release_page_with_context(page) + except Exception: + pass + + # Close the page unless it's the last one in a headless/managed browser + try: + all_contexts = page.context.browser.contexts + total_pages = sum(len(context.pages) for context in all_contexts) + if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)): + await page.close() + except Exception: + pass + + # async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1): + async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1, max_scroll_steps: Optional[int] = None): + """ + Helper method to handle full page scanning. + + How it works: + 1. Get the viewport height. + 2. Scroll to the bottom of the page. + 3. Get the total height of the page. + 4. Scroll back to the top of the page. + 5. Scroll to the bottom of the page again. + 6. Continue scrolling until the bottom of the page is reached. + + Args: + page (Page): The Playwright page object + scroll_delay (float): The delay between page scrolls + max_scroll_steps (Optional[int]): Maximum number of scroll steps to perform. Defaults to 10 to prevent infinite scroll hangs. + + """ + # Default to 10 steps to prevent infinite scroll on dynamic pages + if max_scroll_steps is None: + max_scroll_steps = 10 + + try: + viewport_size = page.viewport_size + if viewport_size is None: + await page.set_viewport_size( + {"width": self.browser_config.viewport_width, "height": self.browser_config.viewport_height} + ) + viewport_size = page.viewport_size + + viewport_height = viewport_size.get( + "height", self.browser_config.viewport_height + ) + current_position = viewport_height + + # await page.evaluate(f"window.scrollTo(0, {current_position})") + await self.safe_scroll(page, 0, current_position, delay=scroll_delay) + # await self.csp_scroll_to(page, 0, current_position) + # await asyncio.sleep(scroll_delay) + + # total_height = await page.evaluate("document.documentElement.scrollHeight") + dimensions = await self.get_page_dimensions(page) + total_height = dimensions["height"] + + scroll_step_count = 0 + while current_position < total_height: + #### + # NEW FEATURE: Check if we've reached the maximum allowed scroll steps + # This prevents infinite scrolling on very long pages or infinite scroll scenarios + # If max_scroll_steps is None, this check is skipped (unlimited scrolling - original behavior) + #### + if max_scroll_steps is not None and scroll_step_count >= max_scroll_steps: + break + current_position = min(current_position + viewport_height, total_height) + await self.safe_scroll(page, 0, current_position, delay=scroll_delay) + + # Increment the step counter for max_scroll_steps tracking + scroll_step_count += 1 + + # await page.evaluate(f"window.scrollTo(0, {current_position})") + # await asyncio.sleep(scroll_delay) + + # new_height = await page.evaluate("document.documentElement.scrollHeight") + dimensions = await self.get_page_dimensions(page) + new_height = dimensions["height"] + + if new_height > total_height: + total_height = new_height + + # await page.evaluate("window.scrollTo(0, 0)") + await self.safe_scroll(page, 0, 0) + + except Exception as e: + self.logger.warning( + message="Failed to perform full page scan: {error}", + tag="PAGE_SCAN", + params={"error": str(e)}, + ) + else: + # await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await self.safe_scroll(page, 0, total_height) + + async def _handle_virtual_scroll(self, page: Page, config: "VirtualScrollConfig"): + """ + Handle virtual scroll containers (e.g., Twitter-like feeds) by capturing + content at different scroll positions and merging unique elements. + + Following the design: + 1. Get container HTML + 2. Scroll by container height + 3. Wait and check if container HTML changed + 4. Three cases: + - No change: continue scrolling + - New items added (appended): continue (items already in page) + - Items replaced: capture HTML chunk and add to list + 5. After N scrolls, merge chunks if any were captured + + Args: + page: The Playwright page object + config: Virtual scroll configuration + """ + try: + # Import VirtualScrollConfig to avoid circular import + from .async_configs import VirtualScrollConfig + + # Ensure config is a VirtualScrollConfig instance + if isinstance(config, dict): + config = VirtualScrollConfig.from_dict(config) + + self.logger.info( + message="Starting virtual scroll capture for container: {selector}", + tag="VSCROLL", + params={"selector": config.container_selector} + ) + + # JavaScript function to handle virtual scroll capture + virtual_scroll_js = """ + async (config) => { + const container = document.querySelector(config.container_selector); + if (!container) { + throw new Error(`Container not found: ${config.container_selector}`); + } + + // List to store HTML chunks when content is replaced + const htmlChunks = []; + let previousHTML = container.innerHTML; + let scrollCount = 0; + + // Determine scroll amount + let scrollAmount; + if (typeof config.scroll_by === 'number') { + scrollAmount = config.scroll_by; + } else if (config.scroll_by === 'page_height') { + scrollAmount = window.innerHeight; + } else { // container_height + scrollAmount = container.offsetHeight; + } + + // Perform scrolling + while (scrollCount < config.scroll_count) { + // Scroll the container + container.scrollTop += scrollAmount; + + // Wait for content to potentially load + await new Promise(resolve => setTimeout(resolve, config.wait_after_scroll * 1000)); + + // Get current HTML + const currentHTML = container.innerHTML; + + // Determine what changed + if (currentHTML === previousHTML) { + // Case 0: No change - continue scrolling + console.log(`Scroll ${scrollCount + 1}: No change in content`); + } else if (currentHTML.startsWith(previousHTML)) { + // Case 1: New items appended - content already in page + console.log(`Scroll ${scrollCount + 1}: New items appended`); + } else { + // Case 2: Items replaced - capture the previous HTML + console.log(`Scroll ${scrollCount + 1}: Content replaced, capturing chunk`); + htmlChunks.push(previousHTML); + } + + // Update previous HTML for next iteration + previousHTML = currentHTML; + scrollCount++; + + // Check if we've reached the end + if (container.scrollTop + container.clientHeight >= container.scrollHeight - 10) { + console.log(`Reached end of scrollable content at scroll ${scrollCount}`); + // Capture final chunk if content was replaced + if (htmlChunks.length > 0) { + htmlChunks.push(currentHTML); + } + break; + } + } + + // If we have chunks (case 2 occurred), merge them + if (htmlChunks.length > 0) { + console.log(`Merging ${htmlChunks.length} HTML chunks`); + + // Parse all chunks to extract unique elements + const tempDiv = document.createElement('div'); + const seenTexts = new Set(); + const uniqueElements = []; + + // Process each chunk + for (const chunk of htmlChunks) { + tempDiv.innerHTML = chunk; + const elements = tempDiv.children; + + for (let i = 0; i < elements.length; i++) { + const element = elements[i]; + // Normalize text for deduplication + const normalizedText = element.innerText + .toLowerCase() + .replace(/[\\s\\W]/g, ''); // Remove spaces and symbols + + if (!seenTexts.has(normalizedText)) { + seenTexts.add(normalizedText); + uniqueElements.push(element.outerHTML); + } + } + } + + // Replace container content with merged unique elements + container.innerHTML = uniqueElements.join('\\n'); + console.log(`Merged ${uniqueElements.length} unique elements from ${htmlChunks.length} chunks`); + + return { + success: true, + chunksCount: htmlChunks.length, + uniqueCount: uniqueElements.length, + replaced: true + }; + } else { + console.log('No content replacement detected, all content remains in page'); + return { + success: true, + chunksCount: 0, + uniqueCount: 0, + replaced: false + }; + } + } + """ + + # Execute virtual scroll capture + result = await self.adapter.evaluate(page, virtual_scroll_js, config.to_dict()) + + if result.get("replaced", False): + self.logger.success( + message="Virtual scroll completed. Merged {unique} unique elements from {chunks} chunks", + tag="VSCROLL", + params={ + "unique": result.get("uniqueCount", 0), + "chunks": result.get("chunksCount", 0) + } + ) + else: + self.logger.info( + message="Virtual scroll completed. Content was appended, no merging needed", + tag="VSCROLL" + ) + + except Exception as e: + self.logger.error( + message="Virtual scroll capture failed: {error}", + tag="VSCROLL", + params={"error": str(e)} + ) + # Continue with normal flow even if virtual scroll fails + + async def _handle_download(self, download): + """ + Handle file downloads. + + How it works: + 1. Get the suggested filename. + 2. Get the download path. + 3. Log the download. + 4. Start the download. + 5. Save the downloaded file. + 6. Log the completion. + + Args: + download (Download): The Playwright download object + + Returns: + None + """ + try: + suggested_filename = download.suggested_filename + download_path = _safe_download_filepath(self.browser_config.downloads_path, suggested_filename) + # Playwright's save_as performs the write itself (no O_NOFOLLOW + # hook), so reject a symlink planted at the target before writing. + if os.path.islink(download_path): + raise ValueError(f"Refusing to write download through symlink: {download_path}") + + self.logger.info( + message="Downloading {filename} to {path}", + tag="FETCH", + params={"filename": suggested_filename, "path": download_path}, + ) + + start_time = time.perf_counter() + await download.save_as(download_path) + end_time = time.perf_counter() + self._downloaded_files.append(download_path) + + self.logger.success( + message="Downloaded {filename} successfully", + tag="COMPLETE", + params={ + "filename": suggested_filename, + "path": download_path, + "duration": f"{end_time - start_time:.2f}s", + }, + ) + except Exception as e: + self.logger.error( + message="Failed to handle download: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + + async def remove_overlay_elements(self, page: Page) -> None: + """ + Removes popup overlays, modals, cookie notices, and other intrusive elements from the page. + + Args: + page (Page): The Playwright page instance + """ + remove_overlays_js = load_js_script("remove_overlay_elements") + + try: + await self.adapter.evaluate(page, + f""" + (async () => {{ + try {{ + const removeOverlays = {remove_overlays_js}; + await removeOverlays(); + return {{ success: true }}; + }} catch (error) {{ + return {{ + success: false, + error: error.toString(), + stack: error.stack + }}; + }} + }})() + """ + ) + await page.wait_for_timeout(500) # Wait for any animations to complete + except Exception as e: + self.logger.warning( + message="Failed to remove overlay elements: {error}", + tag="SCRAPE", + params={"error": str(e)}, + ) + + async def remove_consent_popups(self, page: Page) -> None: + """ + Removes GDPR/cookie consent popups from known CMP providers (OneTrust, Cookiebot, + TrustArc, Quantcast, Didomi, Usercentrics, Sourcepoint, Klaro, Osano, Iubenda, + Complianz, CookieYes, ConsentManager, LiveRamp/Fides, etc.). + + Strategy: + 1. Try clicking "Accept All" buttons (cleanest dismissal, sets cookies) + 2. Try IAB TCF / CMP JavaScript APIs + 3. Remove known CMP containers by selector + 4. Handle iframe-based CMPs + 5. Restore body scroll + + Args: + page (Page): The Playwright page instance + """ + remove_consent_js = load_js_script("remove_consent_popups") + + try: + await self.adapter.evaluate(page, + f""" + (async () => {{ + try {{ + const removeConsent = {remove_consent_js}; + await removeConsent(); + return {{ success: true }}; + }} catch (error) {{ + return {{ + success: false, + error: error.toString(), + stack: error.stack + }}; + }} + }})() + """ + ) + await page.wait_for_timeout(500) # Wait for any animations to complete + except Exception as e: + self.logger.warning( + message="Failed to remove consent popups: {error}", + tag="SCRAPE", + params={"error": str(e)}, + ) + + async def export_pdf(self, page: Page) -> bytes: + """ + Exports the current page as a PDF. + + Args: + page (Page): The Playwright page object + + Returns: + bytes: The PDF data + """ + pdf_data = await page.pdf(print_background=True) + return pdf_data + + async def capture_mhtml(self, page: Page) -> Optional[str]: + """ + Captures the current page as MHTML using CDP. + + MHTML (MIME HTML) is a web page archive format that combines the HTML content + with its resources (images, CSS, etc.) into a single MIME-encoded file. + + Args: + page (Page): The Playwright page object + + Returns: + Optional[str]: The MHTML content as a string, or None if there was an error + """ + try: + # Ensure the page is fully loaded before capturing + try: + # Wait for DOM content and network to be idle + await page.wait_for_load_state("domcontentloaded", timeout=5000) + await page.wait_for_load_state("networkidle", timeout=5000) + + # Give a little extra time for JavaScript execution + await page.wait_for_timeout(1000) + + # Wait for any animations to complete + await page.evaluate(""" + () => new Promise(resolve => { + // First requestAnimationFrame gets scheduled after the next repaint + requestAnimationFrame(() => { + // Second requestAnimationFrame gets called after all animations complete + requestAnimationFrame(resolve); + }); + }) + """) + except Error as e: + if self.logger: + self.logger.warning( + message="Wait for load state timed out: {error}", + tag="MHTML", + params={"error": str(e)}, + ) + + # Create a new CDP session + cdp_session = await page.context.new_cdp_session(page) + + # Call Page.captureSnapshot with format "mhtml" + result = await cdp_session.send("Page.captureSnapshot", {"format": "mhtml"}) + + # The result contains a 'data' field with the MHTML content + mhtml_content = result.get("data") + + # Detach the CDP session to clean up resources + await cdp_session.detach() + + return mhtml_content + except Exception as e: + # Log the error but don't raise it - we'll just return None for the MHTML + if self.logger: + self.logger.error( + message="Failed to capture MHTML: {error}", + tag="MHTML", + params={"error": str(e)}, + ) + return None + + async def _capture_console_messages( + self, page: Page, file_path: str + ) -> List[Dict[str, Union[str, float]]]: + """ + Captures console messages from the page. + Args: + + page (Page): The Playwright page object + Returns: + List[Dict[str, Union[str, float]]]: A list of captured console messages + """ + captured_console = [] + + def handle_console_message(msg): + try: + message_type = msg.type + message_text = msg.text + + entry = { + "type": message_type, + "text": message_text, + "timestamp": time.time(), + } + captured_console.append(entry) + except Exception as e: + if self.logger: + self.logger.warning( + f"Error capturing console message: {e}", tag="CAPTURE" + ) + + page.on("console", handle_console_message) + + await page.goto(file_path) + + return captured_console + + async def _generate_media_from_html( + self, html: str, config: CrawlerRunConfig = None + ) -> tuple: + """ + Generate media (screenshot, PDF, MHTML) from raw HTML content. + + This method is used for raw: and file:// URLs where we have HTML content + but need to render it in a browser to generate media outputs. + + Args: + html (str): The raw HTML content to render + config (CrawlerRunConfig, optional): Configuration for media options + + Returns: + tuple: (screenshot_data, pdf_data, mhtml_data) - any can be None + """ + page = None + screenshot_data = None + pdf_data = None + mhtml_data = None + + try: + # Get a browser page + config = config or CrawlerRunConfig() + page, context = await self.browser_manager.get_page(crawlerRunConfig=config) + + # Load the HTML content into the page + await page.set_content(html, wait_until="domcontentloaded") + + # Generate requested media + if config.pdf: + pdf_data = await self.export_pdf(page) + + if config.capture_mhtml: + mhtml_data = await self.capture_mhtml(page) + + if config.screenshot: + if config.screenshot_wait_for: + await asyncio.sleep(config.screenshot_wait_for) + screenshot_height_threshold = getattr(config, 'screenshot_height_threshold', None) + screenshot_data = await self.take_screenshot( + page, + screenshot_height_threshold=screenshot_height_threshold, + scan_full_page=getattr(config, 'scan_full_page', True), + scroll_delay=config.scroll_delay if config else 0.2 + ) + + return screenshot_data, pdf_data, mhtml_data + + except Exception as e: + error_message = f"Failed to generate media from HTML: {str(e)}" + self.logger.error( + message="HTML media generation failed: {error}", + tag="ERROR", + params={"error": error_message}, + ) + # Return error image for screenshot if it was requested + if config and config.screenshot: + img = Image.new("RGB", (800, 600), color="black") + draw = ImageDraw.Draw(img) + font = ImageFont.load_default() + draw.text((10, 10), error_message, fill=(255, 255, 255), font=font) + buffered = BytesIO() + img.save(buffered, format="JPEG") + screenshot_data = base64.b64encode(buffered.getvalue()).decode("utf-8") + return screenshot_data, pdf_data, mhtml_data + finally: + # Clean up the page + if page: + try: + await self.browser_manager.release_page_with_context(page) + await page.close() + except Exception: + pass + + async def take_screenshot(self, page, **kwargs) -> str: + """ + Take a screenshot of the current page. + + Args: + page (Page): The Playwright page object + kwargs: Additional keyword arguments + + Returns: + str: The base64-encoded screenshot data + """ + # Check if viewport-only screenshot is forced + force_viewport = kwargs.get('force_viewport_screenshot', False) + scan_full_page = kwargs.get('scan_full_page', True) + + if force_viewport or not scan_full_page: + # Use viewport-only screenshot + return await self.take_screenshot_naive(page) + + need_scroll = await self.page_need_scroll(page) + + if not need_scroll: + # Page is short enough, just take a screenshot + return await self.take_screenshot_naive(page) + else: + # Page is too long, try to take a full-page screenshot + return await self.take_screenshot_scroller(page, **kwargs) + # return await self.take_screenshot_from_pdf(await self.export_pdf(page)) + + async def take_screenshot_from_pdf(self, pdf_data: bytes) -> str: + """ + Convert the first page of the PDF to a screenshot. + + Requires pdf2image and poppler. + + Args: + pdf_data (bytes): The PDF data + + Returns: + str: The base64-encoded screenshot data + """ + try: + from pdf2image import convert_from_bytes + + images = convert_from_bytes(pdf_data) + final_img = images[0].convert("RGB") + buffered = BytesIO() + final_img.save(buffered, format="JPEG") + return base64.b64encode(buffered.getvalue()).decode("utf-8") + except Exception as e: + error_message = f"Failed to take PDF-based screenshot: {str(e)}" + self.logger.error( + message="PDF Screenshot failed: {error}", + tag="ERROR", + params={"error": error_message}, + ) + # Return error image as fallback + img = Image.new("RGB", (800, 600), color="black") + draw = ImageDraw.Draw(img) + font = ImageFont.load_default() + draw.text((10, 10), error_message, fill=(255, 255, 255), font=font) + buffered = BytesIO() + img.save(buffered, format="JPEG") + return base64.b64encode(buffered.getvalue()).decode("utf-8") + + async def take_screenshot_scroller(self, page: Page, **kwargs) -> str: + """ + Attempt to set a large viewport and take a full-page screenshot. + If still too large, segment the page as before. + + Requires pdf2image and poppler. + + Args: + page (Page): The Playwright page object + kwargs: Additional keyword arguments + + Returns: + str: The base64-encoded screenshot data + """ + try: + # Save original viewport so we can restore it after capture + original_viewport = page.viewport_size + + # Get page height + dimensions = await self.get_page_dimensions(page) + page_width = dimensions["width"] + page_height = dimensions["height"] + + # Freeze element dimensions before viewport change to prevent + # responsive CSS from rescaling images (fixes Elementor distortion) + await page.evaluate(""" + document.querySelectorAll('img, video, picture, svg, canvas').forEach(el => { + const rect = el.getBoundingClientRect(); + if (rect.width > 0 && rect.height > 0) { + el.style.setProperty('width', rect.width + 'px', 'important'); + el.style.setProperty('height', rect.height + 'px', 'important'); + el.dataset.crawl4aiFrozen = '1'; + } + }); + """) + + # Set a large viewport + large_viewport_height = min( + page_height, + kwargs.get("screenshot_height_threshold", SCREENSHOT_HEIGHT_TRESHOLD), + ) + await page.set_viewport_size( + {"width": page_width, "height": large_viewport_height} + ) + + # Page still too long, segment approach + scroll_delay = kwargs.get("scroll_delay", 0.2) + segments = [] + viewport_size = page.viewport_size + viewport_height = viewport_size["height"] + + num_segments = (page_height // viewport_height) + 1 + for i in range(num_segments): + y_offset = i * viewport_height + # Special handling for the last segment + if i == num_segments - 1: + last_part_height = page_height % viewport_height + + # If page_height is an exact multiple of viewport_height, + # we don't need an extra segment + if last_part_height == 0: + # Skip last segment if page height is exact multiple of viewport + break + + # Adjust viewport to exactly match the remaining content height + await page.set_viewport_size({"width": page_width, "height": last_part_height}) + + await page.evaluate(f"window.scrollTo(0, {y_offset})") + await asyncio.sleep(scroll_delay) # wait for render (respects scroll_delay config) + + # Capture the current segment + seg_shot = await page.screenshot(full_page=False, type="jpeg", quality=85) + img = Image.open(BytesIO(seg_shot)).convert("RGB") + segments.append(img) + + # Unfreeze element dimensions and restore original viewport + await page.evaluate(""" + document.querySelectorAll('[data-crawl4ai-frozen]').forEach(el => { + el.style.removeProperty('width'); + el.style.removeProperty('height'); + delete el.dataset.crawl4aiFrozen; + }); + """) + await page.set_viewport_size(original_viewport) + + total_height = sum(img.height for img in segments) + stitched = Image.new("RGB", (segments[0].width, total_height)) + offset = 0 + for img in segments: + # stitched.paste(img, (0, offset)) + stitched.paste(img.convert("RGB"), (0, offset)) + offset += img.height + + buffered = BytesIO() + stitched = stitched.convert("RGB") + stitched.save(buffered, format="PNG") + encoded = base64.b64encode(buffered.getvalue()).decode("utf-8") + + return encoded + except Exception as e: + error_message = f"Failed to take large viewport screenshot: {str(e)}" + self.logger.error( + message="Large viewport screenshot failed: {error}", + tag="ERROR", + params={"error": error_message}, + ) + # return error image + img = Image.new("RGB", (800, 600), color="black") + draw = ImageDraw.Draw(img) + font = ImageFont.load_default() + draw.text((10, 10), error_message, fill=(255, 255, 255), font=font) + buffered = BytesIO() + img.save(buffered, format="JPEG") + return base64.b64encode(buffered.getvalue()).decode("utf-8") + # finally: + # await page.close() + + async def take_screenshot_naive(self, page: Page) -> str: + """ + Takes a screenshot of the current page. + + Args: + page (Page): The Playwright page instance + + Returns: + str: Base64-encoded screenshot image + """ + try: + # The page is already loaded, just take the screenshot + screenshot = await page.screenshot(full_page=False) + return base64.b64encode(screenshot).decode("utf-8") + except Exception as e: + error_message = f"Failed to take screenshot: {str(e)}" + self.logger.error( + message="Screenshot failed: {error}", + tag="ERROR", + params={"error": error_message}, + ) + + # Generate an error image + img = Image.new("RGB", (800, 600), color="black") + draw = ImageDraw.Draw(img) + font = ImageFont.load_default() + draw.text((10, 10), error_message, fill=(255, 255, 255), font=font) + + buffered = BytesIO() + img.save(buffered, format="JPEG") + return base64.b64encode(buffered.getvalue()).decode("utf-8") + # finally: + # await page.close() + + async def export_storage_state(self, path: str = None) -> dict: + """ + Exports the current storage state (cookies, localStorage, sessionStorage) + to a JSON file at the specified path. + + Args: + path (str): The path to save the storage state JSON file + + Returns: + dict: The exported storage state + """ + if self.default_context: + state = await self.default_context.storage_state(path=path) + self.logger.info( + message="Exported storage state to {path}", + tag="INFO", + params={"path": path}, + ) + return state + else: + self.logger.warning( + message="No default_context available to export storage state.", + tag="WARNING", + ) + + async def robust_execute_user_script( + self, page: Page, js_code: Union[str, List[str]] + ) -> Dict[str, Any]: + """ + Executes user-provided JavaScript code with proper error handling and context, + supporting both synchronous and async user code, plus navigations. + + How it works: + 1. Wait for load state 'domcontentloaded' + 2. If js_code is a string, execute it directly + 3. If js_code is a list, execute each element in sequence + 4. Wait for load state 'networkidle' + 5. Return results + + Args: + page (Page): The Playwright page instance + js_code (Union[str, List[str]]): The JavaScript code to execute + + Returns: + Dict[str, Any]: The results of the execution + """ + try: + await page.wait_for_load_state("domcontentloaded") + + if isinstance(js_code, str): + scripts = [js_code] + else: + scripts = js_code + + results = [] + for script in scripts: + try: + # Attempt the evaluate + # If the user code triggers navigation, we catch the "context destroyed" error + # then wait for the new page to load before continuing + result = None + try: + # OLD VERSION: + # result = await page.evaluate( + # f""" + # (async () => {{ + # try {{ + # const script_result = {script}; + # return {{ success: true, result: script_result }}; + # }} catch (err) {{ + # return {{ success: false, error: err.toString(), stack: err.stack }}; + # }} + # }})(); + # """ + # ) + + # """ NEW VERSION: + # When {script} contains statements (e.g., const link = …; link.click();), + # this forms invalid JavaScript, causing Playwright execution error: SyntaxError: Unexpected token 'const'. + # """ + result = await self.adapter.evaluate(page, + f""" + (async () => {{ + try {{ + return await (async () => {{ + {script} + }})(); + }} catch (err) {{ + return {{ success: false, error: err.toString(), stack: err.stack }}; + }} + }})(); + """ + ) + except Error as e: + # If it's due to navigation destroying the context, handle gracefully + if "Execution context was destroyed" in str(e): + self.logger.info( + "Navigation triggered by script, waiting for load state", + tag="JS_EXEC", + ) + try: + await page.wait_for_load_state("load", timeout=30000) + except Error as nav_err: + self.logger.warning( + message="Navigation wait failed: {error}", + tag="JS_EXEC", + params={"error": str(nav_err)}, + ) + try: + await page.wait_for_load_state( + "networkidle", timeout=30000 + ) + except Error as nav_err: + self.logger.warning( + message="Network idle wait failed: {error}", + tag="JS_EXEC", + params={"error": str(nav_err)}, + ) + # Return partial success, or adapt as you see fit + result = { + "success": True, + "info": "Navigation triggered, ignoring context destroyed error", + } + else: + # It's some other error, log and continue + self.logger.error( + message="Playwright execution error: {error}", + tag="JS_EXEC", + params={"error": str(e)}, + ) + result = {"success": False, "error": str(e)} + + # If we made it this far with no repeated error, do post-load waits + t1 = time.time() + try: + await page.wait_for_load_state("domcontentloaded", timeout=5000) + except Error as e: + self.logger.warning( + message="DOM content load timeout: {error}", + tag="JS_EXEC", + params={"error": str(e)}, + ) + + # t1 = time.time() + # try: + # await page.wait_for_load_state('networkidle', timeout=5000) + # print("Network idle after script execution in", time.time() - t1) + # except Error as e: + # self.logger.warning( + # message="Network idle timeout: {error}", + # tag="JS_EXEC", + # params={"error": str(e)} + # ) + + results.append(result if result else {"success": True}) + + except Exception as e: + # Catch anything else + self.logger.error( + message="Script chunk failed: {error}", + tag="JS_EXEC", + params={"error": str(e)}, + ) + results.append({"success": False, "error": str(e)}) + + return {"success": True, "results": results} + + except Exception as e: + self.logger.error( + message="Script execution failed: {error}", + tag="JS_EXEC", + params={"error": str(e)}, + ) + return {"success": False, "error": str(e)} + + async def execute_user_script( + self, page: Page, js_code: Union[str, List[str]] + ) -> Dict[str, Any]: + """ + Executes user-provided JavaScript code with proper error handling and context. + + Args: + page: Playwright page object + js_code: Single JavaScript string or list of JavaScript code strings + + Returns: + Dict containing execution status and results/errors + """ + try: + # Ensure the page is ready for script execution + await page.wait_for_load_state("domcontentloaded") + + # Handle single script or multiple scripts + if isinstance(js_code, str): + scripts = [js_code] + else: + scripts = js_code + + results = [] + for script in scripts: + try: + # Execute the script and wait for network idle + result = await self.adapter.evaluate(page, + f""" + (() => {{ + return new Promise((resolve) => {{ + try {{ + const result = (function() {{ + {script} + }})(); + + // If result is a promise, wait for it + if (result instanceof Promise) {{ + result.then(() => {{ + // Wait a bit for any triggered effects + setTimeout(() => resolve({{ success: true }}), 100); + }}).catch(error => {{ + resolve({{ + success: false, + error: error.toString(), + stack: error.stack + }}); + }}); + }} else {{ + // For non-promise results, still wait a bit for effects + setTimeout(() => resolve({{ success: true }}), 100); + }} + }} catch (error) {{ + resolve({{ + success: false, + error: error.toString(), + stack: error.stack + }}); + }} + }}); + }})() + """ + ) + + # Wait for network idle after script execution + t1 = time.time() + await page.wait_for_load_state("domcontentloaded", timeout=5000) + + + t1 = time.time() + await page.wait_for_load_state("networkidle", timeout=5000) + + results.append(result if result else {"success": True}) + + except Error as e: + # Handle Playwright-specific errors + self.logger.error( + message="Playwright execution error: {error}", + tag="JS_EXEC", + params={"error": str(e)}, + ) + results.append({"success": False, "error": str(e)}) + + return {"success": True, "results": results} + + except Exception as e: + self.logger.error( + message="Script execution failed: {error}", + tag="JS_EXEC", + params={"error": str(e)}, + ) + return {"success": False, "error": str(e)} + + except Exception as e: + self.logger.error( + message="Script execution failed: {error}", + tag="JS_EXEC", + params={"error": str(e)}, + ) + return {"success": False, "error": str(e)} + + async def check_visibility(self, page): + """ + Checks if an element is visible on the page. + + Args: + page: Playwright page object + + Returns: + Boolean indicating visibility + """ + return await self.adapter.evaluate(page, + """ + () => { + const element = document.body; + if (!element) return false; + const style = window.getComputedStyle(element); + const isVisible = style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0'; + return isVisible; + } + """ + ) + + async def safe_scroll(self, page: Page, x: int, y: int, delay: float = 0.1): + """ + Safely scroll the page with rendering time. + + Args: + page: Playwright page object + x: Horizontal scroll position + y: Vertical scroll position + """ + result = await self.csp_scroll_to(page, x, y) + if result["success"]: + await page.wait_for_timeout(delay * 1000) + return result + + async def csp_scroll_to(self, page: Page, x: int, y: int) -> Dict[str, Any]: + """ + Performs a CSP-compliant scroll operation and returns the result status. + + Args: + page: Playwright page object + x: Horizontal scroll position + y: Vertical scroll position + + Returns: + Dict containing scroll status and position information + """ + try: + result = await self.adapter.evaluate(page, + f"""() => {{ + try {{ + const startX = window.scrollX; + const startY = window.scrollY; + window.scrollTo({x}, {y}); + + // Get final position after scroll + const endX = window.scrollX; + const endY = window.scrollY; + + return {{ + success: true, + startPosition: {{ x: startX, y: startY }}, + endPosition: {{ x: endX, y: endY }}, + targetPosition: {{ x: {x}, y: {y} }}, + delta: {{ + x: Math.abs(endX - {x}), + y: Math.abs(endY - {y}) + }} + }}; + }} catch (e) {{ + return {{ + success: false, + error: e.toString() + }}; + }} + }}""" + ) + + if not result["success"]: + self.logger.warning( + message="Scroll operation failed: {error}", + tag="SCROLL", + params={"error": result.get("error")}, + ) + + return result + + except Exception as e: + self.logger.error( + message="Failed to execute scroll: {error}", + tag="SCROLL", + params={"error": str(e)}, + ) + return {"success": False, "error": str(e)} + + async def get_page_dimensions(self, page: Page): + """ + Get the dimensions of the page. + + Args: + page: Playwright page object + + Returns: + Dict containing width and height of the page + """ + return await self.adapter.evaluate(page, + """ + () => { + const {scrollWidth, scrollHeight} = document.documentElement; + return {width: scrollWidth, height: scrollHeight}; + } + """ + ) + + async def page_need_scroll(self, page: Page) -> bool: + """ + Determine whether the page need to scroll + + Args: + page: Playwright page object + + Returns: + bool: True if page needs scrolling + """ + try: + need_scroll = await self.adapter.evaluate(page, + """ + () => { + const scrollHeight = document.documentElement.scrollHeight; + const viewportHeight = window.innerHeight; + return scrollHeight > viewportHeight; + } + """ + ) + return need_scroll + except Exception as e: + self.logger.warning( + message="Failed to check scroll need: {error}. Defaulting to True for safety.", + tag="SCROLL", + params={"error": str(e)}, + ) + return True # Default to scrolling if check fails + + +#################################################################################################### +# HTTP Crawler Strategy +#################################################################################################### + +def _safe_download_filepath(downloads_path: str, filename: str) -> str: + """Resolve a download destination confined to ``downloads_path``. + + The filename is derived from attacker-influenced input (a remote + Content-Disposition header, or the browser's suggested filename), so it is + reduced to a bare basename (dropping absolute paths and ``..`` traversal) + and the resolved path is re-checked to live inside the downloads root, + rejecting any pre-existing symlink that points outside. Raises ValueError + on any escape. The final write must still use ``_nofollow_opener`` (or an + equivalent ``O_NOFOLLOW`` / pre-write symlink check) to close the TOCTOU + window between this check and the open. + """ + safe_name = os.path.basename(filename or "") + if not safe_name or safe_name in (".", ".."): + safe_name = f"download_{hashlib.md5((filename or '').encode()).hexdigest()[:10]}" + real_root = os.path.realpath(downloads_path) + real_path = os.path.realpath(os.path.join(real_root, safe_name)) + if os.path.commonpath([real_root, real_path]) != real_root: + raise ValueError(f"Unsafe download filename rejected: {filename!r}") + return real_path + + +def _nofollow_opener(path, flags): + """Opener for ``open``/``aiofiles.open`` that refuses to follow a symlink at + the final path component, closing the TOCTOU symlink-swap race after a path + has been confined by ``_safe_download_filepath``.""" + return os.open(path, flags | os.O_NOFOLLOW) + + +class HTTPCrawlerError(Exception): + """Base error class for HTTP crawler specific exceptions""" + pass + + +class ConnectionTimeoutError(HTTPCrawlerError): + """Raised when connection timeout occurs""" + pass + + +class HTTPStatusError(HTTPCrawlerError): + """Raised for unexpected status codes""" + def __init__(self, status_code: int, message: str): + self.status_code = status_code + super().__init__(f"HTTP {status_code}: {message}") + + +class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy): + """ + Fast, lightweight HTTP-only crawler strategy optimized for memory efficiency. + """ + + __slots__ = ('logger', 'max_connections', 'dns_cache_ttl', 'chunk_size', '_session', 'hooks', 'browser_config') + + DEFAULT_TIMEOUT: Final[int] = 30 + DEFAULT_CHUNK_SIZE: Final[int] = 64 * 1024 + DEFAULT_MAX_CONNECTIONS: Final[int] = min(32, (os.cpu_count() or 1) * 4) + DEFAULT_DNS_CACHE_TTL: Final[int] = 300 + VALID_SCHEMES: Final = frozenset({'http', 'https', 'file', 'raw'}) + + _BASE_HEADERS: Final = MappingProxyType({ + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate, br', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' + }) + + def __init__( + self, + browser_config: Optional[HTTPCrawlerConfig] = None, + logger: Optional[AsyncLogger] = None, + max_connections: int = DEFAULT_MAX_CONNECTIONS, + dns_cache_ttl: int = DEFAULT_DNS_CACHE_TTL, + chunk_size: int = DEFAULT_CHUNK_SIZE + ): + """Initialize the HTTP crawler with config""" + self.browser_config = browser_config or HTTPCrawlerConfig() + self.logger = logger + self.max_connections = max_connections + self.dns_cache_ttl = dns_cache_ttl + self.chunk_size = chunk_size + self._session: Optional[aiohttp.ClientSession] = None + + self.hooks = { + k: partial(self._execute_hook, k) + for k in ('before_request', 'after_request', 'on_error') + } + + # Set default hooks + self.set_hook('before_request', lambda *args, **kwargs: None) + self.set_hook('after_request', lambda *args, **kwargs: None) + self.set_hook('on_error', lambda *args, **kwargs: None) + + + async def __aenter__(self) -> AsyncHTTPCrawlerStrategy: + await self.start() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + await self.close() + + @contextlib.asynccontextmanager + async def _session_context(self): + try: + if not self._session: + await self.start() + yield self._session + finally: + pass + + def set_hook(self, hook_type: str, hook_func: Callable) -> None: + if hook_type in self.hooks: + self.hooks[hook_type] = partial(self._execute_hook, hook_type, hook_func) + else: + raise ValueError(f"Invalid hook type: {hook_type}") + + async def _execute_hook( + self, + hook_type: str, + hook_func: Callable, + *args: Any, + **kwargs: Any + ) -> Any: + if asyncio.iscoroutinefunction(hook_func): + return await hook_func(*args, **kwargs) + return hook_func(*args, **kwargs) + + async def start(self) -> None: + if not self._session: + connector = aiohttp.TCPConnector( + limit=self.max_connections, + ttl_dns_cache=self.dns_cache_ttl, + use_dns_cache=True, + force_close=False + ) + self._session = aiohttp.ClientSession( + headers=dict(self._BASE_HEADERS), + connector=connector, + timeout=ClientTimeout(total=self.DEFAULT_TIMEOUT) + ) + + async def close(self) -> None: + if self._session and not self._session.closed: + try: + await asyncio.wait_for(self._session.close(), timeout=5.0) + except asyncio.TimeoutError: + if self.logger: + self.logger.warning( + message="Session cleanup timed out", + tag="CLEANUP" + ) + finally: + self._session = None + + async def _stream_file(self, path: str) -> AsyncGenerator[memoryview, None]: + async with aiofiles.open(path, mode='rb') as f: + while chunk := await f.read(self.chunk_size): + yield memoryview(chunk) + + async def _handle_file(self, path: str) -> AsyncCrawlResponse: + if not os.path.exists(path): + raise FileNotFoundError(f"Local file not found: {path}") + + chunks = [] + async for chunk in self._stream_file(path): + chunks.append(chunk.tobytes().decode('utf-8', errors='replace')) + + return AsyncCrawlResponse( + html=''.join(chunks), + response_headers={}, + status_code=200 + ) + + async def _handle_raw(self, content: str, base_url: str = None) -> AsyncCrawlResponse: + return AsyncCrawlResponse( + html=content, + response_headers={}, + status_code=200, + # For raw: URLs, use base_url if provided; don't fall back to the raw content + redirected_url=base_url + ) + + + def _format_proxy_url(self, proxy_config) -> str: + """Format ProxyConfig into aiohttp-compatible proxy URL.""" + if not proxy_config: + return None + + server = proxy_config.server + username = getattr(proxy_config, 'username', None) + password = getattr(proxy_config, 'password', None) + + if username and password: + # Insert credentials into URL: http://user:pass@host:port + if '://' in server: + protocol, rest = server.split('://', 1) + return f"{protocol}://{username}:{password}@{rest}" + else: + return f"http://{username}:{password}@{server}" + + return server + + # Content types treated as text (decoded into html field) + _TEXT_CONTENT_TYPES: Final = frozenset({ + 'text/csv', 'text/plain', 'text/tab-separated-values', 'text/xml', + 'application/json', 'application/xml', 'application/xhtml+xml', + 'application/rss+xml', 'application/atom+xml', 'application/ld+json', + 'application/x-ndjson', 'text/calendar', 'text/vcard', + }) + + def _is_file_download(self, content_type: str, content_disposition: str) -> bool: + """Detect if the HTTP response is a file download rather than an HTML page.""" + if 'attachment' in content_disposition: + return True + if not content_type or content_type == 'text/html': + return False + # Anything that isn't text/html is a file download + return True + + def _is_text_content(self, content_type: str) -> bool: + """Check if content type is text-based (safe to decode and put in html field).""" + if content_type in self._TEXT_CONTENT_TYPES: + return True + # Catch-all for text/* subtypes not in the explicit set + return content_type.startswith('text/') + + def _extract_filename(self, content_disposition: str, url: str, content_type: str) -> str: + """Extract filename from Content-Disposition header or URL path.""" + # Try Content-Disposition first + if content_disposition: + import re + # filename*=UTF-8''encoded_name (RFC 5987) + match = re.search(r"filename\*=(?:UTF-8''|utf-8'')(.+?)(?:;|$)", content_disposition) + if match: + from urllib.parse import unquote + return unquote(match.group(1).strip()) + # filename="name" or filename=name + match = re.search(r'filename="?([^";]+)"?', content_disposition) + if match: + return match.group(1).strip() + + # Fall back to URL path + path = urlparse(url).path + if path and '/' in path: + basename = path.rsplit('/', 1)[-1] + if '.' in basename and len(basename) <= 255: + return basename + + # Last resort: hash-based name with extension from content type + ext_map = { + 'text/csv': '.csv', 'application/pdf': '.pdf', + 'application/zip': '.zip', 'image/png': '.png', + 'image/jpeg': '.jpg', 'application/json': '.json', + 'text/plain': '.txt', 'application/xml': '.xml', + } + ext = ext_map.get(content_type, '') + return f"download_{hashlib.md5(url.encode()).hexdigest()[:10]}{ext}" + + async def _handle_http( + self, + url: str, + config: CrawlerRunConfig + ) -> AsyncCrawlResponse: + async with self._session_context() as session: + # page_timeout is in ms (Playwright convention), but aiohttp expects seconds + timeout_sec = (config.page_timeout / 1000) if config.page_timeout else self.DEFAULT_TIMEOUT + timeout = ClientTimeout( + total=timeout_sec, + connect=10, + sock_read=30 + ) + + headers = dict(self._BASE_HEADERS) + if self.browser_config.headers: + headers.update(self.browser_config.headers) + + request_kwargs = { + 'timeout': timeout, + 'allow_redirects': self.browser_config.follow_redirects, + 'ssl': self.browser_config.verify_ssl, + 'headers': headers + } + + # Add proxy support - use config.proxy_config (set by arun() from rotation strategy or direct config) + proxy_url = None + if config.proxy_config: + proxy_url = self._format_proxy_url(config.proxy_config) + request_kwargs['proxy'] = proxy_url + + if self.browser_config.method == "POST": + if self.browser_config.data: + request_kwargs['data'] = self.browser_config.data + if self.browser_config.json: + request_kwargs['json'] = self.browser_config.json + + await self.hooks['before_request'](url, request_kwargs) + + try: + async with session.request(self.browser_config.method, url, **request_kwargs) as response: + raw_bytes = await response.read() + content = memoryview(raw_bytes) + + if not (200 <= response.status < 300): + raise HTTPStatusError( + response.status, + f"Unexpected status code for {url}" + ) + + response_headers = dict(response.headers) + content_type = response.content_type or 'text/html' + content_type = content_type.split(';')[0].strip().lower() + content_disposition = response_headers.get('Content-Disposition', '') + + downloaded_files = None + html = "" + + if self._is_file_download(content_type, content_disposition): + # Save file to disk + downloads_path = self.browser_config.downloads_path or os.path.join( + os.path.expanduser("~"), ".crawl4ai", "downloads" + ) + os.makedirs(downloads_path, exist_ok=True) + + filename = self._extract_filename(content_disposition, url, content_type) + filepath = _safe_download_filepath(downloads_path, filename) + + async with aiofiles.open(filepath, 'wb', opener=_nofollow_opener) as f: + await f.write(raw_bytes) + + downloaded_files = [filepath] + + # For text-based files, also decode into html (backward compatible) + if self._is_text_content(content_type): + encoding = response.charset + if not encoding: + detection_result = await asyncio.to_thread(chardet.detect, raw_bytes) + encoding = detection_result['encoding'] or 'utf-8' + html = raw_bytes.decode(encoding, errors='replace') + else: + # Standard HTML response — existing behavior + encoding = response.charset + if not encoding: + detection_result = await asyncio.to_thread(chardet.detect, content.tobytes()) + encoding = detection_result['encoding'] or 'utf-8' + html = content.tobytes().decode(encoding, errors='replace') + + result = AsyncCrawlResponse( + html=html, + response_headers=response_headers, + status_code=response.status, + redirected_url=str(response.url), + downloaded_files=downloaded_files, + ) + + await self.hooks['after_request'](result) + return result + + except aiohttp.ServerTimeoutError as e: + await self.hooks['on_error'](e) + raise ConnectionTimeoutError(f"Request timed out: {str(e)}") + + except aiohttp.ClientConnectorError as e: + await self.hooks['on_error'](e) + raise ConnectionError(f"Connection failed: {str(e)}") + + except aiohttp.ClientError as e: + await self.hooks['on_error'](e) + raise HTTPCrawlerError(f"HTTP client error: {str(e)}") + + except asyncio.exceptions.TimeoutError as e: + await self.hooks['on_error'](e) + raise ConnectionTimeoutError(f"Request timed out: {str(e)}") + + except Exception as e: + await self.hooks['on_error'](e) + raise HTTPCrawlerError(f"HTTP request failed: {str(e)}") + + async def crawl( + self, + url: str, + config: Optional[CrawlerRunConfig] = None, + **kwargs + ) -> AsyncCrawlResponse: + config = config or CrawlerRunConfig.from_kwargs(kwargs) + + parsed = urlparse(url) + scheme = parsed.scheme.rstrip('/') + + if scheme not in self.VALID_SCHEMES: + raise ValueError(f"Unsupported URL scheme: {scheme}") + + try: + if scheme == 'file': + return await self._handle_file(parsed.path) + elif scheme == 'raw': + # Don't use parsed.path - urlparse truncates at '#' which is common in CSS + # Strip prefix directly: "raw://" (6 chars) or "raw:" (4 chars) + raw_content = url[6:] if url.startswith("raw://") else url[4:] + return await self._handle_raw(raw_content, base_url=config.base_url) + else: # http or https + return await self._handle_http(url, config) + + except Exception as e: + if self.logger: + self.logger.error( + message="Crawl failed: {error}", + tag="CRAWL", + params={"error": str(e), "url": url} + ) + raise diff --git a/crawl4ai/async_database.py b/crawl4ai/async_database.py new file mode 100644 index 0000000..7579053 --- /dev/null +++ b/crawl4ai/async_database.py @@ -0,0 +1,678 @@ +import os +import time +from pathlib import Path +import aiosqlite +import asyncio +from typing import Optional, Dict +from contextlib import asynccontextmanager +import json +from .models import CrawlResult, MarkdownGenerationResult, StringCompatibleMarkdown +import aiofiles +from .async_logger import AsyncLogger + +from .utils import ensure_content_dirs, generate_content_hash +from .utils import VersionManager +from .utils import get_error_context, create_box_message + +base_directory = DB_PATH = os.path.join( + os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai" +) +os.makedirs(DB_PATH, exist_ok=True) +DB_PATH = os.path.join(base_directory, "crawl4ai.db") + + +class AsyncDatabaseManager: + def __init__(self, pool_size: int = 10, max_retries: int = 3): + self.db_path = DB_PATH + self.content_paths = ensure_content_dirs(os.path.dirname(DB_PATH)) + self.pool_size = pool_size + self.max_retries = max_retries + self.connection_pool: Dict[int, aiosqlite.Connection] = {} + self.pool_lock = asyncio.Lock() + self.init_lock = asyncio.Lock() + self.connection_semaphore = asyncio.Semaphore(pool_size) + self._initialized = False + self.version_manager = VersionManager() + self.logger = AsyncLogger( + log_file=os.path.join(base_directory, ".crawl4ai", "crawler_db.log"), + verbose=False, + tag_width=10, + ) + + async def initialize(self): + """Initialize the database and connection pool""" + try: + self.logger.info("Initializing database", tag="INIT") + # Ensure the database file exists + os.makedirs(os.path.dirname(self.db_path), exist_ok=True) + + # Check if version update is needed + needs_update = self.version_manager.needs_update() + + # Always ensure base table exists + await self.ainit_db() + + # Verify the table exists + async with aiosqlite.connect(self.db_path, timeout=30.0) as db: + async with db.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='crawled_data'" + ) as cursor: + result = await cursor.fetchone() + if not result: + raise Exception("crawled_data table was not created") + + # If version changed or fresh install, run updates + if needs_update: + self.logger.info("New version detected, running updates", tag="INIT") + await self.update_db_schema() + from .migrations import ( + run_migration, + ) # Import here to avoid circular imports + + await run_migration() + self.version_manager.update_version() # Update stored version after successful migration + self.logger.success( + "Version update completed successfully", tag="COMPLETE" + ) + else: + self.logger.success( + "Database initialization completed successfully", tag="COMPLETE" + ) + + except Exception as e: + self.logger.error( + message="Database initialization error: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + self.logger.info( + message="Database will be initialized on first use", tag="INIT" + ) + + raise + + async def cleanup(self): + """Cleanup connections when shutting down""" + async with self.pool_lock: + for conn in self.connection_pool.values(): + await conn.close() + self.connection_pool.clear() + + @asynccontextmanager + async def get_connection(self): + """Connection pool manager with enhanced error handling""" + if not self._initialized: + async with self.init_lock: + if not self._initialized: + try: + await self.initialize() + self._initialized = True + except Exception as e: + import sys + + error_context = get_error_context(sys.exc_info()) + self.logger.error( + message="Database initialization failed:\n{error}\n\nContext:\n{context}\n\nTraceback:\n{traceback}", + tag="ERROR", + force_verbose=True, + params={ + "error": str(e), + "context": error_context["code_context"], + "traceback": error_context["full_traceback"], + }, + ) + raise + + await self.connection_semaphore.acquire() + task_id = id(asyncio.current_task()) + + try: + async with self.pool_lock: + if task_id not in self.connection_pool: + try: + conn = await aiosqlite.connect(self.db_path, timeout=30.0) + await conn.execute("PRAGMA journal_mode = WAL") + await conn.execute("PRAGMA busy_timeout = 5000") + + # Verify database structure + async with conn.execute( + "PRAGMA table_info(crawled_data)" + ) as cursor: + columns = await cursor.fetchall() + column_names = [col[1] for col in columns] + expected_columns = { + "url", + "html", + "cleaned_html", + "markdown", + "extracted_content", + "success", + "media", + "links", + "metadata", + "screenshot", + "response_headers", + "downloaded_files", + } + missing_columns = expected_columns - set(column_names) + if missing_columns: + raise ValueError( + f"Database missing columns: {missing_columns}" + ) + + self.connection_pool[task_id] = conn + except Exception as e: + import sys + + error_context = get_error_context(sys.exc_info()) + error_message = ( + f"Unexpected error in db get_connection at line {error_context['line_no']} " + f"in {error_context['function']} ({error_context['filename']}):\n" + f"Error: {str(e)}\n\n" + f"Code context:\n{error_context['code_context']}" + ) + self.logger.error( + message="{error}", + tag="ERROR", + params={"error": str(error_message)}, + boxes=["error"], + ) + + raise + + yield self.connection_pool[task_id] + + except Exception as e: + import sys + + error_context = get_error_context(sys.exc_info()) + error_message = ( + f"Unexpected error in db get_connection at line {error_context['line_no']} " + f"in {error_context['function']} ({error_context['filename']}):\n" + f"Error: {str(e)}\n\n" + f"Code context:\n{error_context['code_context']}" + ) + self.logger.error( + message="{error}", + tag="ERROR", + params={"error": str(error_message)}, + boxes=["error"], + ) + raise + finally: + async with self.pool_lock: + if task_id in self.connection_pool: + await self.connection_pool[task_id].close() + del self.connection_pool[task_id] + self.connection_semaphore.release() + + async def execute_with_retry(self, operation, *args): + """Execute database operations with retry logic""" + for attempt in range(self.max_retries): + try: + async with self.get_connection() as db: + result = await operation(db, *args) + await db.commit() + return result + except Exception as e: + if attempt == self.max_retries - 1: + self.logger.error( + message="Operation failed after {retries} attempts: {error}", + tag="ERROR", + force_verbose=True, + params={"retries": self.max_retries, "error": str(e)}, + ) + raise + await asyncio.sleep(1 * (attempt + 1)) # Exponential backoff + + async def ainit_db(self): + """Initialize database schema""" + async with aiosqlite.connect(self.db_path, timeout=30.0) as db: + await db.execute( + """ + CREATE TABLE IF NOT EXISTS crawled_data ( + url TEXT PRIMARY KEY, + html TEXT, + cleaned_html TEXT, + markdown TEXT, + extracted_content TEXT, + success BOOLEAN, + media TEXT DEFAULT "{}", + links TEXT DEFAULT "{}", + metadata TEXT DEFAULT "{}", + screenshot TEXT DEFAULT "", + response_headers TEXT DEFAULT "{}", + downloaded_files TEXT DEFAULT "{}" -- New column added + ) + """ + ) + await db.commit() + + async def update_db_schema(self): + """Update database schema if needed""" + async with aiosqlite.connect(self.db_path, timeout=30.0) as db: + cursor = await db.execute("PRAGMA table_info(crawled_data)") + columns = await cursor.fetchall() + column_names = [column[1] for column in columns] + + # List of new columns to add + new_columns = [ + "media", + "links", + "metadata", + "screenshot", + "response_headers", + "downloaded_files", + # Smart cache validation columns (added in 0.8.x) + "etag", + "last_modified", + "head_fingerprint", + "cached_at", + ] + + for column in new_columns: + if column not in column_names: + await self.aalter_db_add_column(column, db) + await db.commit() + + async def aalter_db_add_column(self, new_column: str, db): + """Add new column to the database""" + if new_column == "response_headers": + await db.execute( + f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT "{{}}"' + ) + elif new_column == "cached_at": + # Timestamp column for cache validation + await db.execute( + f"ALTER TABLE crawled_data ADD COLUMN {new_column} REAL DEFAULT 0" + ) + else: + await db.execute( + f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""' + ) + self.logger.info( + message="Added column '{column}' to the database", + tag="INIT", + params={"column": new_column}, + ) + + async def aget_cached_url(self, url: str) -> Optional[CrawlResult]: + """Retrieve cached URL data as CrawlResult""" + + async def _get(db): + async with db.execute( + "SELECT * FROM crawled_data WHERE url = ?", (url,) + ) as cursor: + row = await cursor.fetchone() + if not row: + return None + + # Get column names + columns = [description[0] for description in cursor.description] + # Create dict from row data + row_dict = dict(zip(columns, row)) + + # Load content from files using stored hashes + content_fields = { + "html": row_dict["html"], + "cleaned_html": row_dict["cleaned_html"], + "markdown": row_dict["markdown"], + "extracted_content": row_dict["extracted_content"], + "screenshot": row_dict["screenshot"], + "screenshots": row_dict["screenshot"], + } + + for field, hash_value in content_fields.items(): + if hash_value: + content = await self._load_content( + hash_value, + field.split("_")[0], # Get content type from field name + ) + row_dict[field] = content or "" + else: + row_dict[field] = "" + + # Parse JSON fields + json_fields = [ + "media", + "links", + "metadata", + "response_headers", + "markdown", + ] + for field in json_fields: + try: + row_dict[field] = ( + json.loads(row_dict[field]) if row_dict[field] else {} + ) + except json.JSONDecodeError: + # Very UGLY, never mention it to me please + if field == "markdown" and isinstance(row_dict[field], str): + row_dict[field] = MarkdownGenerationResult( + raw_markdown=row_dict[field] or "", + markdown_with_citations="", + references_markdown="", + fit_markdown="", + fit_html="", + ) + else: + row_dict[field] = {} + + if isinstance(row_dict["markdown"], Dict): + if row_dict["markdown"].get("raw_markdown"): + row_dict["markdown"] = row_dict["markdown"]["raw_markdown"] + + # Parse downloaded_files + try: + row_dict["downloaded_files"] = ( + json.loads(row_dict["downloaded_files"]) + if row_dict["downloaded_files"] + else [] + ) + except json.JSONDecodeError: + row_dict["downloaded_files"] = [] + + # Remove any fields not in CrawlResult model + valid_fields = CrawlResult.__annotations__.keys() + filtered_dict = {k: v for k, v in row_dict.items() if k in valid_fields} + filtered_dict["markdown"] = row_dict["markdown"] + return CrawlResult(**filtered_dict) + + try: + return await self.execute_with_retry(_get) + except Exception as e: + self.logger.error( + message="Error retrieving cached URL: {error}", + tag="ERROR", + force_verbose=True, + params={"error": str(e)}, + ) + return None + + async def aget_cache_metadata(self, url: str) -> Optional[Dict]: + """ + Retrieve only cache validation metadata for a URL (lightweight query). + + Returns dict with: url, etag, last_modified, head_fingerprint, cached_at, response_headers + This is used for cache validation without loading full content. + """ + async def _get_metadata(db): + async with db.execute( + """SELECT url, etag, last_modified, head_fingerprint, cached_at, response_headers + FROM crawled_data WHERE url = ?""", + (url,) + ) as cursor: + row = await cursor.fetchone() + if not row: + return None + + columns = [description[0] for description in cursor.description] + row_dict = dict(zip(columns, row)) + + # Parse response_headers JSON + try: + row_dict["response_headers"] = ( + json.loads(row_dict["response_headers"]) + if row_dict["response_headers"] else {} + ) + except json.JSONDecodeError: + row_dict["response_headers"] = {} + + return row_dict + + try: + return await self.execute_with_retry(_get_metadata) + except Exception as e: + self.logger.error( + message="Error retrieving cache metadata: {error}", + tag="ERROR", + force_verbose=True, + params={"error": str(e)}, + ) + return None + + async def aupdate_cache_metadata( + self, + url: str, + etag: Optional[str] = None, + last_modified: Optional[str] = None, + head_fingerprint: Optional[str] = None, + ): + """ + Update only the cache validation metadata for a URL. + Used to update etag/last_modified after a successful validation. + """ + async def _update(db): + updates = [] + values = [] + + if etag is not None: + updates.append("etag = ?") + values.append(etag) + if last_modified is not None: + updates.append("last_modified = ?") + values.append(last_modified) + if head_fingerprint is not None: + updates.append("head_fingerprint = ?") + values.append(head_fingerprint) + + if not updates: + return + + values.append(url) + await db.execute( + f"UPDATE crawled_data SET {', '.join(updates)} WHERE url = ?", + tuple(values) + ) + + try: + await self.execute_with_retry(_update) + except Exception as e: + self.logger.error( + message="Error updating cache metadata: {error}", + tag="ERROR", + force_verbose=True, + params={"error": str(e)}, + ) + + async def acache_url(self, result: CrawlResult): + """Cache CrawlResult data""" + # Store content files and get hashes + content_map = { + "html": (result.html, "html"), + "cleaned_html": (result.cleaned_html or "", "cleaned"), + "markdown": None, + "extracted_content": (result.extracted_content or "", "extracted"), + "screenshot": (result.screenshot or "", "screenshots"), + } + + try: + if isinstance(result.markdown, StringCompatibleMarkdown): + content_map["markdown"] = ( + result.markdown, + "markdown", + ) + elif isinstance(result.markdown, MarkdownGenerationResult): + content_map["markdown"] = ( + result.markdown.model_dump_json(), + "markdown", + ) + elif isinstance(result.markdown, str): + markdown_result = MarkdownGenerationResult(raw_markdown=result.markdown) + content_map["markdown"] = ( + markdown_result.model_dump_json(), + "markdown", + ) + else: + content_map["markdown"] = ( + MarkdownGenerationResult().model_dump_json(), + "markdown", + ) + except Exception as e: + self.logger.warning( + message=f"Error processing markdown content: {str(e)}", tag="WARNING" + ) + # Fallback to empty markdown result + content_map["markdown"] = ( + MarkdownGenerationResult().model_dump_json(), + "markdown", + ) + + content_hashes = {} + for field, (content, content_type) in content_map.items(): + content_hashes[field] = await self._store_content(content, content_type) + + # Extract cache validation headers from response + response_headers = result.response_headers or {} + etag = response_headers.get("etag") or response_headers.get("ETag") or "" + last_modified = response_headers.get("last-modified") or response_headers.get("Last-Modified") or "" + # head_fingerprint is set by caller via result attribute (if available) + head_fingerprint = getattr(result, "head_fingerprint", None) or "" + cached_at = time.time() + + async def _cache(db): + await db.execute( + """ + INSERT INTO crawled_data ( + url, html, cleaned_html, markdown, + extracted_content, success, media, links, metadata, + screenshot, response_headers, downloaded_files, + etag, last_modified, head_fingerprint, cached_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(url) DO UPDATE SET + html = excluded.html, + cleaned_html = excluded.cleaned_html, + markdown = excluded.markdown, + extracted_content = excluded.extracted_content, + success = excluded.success, + media = excluded.media, + links = excluded.links, + metadata = excluded.metadata, + screenshot = excluded.screenshot, + response_headers = excluded.response_headers, + downloaded_files = excluded.downloaded_files, + etag = excluded.etag, + last_modified = excluded.last_modified, + head_fingerprint = excluded.head_fingerprint, + cached_at = excluded.cached_at + """, + ( + result.url, + content_hashes["html"], + content_hashes["cleaned_html"], + content_hashes["markdown"], + content_hashes["extracted_content"], + result.success, + json.dumps(result.media), + json.dumps(result.links), + json.dumps(result.metadata or {}), + content_hashes["screenshot"], + json.dumps(result.response_headers or {}), + json.dumps(result.downloaded_files or []), + etag, + last_modified, + head_fingerprint, + cached_at, + ), + ) + + try: + await self.execute_with_retry(_cache) + except Exception as e: + self.logger.error( + message="Error caching URL: {error}", + tag="ERROR", + force_verbose=True, + params={"error": str(e)}, + ) + + async def aget_total_count(self) -> int: + """Get total number of cached URLs""" + + async def _count(db): + async with db.execute("SELECT COUNT(*) FROM crawled_data") as cursor: + result = await cursor.fetchone() + return result[0] if result else 0 + + try: + return await self.execute_with_retry(_count) + except Exception as e: + self.logger.error( + message="Error getting total count: {error}", + tag="ERROR", + force_verbose=True, + params={"error": str(e)}, + ) + return 0 + + async def aclear_db(self): + """Clear all data from the database""" + + async def _clear(db): + await db.execute("DELETE FROM crawled_data") + + try: + await self.execute_with_retry(_clear) + except Exception as e: + self.logger.error( + message="Error clearing database: {error}", + tag="ERROR", + force_verbose=True, + params={"error": str(e)}, + ) + + async def aflush_db(self): + """Drop the entire table""" + + async def _flush(db): + await db.execute("DROP TABLE IF EXISTS crawled_data") + + try: + await self.execute_with_retry(_flush) + except Exception as e: + self.logger.error( + message="Error flushing database: {error}", + tag="ERROR", + force_verbose=True, + params={"error": str(e)}, + ) + + async def _store_content(self, content: str, content_type: str) -> str: + """Store content in filesystem and return hash""" + if not content: + return "" + + content_hash = generate_content_hash(content) + file_path = os.path.join(self.content_paths[content_type], content_hash) + + # Only write if file doesn't exist + if not os.path.exists(file_path): + async with aiofiles.open(file_path, "w", encoding="utf-8") as f: + await f.write(content) + + return content_hash + + async def _load_content( + self, content_hash: str, content_type: str + ) -> Optional[str]: + """Load content from filesystem by hash""" + if not content_hash: + return None + + file_path = os.path.join(self.content_paths[content_type], content_hash) + try: + async with aiofiles.open(file_path, "r", encoding="utf-8") as f: + return await f.read() + except: + self.logger.error( + message="Failed to load content: {file_path}", + tag="ERROR", + force_verbose=True, + params={"file_path": file_path}, + ) + return None + + +# Create a singleton instance +async_db_manager = AsyncDatabaseManager() diff --git a/crawl4ai/async_dispatcher.py b/crawl4ai/async_dispatcher.py new file mode 100644 index 0000000..1d6a236 --- /dev/null +++ b/crawl4ai/async_dispatcher.py @@ -0,0 +1,772 @@ +from typing import Dict, Optional, List, Tuple, Union +from .async_configs import CrawlerRunConfig +from .models import ( + CrawlResult, + CrawlerTaskResult, + CrawlStatus, + DomainState, +) + +from .components.crawler_monitor import CrawlerMonitor + +from .types import AsyncWebCrawler + +from collections.abc import AsyncGenerator + +import time +import psutil +import asyncio +import uuid + +from urllib.parse import urlparse +import random +from abc import ABC, abstractmethod + +from .utils import get_true_memory_usage_percent + + +class RateLimiter: + def __init__( + self, + base_delay: Tuple[float, float] = (1.0, 3.0), + max_delay: float = 60.0, + max_retries: int = 3, + rate_limit_codes: List[int] = None, + ): + self.base_delay = base_delay + self.max_delay = max_delay + self.max_retries = max_retries + self.rate_limit_codes = rate_limit_codes or [429, 503] + self.domains: Dict[str, DomainState] = {} + + def get_domain(self, url: str) -> str: + return urlparse(url).netloc + + async def wait_if_needed(self, url: str) -> None: + domain = self.get_domain(url) + state = self.domains.get(domain) + + if not state: + self.domains[domain] = DomainState() + state = self.domains[domain] + + now = time.time() + if state.last_request_time: + wait_time = max(0, state.current_delay - (now - state.last_request_time)) + if wait_time > 0: + await asyncio.sleep(wait_time) + + # Random delay within base range if no current delay + if state.current_delay == 0: + state.current_delay = random.uniform(*self.base_delay) + + state.last_request_time = time.time() + + def update_delay(self, url: str, status_code: int) -> bool: + domain = self.get_domain(url) + state = self.domains[domain] + + if status_code in self.rate_limit_codes: + state.fail_count += 1 + if state.fail_count > self.max_retries: + return False + + # Exponential backoff with random jitter + state.current_delay = min( + state.current_delay * 2 * random.uniform(0.75, 1.25), self.max_delay + ) + else: + # Gradually reduce delay on success + state.current_delay = max( + random.uniform(*self.base_delay), state.current_delay * 0.75 + ) + state.fail_count = 0 + + return True + + + +class BaseDispatcher(ABC): + def __init__( + self, + rate_limiter: Optional[RateLimiter] = None, + monitor: Optional[CrawlerMonitor] = None, + ): + self.crawler = None + self._domain_last_hit: Dict[str, float] = {} + self.concurrent_sessions = 0 + self.rate_limiter = rate_limiter + self.monitor = monitor + + def select_config(self, url: str, configs: Union[CrawlerRunConfig, List[CrawlerRunConfig]]) -> Optional[CrawlerRunConfig]: + """Select the appropriate config for a given URL. + + Args: + url: The URL to match against + configs: Single config or list of configs to choose from + + Returns: + The matching config, or None if no match found + """ + # Single config - return as is + if isinstance(configs, CrawlerRunConfig): + return configs + + # Empty list - return None + if not configs: + return None + + # Find first matching config + for config in configs: + if config.is_match(url): + return config + + # No match found - return None to indicate URL should be skipped + return None + + @abstractmethod + async def crawl_url( + self, + url: str, + config: Union[CrawlerRunConfig, List[CrawlerRunConfig]], + task_id: str, + monitor: Optional[CrawlerMonitor] = None, + ) -> CrawlerTaskResult: + pass + + @abstractmethod + async def run_urls( + self, + urls: List[str], + crawler: AsyncWebCrawler, # noqa: F821 + config: Union[CrawlerRunConfig, List[CrawlerRunConfig]], + monitor: Optional[CrawlerMonitor] = None, + ) -> List[CrawlerTaskResult]: + pass + + +class MemoryAdaptiveDispatcher(BaseDispatcher): + def __init__( + self, + memory_threshold_percent: float = 90.0, + critical_threshold_percent: float = 95.0, # New critical threshold + recovery_threshold_percent: float = 85.0, # New recovery threshold + check_interval: float = 1.0, + max_session_permit: int = 20, + fairness_timeout: float = 600.0, # 10 minutes before prioritizing long-waiting URLs + memory_wait_timeout: Optional[float] = 600.0, + rate_limiter: Optional[RateLimiter] = None, + monitor: Optional[CrawlerMonitor] = None, + ): + super().__init__(rate_limiter, monitor) + self.memory_threshold_percent = memory_threshold_percent + self.critical_threshold_percent = critical_threshold_percent + self.recovery_threshold_percent = recovery_threshold_percent + self.check_interval = check_interval + self.max_session_permit = max_session_permit + self.fairness_timeout = fairness_timeout + self.memory_wait_timeout = memory_wait_timeout + self.result_queue = asyncio.Queue() + self.task_queue = asyncio.PriorityQueue() # Priority queue for better management + self.memory_pressure_mode = False # Flag to indicate when we're in memory pressure mode + self.current_memory_percent = 0.0 # Track current memory usage + self._high_memory_start_time: Optional[float] = None + + async def _memory_monitor_task(self): + """Background task to continuously monitor memory usage and update state""" + while True: + self.current_memory_percent = get_true_memory_usage_percent() + + # Enter memory pressure mode if we cross the threshold + if self.current_memory_percent >= self.memory_threshold_percent: + if not self.memory_pressure_mode: + self.memory_pressure_mode = True + self._high_memory_start_time = time.time() + if self.monitor: + self.monitor.update_memory_status("PRESSURE") + else: + if self._high_memory_start_time is None: + self._high_memory_start_time = time.time() + if ( + self.memory_wait_timeout is not None + and self._high_memory_start_time is not None + and time.time() - self._high_memory_start_time >= self.memory_wait_timeout + ): + raise MemoryError( + "Memory usage exceeded threshold for" + f" {self.memory_wait_timeout} seconds" + ) + + # Exit memory pressure mode if we go below recovery threshold + elif self.memory_pressure_mode and self.current_memory_percent <= self.recovery_threshold_percent: + self.memory_pressure_mode = False + self._high_memory_start_time = None + if self.monitor: + self.monitor.update_memory_status("NORMAL") + elif self.current_memory_percent < self.memory_threshold_percent: + self._high_memory_start_time = None + + # In critical mode, we might need to take more drastic action + if self.current_memory_percent >= self.critical_threshold_percent: + if self.monitor: + self.monitor.update_memory_status("CRITICAL") + # We could implement additional memory-saving measures here + + await asyncio.sleep(self.check_interval) + + def _get_priority_score(self, wait_time: float, retry_count: int) -> float: + """Calculate priority score (lower is higher priority) + - URLs waiting longer than fairness_timeout get higher priority + - More retry attempts decreases priority + """ + if wait_time > self.fairness_timeout: + # High priority for long-waiting URLs + return -wait_time + # Standard priority based on retries + return retry_count + + async def crawl_url( + self, + url: str, + config: Union[CrawlerRunConfig, List[CrawlerRunConfig]], + task_id: str, + retry_count: int = 0, + ) -> CrawlerTaskResult: + start_time = time.time() + error_message = "" + memory_usage = peak_memory = 0.0 + + # Select appropriate config for this URL + selected_config = self.select_config(url, config) + + # If no config matches, return failed result + if selected_config is None: + error_message = f"No matching configuration found for URL: {url}" + if self.monitor: + self.monitor.update_task( + task_id, + status=CrawlStatus.FAILED, + error_message=error_message + ) + + return CrawlerTaskResult( + task_id=task_id, + url=url, + result=CrawlResult( + url=url, + html="", + metadata={"status": "no_config_match"}, + success=False, + error_message=error_message + ), + memory_usage=0, + peak_memory=0, + start_time=start_time, + end_time=time.time(), + error_message=error_message, + retry_count=retry_count + ) + + # Get starting memory for accurate measurement + process = psutil.Process() + start_memory = process.memory_info().rss / (1024 * 1024) + + try: + if self.monitor: + self.monitor.update_task( + task_id, + status=CrawlStatus.IN_PROGRESS, + start_time=start_time, + retry_count=retry_count + ) + + self.concurrent_sessions += 1 + + if self.rate_limiter: + await self.rate_limiter.wait_if_needed(url) + + # Check if we're in critical memory state + if self.current_memory_percent >= self.critical_threshold_percent: + # Requeue this task with increased priority and retry count + enqueue_time = time.time() + priority = self._get_priority_score(enqueue_time - start_time, retry_count + 1) + await self.task_queue.put((priority, (url, task_id, retry_count + 1, enqueue_time))) + + # Update monitoring + if self.monitor: + self.monitor.update_task( + task_id, + status=CrawlStatus.QUEUED, + error_message="Requeued due to critical memory pressure" + ) + + # Return placeholder result with requeued status + return CrawlerTaskResult( + task_id=task_id, + url=url, + result=CrawlResult( + url=url, html="", metadata={"status": "requeued"}, + success=False, error_message="Requeued due to critical memory pressure" + ), + memory_usage=0, + peak_memory=0, + start_time=start_time, + end_time=time.time(), + error_message="Requeued due to critical memory pressure", + retry_count=retry_count + 1 + ) + + # Execute the crawl with selected config + result = await self.crawler.arun(url, config=selected_config, session_id=task_id) + + # Measure memory usage + end_memory = process.memory_info().rss / (1024 * 1024) + memory_usage = peak_memory = end_memory - start_memory + + # Handle rate limiting + if self.rate_limiter and result.status_code: + if not self.rate_limiter.update_delay(url, result.status_code): + error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}" + if self.monitor: + self.monitor.update_task(task_id, status=CrawlStatus.FAILED) + + # Update status based on result + if not result.success: + error_message = result.error_message + if self.monitor: + self.monitor.update_task(task_id, status=CrawlStatus.FAILED) + elif self.monitor: + self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED) + + except Exception as e: + error_message = str(e) + if self.monitor: + self.monitor.update_task(task_id, status=CrawlStatus.FAILED) + result = CrawlResult( + url=url, html="", metadata={}, success=False, error_message=str(e) + ) + + finally: + end_time = time.time() + if self.monitor: + self.monitor.update_task( + task_id, + end_time=end_time, + memory_usage=memory_usage, + peak_memory=peak_memory, + error_message=error_message, + retry_count=retry_count + ) + self.concurrent_sessions -= 1 + + return CrawlerTaskResult( + task_id=task_id, + url=url, + result=result, + memory_usage=memory_usage, + peak_memory=peak_memory, + start_time=start_time, + end_time=end_time, + error_message=error_message, + retry_count=retry_count + ) + + async def run_urls( + self, + urls: List[str], + crawler: AsyncWebCrawler, + config: Union[CrawlerRunConfig, List[CrawlerRunConfig]], + ) -> List[CrawlerTaskResult]: + self.crawler = crawler + + # Start the memory monitor task + memory_monitor = asyncio.create_task(self._memory_monitor_task()) + + if self.monitor: + self.monitor.start() + + results = [] + + try: + # Initialize task queue + for url in urls: + task_id = str(uuid.uuid4()) + if self.monitor: + self.monitor.add_task(task_id, url) + # Add to queue with initial priority 0, retry count 0, and current time + await self.task_queue.put((0, (url, task_id, 0, time.time()))) + + active_tasks = [] + + # Process until both queues are empty + while not self.task_queue.empty() or active_tasks: + if memory_monitor.done(): + exc = memory_monitor.exception() + if exc: + for t in active_tasks: + t.cancel() + raise exc + + # If memory pressure is low, greedily fill all available slots + if not self.memory_pressure_mode: + slots = self.max_session_permit - len(active_tasks) + while slots > 0: + try: + # Use get_nowait() to immediately get tasks without blocking + priority, (url, task_id, retry_count, enqueue_time) = self.task_queue.get_nowait() + + # Create and start the task + task = asyncio.create_task( + self.crawl_url(url, config, task_id, retry_count) + ) + active_tasks.append(task) + + # Update waiting time in monitor + if self.monitor: + wait_time = time.time() - enqueue_time + self.monitor.update_task( + task_id, + wait_time=wait_time, + status=CrawlStatus.IN_PROGRESS + ) + + slots -= 1 + + except asyncio.QueueEmpty: + # No more tasks in queue, exit the loop + break + + # Wait for completion even if queue is starved + if active_tasks: + done, pending = await asyncio.wait( + active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED + ) + + # Process completed tasks + for completed_task in done: + result = await completed_task + results.append(result) + + # Update active tasks list + active_tasks = list(pending) + else: + # If no active tasks but still waiting, sleep briefly + await asyncio.sleep(self.check_interval / 2) + + # Update priorities for waiting tasks if needed + await self._update_queue_priorities() + + except Exception as e: + if self.monitor: + self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}") + raise + + finally: + # Clean up + memory_monitor.cancel() + if self.monitor: + self.monitor.stop() + return results + + async def _update_queue_priorities(self): + """Periodically update priorities of items in the queue to prevent starvation""" + # Skip if queue is empty + if self.task_queue.empty(): + return + + # Use a drain-and-refill approach to update all priorities + temp_items = [] + + # Drain the queue (with a safety timeout to prevent blocking) + try: + drain_start = time.time() + while not self.task_queue.empty() and time.time() - drain_start < 5.0: # 5 second safety timeout + try: + # Get item from queue with timeout + priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for( + self.task_queue.get(), timeout=0.1 + ) + + # Calculate new priority based on current wait time + current_time = time.time() + wait_time = current_time - enqueue_time + new_priority = self._get_priority_score(wait_time, retry_count) + + # Store with updated priority + temp_items.append((new_priority, (url, task_id, retry_count, enqueue_time))) + + # Update monitoring stats for this task + if self.monitor and task_id in self.monitor.stats: + self.monitor.update_task(task_id, wait_time=wait_time) + + except asyncio.TimeoutError: + # Queue might be empty or very slow + break + except Exception as e: + # If anything goes wrong, make sure we refill the queue with what we've got + self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}") + + # Calculate queue statistics + if temp_items and self.monitor: + total_queued = len(temp_items) + wait_times = [item[1][3] for item in temp_items] + highest_wait_time = time.time() - min(wait_times) if wait_times else 0 + avg_wait_time = sum(time.time() - t for t in wait_times) / len(wait_times) if wait_times else 0 + + # Update queue statistics in monitor + self.monitor.update_queue_statistics( + total_queued=total_queued, + highest_wait_time=highest_wait_time, + avg_wait_time=avg_wait_time + ) + + # Sort by priority (lowest number = highest priority) + temp_items.sort(key=lambda x: x[0]) + + # Refill the queue with updated priorities + for item in temp_items: + await self.task_queue.put(item) + + async def run_urls_stream( + self, + urls: List[str], + crawler: AsyncWebCrawler, + config: Union[CrawlerRunConfig, List[CrawlerRunConfig]], + ) -> AsyncGenerator[CrawlerTaskResult, None]: + self.crawler = crawler + + # Start the memory monitor task + memory_monitor = asyncio.create_task(self._memory_monitor_task()) + + if self.monitor: + self.monitor.start() + + try: + # Initialize task queue + for url in urls: + task_id = str(uuid.uuid4()) + if self.monitor: + self.monitor.add_task(task_id, url) + # Add to queue with initial priority 0, retry count 0, and current time + await self.task_queue.put((0, (url, task_id, 0, time.time()))) + + active_tasks = [] + completed_count = 0 + total_urls = len(urls) + + while completed_count < total_urls: + if memory_monitor.done(): + exc = memory_monitor.exception() + if exc: + for t in active_tasks: + t.cancel() + raise exc + # If memory pressure is low, greedily fill all available slots + if not self.memory_pressure_mode: + slots = self.max_session_permit - len(active_tasks) + while slots > 0: + try: + # Use get_nowait() to immediately get tasks without blocking + priority, (url, task_id, retry_count, enqueue_time) = self.task_queue.get_nowait() + + # Create and start the task + task = asyncio.create_task( + self.crawl_url(url, config, task_id, retry_count) + ) + active_tasks.append(task) + + # Update waiting time in monitor + if self.monitor: + wait_time = time.time() - enqueue_time + self.monitor.update_task( + task_id, + wait_time=wait_time, + status=CrawlStatus.IN_PROGRESS + ) + + slots -= 1 + + except asyncio.QueueEmpty: + # No more tasks in queue, exit the loop + break + + # Process completed tasks and yield results + if active_tasks: + done, pending = await asyncio.wait( + active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED + ) + + for completed_task in done: + result = await completed_task + + # Only count as completed if it wasn't requeued + if "requeued" not in result.error_message: + completed_count += 1 + yield result + + # Update active tasks list + active_tasks = list(pending) + else: + # If no active tasks but still waiting, sleep briefly + await asyncio.sleep(self.check_interval / 2) + + # Update priorities for waiting tasks if needed + await self._update_queue_priorities() + + finally: + # Clean up + memory_monitor.cancel() + if self.monitor: + self.monitor.stop() + + +class SemaphoreDispatcher(BaseDispatcher): + def __init__( + self, + semaphore_count: int = 5, + max_session_permit: int = 20, + rate_limiter: Optional[RateLimiter] = None, + monitor: Optional[CrawlerMonitor] = None, + ): + super().__init__(rate_limiter, monitor) + self.semaphore_count = semaphore_count + self.max_session_permit = max_session_permit + + async def crawl_url( + self, + url: str, + config: Union[CrawlerRunConfig, List[CrawlerRunConfig]], + task_id: str, + semaphore: asyncio.Semaphore = None, + ) -> CrawlerTaskResult: + start_time = time.time() + error_message = "" + memory_usage = peak_memory = 0.0 + + # Select appropriate config for this URL + selected_config = self.select_config(url, config) + + # If no config matches, return failed result + if selected_config is None: + error_message = f"No matching configuration found for URL: {url}" + if self.monitor: + self.monitor.update_task( + task_id, + status=CrawlStatus.FAILED, + error_message=error_message + ) + + return CrawlerTaskResult( + task_id=task_id, + url=url, + result=CrawlResult( + url=url, + html="", + metadata={"status": "no_config_match"}, + success=False, + error_message=error_message + ), + memory_usage=0, + peak_memory=0, + start_time=start_time, + end_time=time.time(), + error_message=error_message + ) + + try: + if self.monitor: + self.monitor.update_task( + task_id, status=CrawlStatus.IN_PROGRESS, start_time=start_time + ) + + if self.rate_limiter: + await self.rate_limiter.wait_if_needed(url) + + async with semaphore: + process = psutil.Process() + start_memory = process.memory_info().rss / (1024 * 1024) + result = await self.crawler.arun(url, config=selected_config, session_id=task_id) + end_memory = process.memory_info().rss / (1024 * 1024) + + memory_usage = peak_memory = end_memory - start_memory + + if self.rate_limiter and result.status_code: + if not self.rate_limiter.update_delay(url, result.status_code): + error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}" + if self.monitor: + self.monitor.update_task(task_id, status=CrawlStatus.FAILED) + return CrawlerTaskResult( + task_id=task_id, + url=url, + result=result, + memory_usage=memory_usage, + peak_memory=peak_memory, + start_time=start_time, + end_time=time.time(), + error_message=error_message, + ) + + if not result.success: + error_message = result.error_message + if self.monitor: + self.monitor.update_task(task_id, status=CrawlStatus.FAILED) + elif self.monitor: + self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED) + + except Exception as e: + error_message = str(e) + if self.monitor: + self.monitor.update_task(task_id, status=CrawlStatus.FAILED) + result = CrawlResult( + url=url, html="", metadata={}, success=False, error_message=str(e) + ) + + finally: + end_time = time.time() + if self.monitor: + self.monitor.update_task( + task_id, + end_time=end_time, + memory_usage=memory_usage, + peak_memory=peak_memory, + error_message=error_message, + ) + + return CrawlerTaskResult( + task_id=task_id, + url=url, + result=result, + memory_usage=memory_usage, + peak_memory=peak_memory, + start_time=start_time, + end_time=end_time, + error_message=error_message, + ) + + async def run_urls( + self, + crawler: AsyncWebCrawler, # noqa: F821 + urls: List[str], + config: Union[CrawlerRunConfig, List[CrawlerRunConfig]], + ) -> List[CrawlerTaskResult]: + self.crawler = crawler + if self.monitor: + self.monitor.start() + + try: + semaphore = asyncio.Semaphore(self.semaphore_count) + tasks = [] + + for url in urls: + task_id = str(uuid.uuid4()) + if self.monitor: + self.monitor.add_task(task_id, url) + task = asyncio.create_task( + self.crawl_url(url, config, task_id, semaphore) + ) + tasks.append(task) + + return await asyncio.gather(*tasks, return_exceptions=True) + finally: + if self.monitor: + self.monitor.stop() \ No newline at end of file diff --git a/crawl4ai/async_logger.py b/crawl4ai/async_logger.py new file mode 100644 index 0000000..a469731 --- /dev/null +++ b/crawl4ai/async_logger.py @@ -0,0 +1,386 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import Optional, Dict, Any, List +import os +import sys +from datetime import datetime +from urllib.parse import unquote +from rich.console import Console +from rich.text import Text +from .utils import create_box_message + + +class LogLevel(Enum): + DEFAULT = 0 + DEBUG = 1 + INFO = 2 + SUCCESS = 3 + WARNING = 4 + ERROR = 5 + CRITICAL = 6 + ALERT = 7 + NOTICE = 8 + EXCEPTION = 9 + FATAL = 10 + + + def __str__(self): + return self.name.lower() + +class LogColor(str, Enum): + """Enum for log colors.""" + + DEBUG = "bright_black" + INFO = "cyan" + SUCCESS = "green" + WARNING = "yellow" + ERROR = "red" + CYAN = "cyan" + GREEN = "green" + YELLOW = "yellow" + MAGENTA = "magenta" + DIM_MAGENTA = "dim magenta" + RED = "red" + + def __str__(self): + """Automatically convert rich color to string.""" + return self.value + + +class AsyncLoggerBase(ABC): + @abstractmethod + def debug(self, message: str, tag: str = "DEBUG", **kwargs): + pass + + @abstractmethod + def info(self, message: str, tag: str = "INFO", **kwargs): + pass + + @abstractmethod + def success(self, message: str, tag: str = "SUCCESS", **kwargs): + pass + + @abstractmethod + def warning(self, message: str, tag: str = "WARNING", **kwargs): + pass + + @abstractmethod + def error(self, message: str, tag: str = "ERROR", **kwargs): + pass + + @abstractmethod + def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 100): + pass + + @abstractmethod + def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 100): + pass + + +class AsyncLogger(AsyncLoggerBase): + """ + Asynchronous logger with support for colored console output and file logging. + Supports templated messages with colored components. + """ + + DEFAULT_ICONS = { + "INIT": "→", + "READY": "✓", + "FETCH": "↓", + "SCRAPE": "◆", + "EXTRACT": "■", + "COMPLETE": "●", + "ERROR": "×", + "DEBUG": "⋯", + "INFO": "ℹ", + "WARNING": "⚠", + "SUCCESS": "✔", + "CRITICAL": "‼", + "ALERT": "⚡", + "NOTICE": "ℹ", + "EXCEPTION": "❗", + "FATAL": "☠", + "DEFAULT": "•", + } + + DEFAULT_COLORS = { + LogLevel.DEBUG: LogColor.DEBUG, + LogLevel.INFO: LogColor.INFO, + LogLevel.SUCCESS: LogColor.SUCCESS, + LogLevel.WARNING: LogColor.WARNING, + LogLevel.ERROR: LogColor.ERROR, + } + + def __init__( + self, + log_file: Optional[str] = None, + log_level: LogLevel = LogLevel.DEBUG, + tag_width: int = 10, + icons: Optional[Dict[str, str]] = None, + colors: Optional[Dict[LogLevel, LogColor]] = None, + verbose: bool = True, + console: Optional[Console] = None, + ): + """ + Initialize the logger. + + Args: + log_file: Optional file path for logging + log_level: Minimum log level to display + tag_width: Width for tag formatting + icons: Custom icons for different tags + colors: Custom colors for different log levels + verbose: Whether to output to console + console: Optional custom Rich Console instance. Defaults to stderr + so that log output does not pollute stdout-based transports + such as MCP stdio. Pass ``Console(file=sys.stdout)`` to + restore the previous behaviour. + """ + self.log_file = log_file + self.log_level = log_level + self.tag_width = tag_width + self.icons = icons or self.DEFAULT_ICONS + self.colors = colors or self.DEFAULT_COLORS + self.verbose = verbose + # Default to stderr so log lines do not corrupt stdout-based protocols + # (e.g. MCP stdio transport, piped shell commands). + # width=200: Rich's default falls back to 80 in non-TTY contexts + # (e.g. Docker `docker logs`, CI, captured stdout), which truncates + # useful diagnostic lines. TTYs still auto-detect actual terminal + # width; this only lifts the non-TTY fallback cap. + self.console = console if console is not None else Console(stderr=True, width=200) + + # Create log file directory if needed + if log_file: + os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True) + + def _format_tag(self, tag: str) -> str: + """Format a tag with consistent width.""" + return f"[{tag}]".ljust(self.tag_width, ".") + + def _get_icon(self, tag: str) -> str: + """Get the icon for a tag, defaulting to info icon if not found.""" + return self.icons.get(tag, self.icons["INFO"]) + + def _shorten(self, text, length, placeholder="..."): + """Truncate text in the middle if longer than length, or pad if shorter.""" + if len(text) <= length: + return text.ljust(length) # Pad with spaces to reach desired length + half = (length - len(placeholder)) // 2 + shortened = text[:half] + placeholder + text[-half:] + return shortened.ljust(length) # Also pad shortened text to consistent length + + def _write_to_file(self, message: str): + """Write a message to the log file if configured.""" + if self.log_file: + text = Text.from_markup(message) + plain_text = text.plain + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + with open(self.log_file, "a", encoding="utf-8") as f: + f.write(f"[{timestamp}] {plain_text}\n") + + def _log( + self, + level: LogLevel, + message: str, + tag: str, + params: Optional[Dict[str, Any]] = None, + colors: Optional[Dict[str, LogColor]] = None, + boxes: Optional[List[str]] = None, + base_color: Optional[LogColor] = None, + **kwargs, + ): + """ + Core logging method that handles message formatting and output. + + Args: + level: Log level for this message + message: Message template string + tag: Tag for the message + params: Parameters to format into the message + colors: Color overrides for specific parameters + boxes: Box overrides for specific parameters + base_color: Base color for the entire message + """ + if level.value < self.log_level.value: + return + + # avoid conflict with rich formatting + parsed_message = message.replace("[", "[[").replace("]", "]]") + if params: + # FIXME: If there are formatting strings in floating point format, + # this may result in colors and boxes not being applied properly. + # such as {value:.2f}, the value is 0.23333 format it to 0.23, + # but we replace("0.23333", "[color]0.23333[/color]") + formatted_message = parsed_message.format(**params) + for key, value in params.items(): + # value_str may discard `[` and `]`, so we need to replace it. + value_str = str(value).replace("[", "[[").replace("]", "]]") + # check is need apply color + if colors and key in colors: + color_str = f"[{colors[key]}]{value_str}[/{colors[key]}]" + formatted_message = formatted_message.replace(value_str, color_str) + value_str = color_str + + # check is need apply box + if boxes and key in boxes: + formatted_message = formatted_message.replace(value_str, + create_box_message(value_str, type=str(level))) + + else: + formatted_message = parsed_message + + # Construct the full log line + color: LogColor = base_color or self.colors[level] + log_line = f"[{color}]{self._format_tag(tag)} {self._get_icon(tag)} {formatted_message} [/{color}]" + + # Output to console if verbose + if self.verbose or kwargs.get("force_verbose", False): + self.console.print(log_line) + + # Write to file if configured + self._write_to_file(log_line) + + def debug(self, message: str, tag: str = "DEBUG", **kwargs): + """Log a debug message.""" + self._log(LogLevel.DEBUG, message, tag, **kwargs) + + def info(self, message: str, tag: str = "INFO", **kwargs): + """Log an info message.""" + self._log(LogLevel.INFO, message, tag, **kwargs) + + def success(self, message: str, tag: str = "SUCCESS", **kwargs): + """Log a success message.""" + self._log(LogLevel.SUCCESS, message, tag, **kwargs) + + def warning(self, message: str, tag: str = "WARNING", **kwargs): + """Log a warning message.""" + self._log(LogLevel.WARNING, message, tag, **kwargs) + + def critical(self, message: str, tag: str = "CRITICAL", **kwargs): + """Log a critical message.""" + self._log(LogLevel.ERROR, message, tag, **kwargs) + def exception(self, message: str, tag: str = "EXCEPTION", **kwargs): + """Log an exception message.""" + self._log(LogLevel.ERROR, message, tag, **kwargs) + def fatal(self, message: str, tag: str = "FATAL", **kwargs): + """Log a fatal message.""" + self._log(LogLevel.ERROR, message, tag, **kwargs) + def alert(self, message: str, tag: str = "ALERT", **kwargs): + """Log an alert message.""" + self._log(LogLevel.ERROR, message, tag, **kwargs) + def notice(self, message: str, tag: str = "NOTICE", **kwargs): + """Log a notice message.""" + self._log(LogLevel.INFO, message, tag, **kwargs) + + def error(self, message: str, tag: str = "ERROR", **kwargs): + """Log an error message.""" + self._log(LogLevel.ERROR, message, tag, **kwargs) + + def url_status( + self, + url: str, + success: bool, + timing: float, + tag: str = "FETCH", + url_length: int = 100, + ): + """ + Convenience method for logging URL fetch status. + + Args: + url: The URL being processed + success: Whether the operation was successful + timing: Time taken for the operation + tag: Tag for the message + url_length: Maximum length for URL in log + """ + decoded_url = unquote(url) + readable_url = self._shorten(decoded_url, url_length) + self._log( + level=LogLevel.SUCCESS if success else LogLevel.ERROR, + message="{url} | {status} | ⏱: {timing:.2f}s", + tag=tag, + params={ + "url": readable_url, + "status": "✓" if success else "✗", + "timing": timing, + }, + colors={ + "status": LogColor.SUCCESS if success else LogColor.ERROR, + "timing": LogColor.WARNING, + }, + ) + + def error_status( + self, url: str, error: str, tag: str = "ERROR", url_length: int = 50 + ): + """ + Convenience method for logging error status. + + Args: + url: The URL being processed + error: Error message + tag: Tag for the message + url_length: Maximum length for URL in log + """ + decoded_url = unquote(url) + readable_url = self._shorten(decoded_url, url_length) + self._log( + level=LogLevel.ERROR, + message="{url} | Error: {error}", + tag=tag, + params={"url": readable_url, "error": error}, + ) + +class AsyncFileLogger(AsyncLoggerBase): + """ + File-only asynchronous logger that writes logs to a specified file. + """ + + def __init__(self, log_file: str): + """ + Initialize the file logger. + + Args: + log_file: File path for logging + """ + self.log_file = log_file + os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True) + + def _write_to_file(self, level: str, message: str, tag: str): + """Write a message to the log file.""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + with open(self.log_file, "a", encoding="utf-8") as f: + f.write(f"[{timestamp}] [{level}] [{tag}] {message}\n") + + def debug(self, message: str, tag: str = "DEBUG", **kwargs): + """Log a debug message to file.""" + self._write_to_file("DEBUG", message, tag) + + def info(self, message: str, tag: str = "INFO", **kwargs): + """Log an info message to file.""" + self._write_to_file("INFO", message, tag) + + def success(self, message: str, tag: str = "SUCCESS", **kwargs): + """Log a success message to file.""" + self._write_to_file("SUCCESS", message, tag) + + def warning(self, message: str, tag: str = "WARNING", **kwargs): + """Log a warning message to file.""" + self._write_to_file("WARNING", message, tag) + + def error(self, message: str, tag: str = "ERROR", **kwargs): + """Log an error message to file.""" + self._write_to_file("ERROR", message, tag) + + def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 100): + """Log URL fetch status to file.""" + status = "SUCCESS" if success else "FAILED" + message = f"{url[:url_length]}... | Status: {status} | Time: {timing:.2f}s" + self._write_to_file("URL_STATUS", message, tag) + + def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 100): + """Log error status to file.""" + message = f"{url[:url_length]}... | Error: {error}" + self._write_to_file("ERROR", message, tag) diff --git a/crawl4ai/async_url_seeder.py b/crawl4ai/async_url_seeder.py new file mode 100644 index 0000000..22fa8f6 --- /dev/null +++ b/crawl4ai/async_url_seeder.py @@ -0,0 +1,1794 @@ +""" +async_url_seeder.py +Fast async URL discovery for Crawl4AI + +Features +-------- +* Common-Crawl streaming via httpx.AsyncClient (HTTP/2, keep-alive) +* robots.txt → sitemap chain (.gz + nested indexes) via async httpx +* Per-domain CDX result cache on disk (~/.crawl4ai/<index>_<domain>_<hash>.jsonl) +* Optional HEAD-only liveness check +* Optional partial <head> download + meta parsing +* Global hits-per-second rate-limit via asyncio.Semaphore +* Concurrency in the thousands — fine on a single event-loop +""" + +from __future__ import annotations +import aiofiles +import asyncio +import gzip +import hashlib +import io +import json +import os +import pathlib +import re +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Union +from urllib.parse import quote, urljoin + +import httpx +import fnmatch +try: + from lxml import html as lxml_html + from lxml import etree + LXML = True +except ImportError: + LXML = False +try: + import brotli + HAS_BROTLI = True +except ImportError: + HAS_BROTLI = False +try: + import rank_bm25 + HAS_BM25 = True +except ImportError: + HAS_BM25 = False + +# Import AsyncLoggerBase from crawl4ai's logger module +# Assuming crawl4ai/async_logger.py defines AsyncLoggerBase +# You might need to adjust this import based on your exact file structure +# Import AsyncLogger for default if needed +from .async_logger import AsyncLoggerBase, AsyncLogger + +# Import SeedingConfig for type hints +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from .async_configs import SeedingConfig + + +# ────────────────────────────────────────────────────────────────────────── consts +COLLINFO_URL = "https://index.commoncrawl.org/collinfo.json" +# CACHE_DIR = pathlib.Path("~/.crawl4ai").expanduser() # REMOVED: now managed by __init__ +# CACHE_DIR.mkdir(exist_ok=True) # REMOVED: now managed by __init__ +# INDEX_CACHE = CACHE_DIR / "latest_cc_index.txt" # REMOVED: now managed by __init__ +TTL = timedelta(days=7) # Keeping this constant as it's a seeder-specific TTL + +_meta_rx = re.compile( + r'<meta\s+(?:[^>]*?(?:name|property|http-equiv)\s*=\s*["\']?([^"\' >]+)[^>]*?content\s*=\s*["\']?([^"\' >]+)[^>]*?)\/?>', + re.I) +_charset_rx = re.compile(r'<meta\s+[^>]*charset=["\']?([^"\' >]+)', re.I) +_title_rx = re.compile(r'<title>(.*?)', re.I | re.S) +_link_rx = re.compile( + r']*rel=["\']?([^"\' >]+)[^>]*href=["\']?([^"\' >]+)', re.I) + +# ────────────────────────────────────────────────────────────────────────── helpers + + +def _parse_sitemap_lastmod(xml_content: bytes) -> Optional[str]: + """Extract the most recent lastmod from sitemap XML.""" + try: + if LXML: + root = etree.fromstring(xml_content) + # Get all lastmod elements (namespace-agnostic) + lastmods = root.xpath("//*[local-name()='lastmod']/text()") + if lastmods: + # Return the most recent one + return max(lastmods) + except Exception: + pass + return None + + +def _is_cache_valid( + cache_path: pathlib.Path, + ttl_hours: int, + validate_lastmod: bool, + current_lastmod: Optional[str] = None +) -> bool: + """ + Check if sitemap cache is still valid. + + Returns False (invalid) if: + - File doesn't exist + - File is corrupted/unreadable + - TTL expired (if ttl_hours > 0) + - Sitemap lastmod is newer than cache (if validate_lastmod=True) + """ + if not cache_path.exists(): + return False + + try: + with open(cache_path, "r") as f: + data = json.load(f) + + # Check version + if data.get("version") != 1: + return False + + # Check TTL + if ttl_hours > 0: + created_at = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")) + age_hours = (datetime.now(timezone.utc) - created_at).total_seconds() / 3600 + if age_hours > ttl_hours: + return False + + # Check lastmod + if validate_lastmod and current_lastmod: + cached_lastmod = data.get("sitemap_lastmod") + if cached_lastmod and current_lastmod > cached_lastmod: + return False + + # Check URL count (sanity check - if 0, likely corrupted) + if data.get("url_count", 0) == 0: + return False + + return True + + except (json.JSONDecodeError, KeyError, ValueError, IOError): + # Corrupted cache - return False to trigger refetch + return False + + +def _read_cache(cache_path: pathlib.Path) -> List[str]: + """Read URLs from cache file. Returns empty list on error.""" + try: + with open(cache_path, "r") as f: + data = json.load(f) + return data.get("urls", []) + except Exception: + return [] + + +def _write_cache( + cache_path: pathlib.Path, + urls: List[str], + sitemap_url: str, + sitemap_lastmod: Optional[str] +) -> None: + """Write URLs to cache with metadata.""" + data = { + "version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "sitemap_lastmod": sitemap_lastmod, + "sitemap_url": sitemap_url, + "url_count": len(urls), + "urls": urls + } + try: + with open(cache_path, "w") as f: + json.dump(data, f) + except Exception: + pass # Fail silently - cache is optional + + +def _match(url: str, pattern: str) -> bool: + if fnmatch.fnmatch(url, pattern): + return True + canon = url.split("://", 1)[-1] + return (fnmatch.fnmatch(canon, pattern) + or (canon.startswith("www.") and fnmatch.fnmatch(canon[4:], pattern))) + + +def _parse_head(src: str) -> Dict[str, Any]: + if LXML: + try: + if isinstance(src, str): + # strip Unicode, let lxml decode + src = src.encode("utf-8", "replace") + doc = lxml_html.fromstring(src) + except (ValueError, etree.ParserError): + return {} # malformed, bail gracefully + info: Dict[str, Any] = { + "title": (doc.find(".//title").text or "").strip() + if doc.find(".//title") is not None else None, + "charset": None, + "meta": {}, "link": {}, "jsonld": [] + } + for el in doc.xpath(".//meta"): + k = el.attrib.get("name") or el.attrib.get( + "property") or el.attrib.get("http-equiv") + if k: + info["meta"][k.lower()] = el.attrib.get("content", "") + elif "charset" in el.attrib: + info["charset"] = el.attrib["charset"].lower() + for el in doc.xpath(".//link"): + rel_attr = el.attrib.get("rel", "") + if not rel_attr: + continue + # Handle multiple space-separated rel values + rel_values = rel_attr.lower().split() + entry = {a: el.attrib[a] for a in ( + "href", "as", "type", "hreflang") if a in el.attrib} + # Add entry for each rel value + for rel in rel_values: + info["link"].setdefault(rel, []).append(entry) + # Extract JSON-LD structured data + for script in doc.xpath('.//script[@type="application/ld+json"]'): + if script.text: + try: + jsonld_data = json.loads(script.text.strip()) + info["jsonld"].append(jsonld_data) + except json.JSONDecodeError: + pass + # Extract html lang attribute + html_elem = doc.find(".//html") + if html_elem is not None: + info["lang"] = html_elem.attrib.get("lang", "") + return info + # regex fallback + info: Dict[str, Any] = {"title": None, "charset": None, + "meta": {}, "link": {}, "jsonld": [], "lang": ""} + m = _title_rx.search(src) + info["title"] = m.group(1).strip() if m else None + for k, v in _meta_rx.findall(src): + info["meta"][k.lower()] = v + m = _charset_rx.search(src) + info["charset"] = m.group(1).lower() if m else None + for rel, href in _link_rx.findall(src): + info["link"].setdefault(rel.lower(), []).append({"href": href}) + # Try to extract JSON-LD with regex + jsonld_pattern = re.compile( + r']*type=["\']application/ld\+json["\'][^>]*>(.*?)', re.I | re.S) + for match in jsonld_pattern.findall(src): + try: + jsonld_data = json.loads(match.strip()) + info["jsonld"].append(jsonld_data) + except json.JSONDecodeError: + pass + # Try to extract lang attribute + lang_match = re.search(r']*lang=["\']?([^"\' >]+)', src, re.I) + if lang_match: + info["lang"] = lang_match.group(1) + return info + +# ────────────────────────────────────────────────────────────────────────── class + + +class AsyncUrlSeeder: + """ + Async version of UrlSeeder. + Call pattern is await/async for / async with. + + Public coroutines + ----------------- + await seed.urls(...) + returns List[Dict[str,Any]] (url, status, head_data) + await seed.many_urls(...) + returns Dict[str, List[Dict[str,Any]]] + await seed.close() + closes the HTTP client if owned by seeder + + Usage examples + -------------- + # Manual cleanup: + seeder = AsyncUrlSeeder() + try: + urls = await seeder.urls("example.com", config) + finally: + await seeder.close() + + # Using async context manager (recommended): + async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("example.com", config) + + # Reusing existing client: + client = httpx.AsyncClient() + seeder = AsyncUrlSeeder(client=client) + urls = await seeder.urls("example.com", config) + # No need to close seeder, as it doesn't own the client + """ + + def __init__( + self, + ttl: timedelta = TTL, + client: Optional[httpx.AsyncClient] = None, + logger: Optional[AsyncLoggerBase] = None, # NEW: Add logger parameter + # NEW: Add base_directory + base_directory: Optional[Union[str, pathlib.Path]] = None, + cache_root: Optional[Union[str, Path]] = None, + ): + self.ttl = ttl + self._owns_client = client is None # Track if we created the client + self.client = client or httpx.AsyncClient(http2=True, timeout=20, headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) +AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" + }) + self.logger = logger # Store the logger instance + self.base_directory = pathlib.Path(base_directory or os.getenv( + "CRAWL4_AI_BASE_DIRECTORY", Path.home())) # Resolve base_directory + self.cache_dir = self.base_directory / ".crawl4ai" / \ + "seeder_cache" # NEW: Specific cache dir for seeder + self.cache_dir.mkdir(parents=True, exist_ok=True) # Ensure it exists + self.index_cache_path = self.cache_dir / \ + "latest_cc_index.txt" # NEW: Index cache path + + # defer – grabbing the index inside an active loop blows up + self.index_id: Optional[str] = None + self._rate_sem: Optional[asyncio.Semaphore] = None + + # ───────── cache dirs ───────── + self.cache_root = Path(os.path.expanduser( + cache_root or "~/.cache/url_seeder")) + (self.cache_root / "live").mkdir(parents=True, exist_ok=True) + (self.cache_root / "head").mkdir(exist_ok=True) + + def _log(self, level: str, message: str, tag: str = "URL_SEED", **kwargs: Any): + """Helper to log messages using the provided logger, if available.""" + if self.logger: + log_method = getattr(self.logger, level, None) + if log_method: + log_method(message=message, tag=tag, + params=kwargs.get('params', {})) + # else: # Fallback for unknown level, should not happen with AsyncLoggerBase + # print(f"[{tag}] {level.upper()}: {message.format(**kwargs)}") + + # ───────── cache helpers ───────── + def _cache_path(self, kind: str, url: str) -> Path: + h = hashlib.sha1(url.encode()).hexdigest() + return self.cache_root / kind / f"{h}.json" + + async def _cache_get(self, kind: str, url: str) -> Optional[Dict[str, Any]]: + p = self._cache_path(kind, url) + if not p.exists(): + return None + if time.time()-p.stat().st_mtime > self.ttl.total_seconds(): + return None + try: + async with aiofiles.open(p, "r") as f: + return json.loads(await f.read()) + except Exception: + return None + + async def _cache_set(self, kind: str, url: str, data: Dict[str, Any]) -> None: + try: + async with aiofiles.open(self._cache_path(kind, url), "w") as f: + await f.write(json.dumps(data, separators=(",", ":"))) + except Exception: + pass + + # ─────────────────────────────── discovery entry + + async def urls(self, + domain: str, + config: "SeedingConfig", + ) -> List[Dict[str, Any]]: + """ + Fetch URLs for a domain using configuration from SeedingConfig. + + Parameters + ---------- + domain : str + The domain to fetch URLs for (e.g., "example.com") + config : SeedingConfig + Configuration object containing all seeding parameters + """ + # Extract parameters from config + pattern = config.pattern or "*" + source = config.source + live_check = config.live_check + extract_head = config.extract_head + concurrency = config.concurrency + head_timeout = 5 # Default timeout for HEAD requests + hits_per_sec = config.hits_per_sec + self.force = config.force # Store force flag as instance attribute + force = config.force + verbose = config.verbose if config.verbose is not None else ( + self.logger.verbose if self.logger else False) + max_urls = config.max_urls if config.max_urls is not None else -1 + query = config.query + score_threshold = config.score_threshold + scoring_method = config.scoring_method + + # Store cache config for use in _from_sitemaps + self._cache_ttl_hours = getattr(config, 'cache_ttl_hours', 24) + self._validate_sitemap_lastmod = getattr(config, 'validate_sitemap_lastmod', True) + + # Ensure seeder's logger verbose matches the config's verbose if it's set + if self.logger and hasattr(self.logger, 'verbose') and config.verbose is not None: + self.logger.verbose = config.verbose + + # Parse source parameter - split by '+' to get list of sources + sources = [s.strip().lower() for s in source.split("+") if s.strip()] + + valid_sources = {"cc", "sitemap"} + for s in sources: + if s not in valid_sources: + raise ValueError( + f"Invalid source '{s}'. Valid sources are: {', '.join(valid_sources)}") + + # ensure we have the latest CC collection id when the source is cc + if s == "cc" and self.index_id is None: + self.index_id = await self._latest_index() + + + if hits_per_sec: + if hits_per_sec <= 0: + self._log( + "warning", "hits_per_sec must be positive. Disabling rate limiting.", tag="URL_SEED") + self._rate_sem = None + else: + self._rate_sem = asyncio.Semaphore(hits_per_sec) + else: + self._rate_sem = None # Ensure it's None if no rate limiting + + self._log("info", "Starting URL seeding for {domain} with source={source}", + params={"domain": domain, "source": source}, tag="URL_SEED") + + # choose stream + async def gen(): + if "sitemap" in sources: + self._log("debug", "Fetching from sitemaps...", tag="URL_SEED") + async for u in self._from_sitemaps(domain, pattern, force): + yield u + if "cc" in sources: + self._log("debug", "Fetching from Common Crawl...", + tag="URL_SEED") + async for u in self._from_cc(domain, pattern, force): + yield u + + # Use bounded queue to prevent RAM spikes with large domains + queue_size = min(10000, max(1000, concurrency * 100)) # Dynamic size based on concurrency + queue = asyncio.Queue(maxsize=queue_size) + producer_done = asyncio.Event() + stop_event = asyncio.Event() + seen: set[str] = set() + filter_nonsense = config.filter_nonsense_urls # Extract this for passing to workers + + async def producer(): + try: + async for u in gen(): + try: + if u in seen: + self._log("debug", "Skipping duplicate URL: {url}", + params={"url": u}, tag="URL_SEED") + continue + if stop_event.is_set(): + self._log( + "info", "Producer stopping due to max_urls limit.", tag="URL_SEED") + break + seen.add(u) + await queue.put(u) # Will block if queue is full, providing backpressure + except UnicodeEncodeError: + # Skip URLs that cause encoding errors (e.g. on Windows) + continue + except Exception as e: + self._log("error", "Producer encountered an error: {error}", params={ + "error": str(e)}, tag="URL_SEED") + finally: + producer_done.set() + self._log("debug", "Producer finished.", tag="URL_SEED") + + async def worker(res_list: List[Dict[str, Any]]): + while True: + if queue.empty() and producer_done.is_set(): + # self._log("debug", "Worker exiting: queue empty and producer done.", tag="URL_SEED") + break + try: + # Increased timeout slightly + url = await asyncio.wait_for(queue.get(), 5) + except asyncio.TimeoutError: + continue # Keep checking queue and producer_done status + except Exception as e: + self._log("error", "Worker failed to get URL from queue: {error}", params={ + "error": str(e)}, tag="URL_SEED") + continue + + if max_urls > 0 and len(res_list) >= max_urls: + self._log( + "info", + "Worker stopping due to max_urls limit.", + tag="URL_SEED", + ) + stop_event.set() + + # mark the current item done + queue.task_done() + + # flush whatever is still sitting in the queue so + # queue.join() can finish cleanly + while not queue.empty(): + try: + queue.get_nowait() + queue.task_done() + except asyncio.QueueEmpty: + break + break + + if self._rate_sem: # global QPS control + async with self._rate_sem: + await self._validate(url, res_list, live_check, extract_head, + head_timeout, verbose, query, score_threshold, scoring_method, + filter_nonsense) + else: + await self._validate(url, res_list, live_check, extract_head, + head_timeout, verbose, query, score_threshold, scoring_method, + filter_nonsense) + queue.task_done() # Mark task as done for queue.join() if ever used + + # launch + results: List[Dict[str, Any]] = [] + prod_task = asyncio.create_task(producer()) + workers = [asyncio.create_task(worker(results)) + for _ in range(concurrency)] + + # Wait for all workers to finish + await asyncio.gather(prod_task, *workers) + await queue.join() # Ensure all queued items are processed + + self._log("info", "Finished URL seeding for {domain}. Total URLs: {count}", + params={"domain": domain, "count": len(results)}, tag="URL_SEED") + + # Apply BM25 scoring if query was provided + if query and extract_head and scoring_method == "bm25": + # Apply collective BM25 scoring across all documents + results = await self._apply_bm25_scoring(results, config) + + # Filter by score threshold if specified + if score_threshold is not None: + original_count = len(results) + results = [r for r in results if r.get("relevance_score", 0) >= score_threshold] + if original_count > len(results): + self._log("info", "Filtered {filtered} URLs below score threshold {threshold}", + params={"filtered": original_count - len(results), "threshold": score_threshold}, tag="URL_SEED") + + # Sort by relevance score + results.sort(key=lambda x: x.get("relevance_score", 0.0), reverse=True) + self._log("info", "Sorted {count} URLs by relevance score for query: '{query}'", + params={"count": len(results), "query": query}, tag="URL_SEED") + elif query and not extract_head: + self._log( + "warning", "Query provided but extract_head is False. Enable extract_head for relevance scoring.", tag="URL_SEED") + + return results[:max_urls] if max_urls > 0 else results + + async def many_urls( + self, + domains: Sequence[str], + config: "SeedingConfig", + ) -> Dict[str, List[Dict[str, Any]]]: + """ + Fetch URLs for many domains in parallel. + + Parameters + ---------- + domains : Sequence[str] + List of domains to fetch URLs for + config : SeedingConfig + Configuration object containing all seeding parameters + + Returns a {domain: urls-list} dict. + """ + self._log("info", "Starting URL seeding for {count} domains...", + params={"count": len(domains)}, tag="URL_SEED") + + # Ensure seeder's logger verbose matches the config's verbose if it's set + if self.logger and hasattr(self.logger, 'verbose') and config.verbose is not None: + self.logger.verbose = config.verbose + + tasks = [ + self.urls(domain, config) + for domain in domains + ] + results = await asyncio.gather(*tasks) + + final_results = dict(zip(domains, results)) + self._log( + "info", "Finished URL seeding for multiple domains.", tag="URL_SEED") + return final_results + + async def extract_head_for_urls( + self, + urls: List[str], + config: Optional["SeedingConfig"] = None, + concurrency: int = 10, + timeout: int = 5 + ) -> List[Dict[str, Any]]: + """ + Extract head content for a custom list of URLs using URLSeeder's parallel processing. + + This method reuses URLSeeder's efficient parallel processing, caching, and head extraction + logic to process a custom list of URLs rather than discovering URLs from sources. + + Parameters + ---------- + urls : List[str] + List of URLs to extract head content from + config : SeedingConfig, optional + Configuration object. If None, uses default settings for head extraction + concurrency : int, default=10 + Number of concurrent requests + timeout : int, default=5 + Timeout for each request in seconds + + Returns + ------- + List[Dict[str, Any]] + List of dictionaries containing url, status, head_data, and optional relevance_score + """ + # Create default config if none provided + if config is None: + # Import here to avoid circular imports + from .async_configs import SeedingConfig + config = SeedingConfig( + extract_head=True, + concurrency=concurrency, + verbose=False + ) + + # Override concurrency and ensure head extraction is enabled + config.concurrency = concurrency + config.extract_head = True + + self._log("info", "Starting head extraction for {count} custom URLs", + params={"count": len(urls)}, tag="URL_SEED") + + # Setup rate limiting if specified in config + if config.hits_per_sec: + if config.hits_per_sec <= 0: + self._log("warning", "hits_per_sec must be positive. Disabling rate limiting.", tag="URL_SEED") + self._rate_sem = None + else: + self._rate_sem = asyncio.Semaphore(config.hits_per_sec) + else: + self._rate_sem = None + + # Use bounded queue to prevent memory issues with large URL lists + queue_size = min(10000, max(1000, concurrency * 100)) + queue = asyncio.Queue(maxsize=queue_size) + producer_done = asyncio.Event() + stop_event = asyncio.Event() + seen: set[str] = set() + + # Results collection + results: List[Dict[str, Any]] = [] + + async def producer(): + """Producer to feed URLs into the queue.""" + try: + for url in urls: + if url in seen: + self._log("debug", "Skipping duplicate URL: {url}", + params={"url": url}, tag="URL_SEED") + continue + if stop_event.is_set(): + break + seen.add(url) + await queue.put(url) + finally: + producer_done.set() + + async def worker(res_list: List[Dict[str, Any]]): + """Worker to process URLs from the queue.""" + while True: + try: + # Wait for URL or producer completion + url = await asyncio.wait_for(queue.get(), timeout=1.0) + except asyncio.TimeoutError: + if producer_done.is_set() and queue.empty(): + break + continue + + try: + # Use existing _validate method which handles head extraction, caching, etc. + await self._validate( + url, res_list, + live=False, # We're not doing live checks, just head extraction + extract=True, # Always extract head content + timeout=timeout, + verbose=config.verbose or False, + query=config.query, + score_threshold=config.score_threshold, + scoring_method=config.scoring_method or "bm25", + filter_nonsense=config.filter_nonsense_urls + ) + except Exception as e: + self._log("error", "Failed to process URL {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + # Add failed entry to results + res_list.append({ + "url": url, + "status": "failed", + "head_data": {}, + "error": str(e) + }) + finally: + queue.task_done() + + # Start producer + producer_task = asyncio.create_task(producer()) + + # Start workers + worker_tasks = [] + for _ in range(concurrency): + worker_task = asyncio.create_task(worker(results)) + worker_tasks.append(worker_task) + + # Wait for producer to finish + await producer_task + + # Wait for all items to be processed + await queue.join() + + # Cancel workers + for task in worker_tasks: + task.cancel() + + # Wait for workers to finish canceling + await asyncio.gather(*worker_tasks, return_exceptions=True) + + # Apply BM25 scoring if query is provided + if config.query and config.scoring_method == "bm25": + results = await self._apply_bm25_scoring(results, config) + + # Apply score threshold filtering + if config.score_threshold is not None: + results = [r for r in results if r.get("relevance_score", 0) >= config.score_threshold] + + # Sort by relevance score if available + if any("relevance_score" in r for r in results): + results.sort(key=lambda x: x.get("relevance_score", 0), reverse=True) + + self._log("info", "Completed head extraction for {count} URLs, {success} successful", + params={ + "count": len(urls), + "success": len([r for r in results if r.get("status") == "valid"]) + }, tag="URL_SEED") + + return results + + async def _apply_bm25_scoring(self, results: List[Dict[str, Any]], config: "SeedingConfig") -> List[Dict[str, Any]]: + """Apply BM25 scoring to results that have head_data.""" + if not HAS_BM25: + self._log("warning", "BM25 scoring requested but rank_bm25 not available", tag="URL_SEED") + return results + + # Extract text contexts from head data + text_contexts = [] + valid_results = [] + + for result in results: + if result.get("status") == "valid" and result.get("head_data"): + text_context = self._extract_text_context(result["head_data"]) + if text_context: + text_contexts.append(text_context) + valid_results.append(result) + else: + # Use URL-based scoring as fallback + score = self._calculate_url_relevance_score(config.query, result["url"]) + result["relevance_score"] = float(score) + elif result.get("status") == "valid": + # No head data but valid URL - use URL-based scoring + score = self._calculate_url_relevance_score(config.query, result["url"]) + result["relevance_score"] = float(score) + + # Calculate BM25 scores for results with text context + if text_contexts and valid_results: + scores = await asyncio.to_thread(self._calculate_bm25_score, config.query, text_contexts) + for i, result in enumerate(valid_results): + if i < len(scores): + result["relevance_score"] = float(scores[i]) + + return results + + async def _resolve_head(self, url: str) -> Optional[str]: + """ + HEAD-probe a URL. + + Returns: + * the same URL if it answers 2xx, + * the verified absolute redirect target if it answers 3xx + and the target also answers 2xx, + * None on any other status or network error. + """ + try: + r = await self.client.head(url, timeout=10, follow_redirects=False) + + # direct hit + if 200 <= r.status_code < 300: + return str(r.url) + + # single level redirect — verify target is alive + if r.status_code in (301, 302, 303, 307, 308): + loc = r.headers.get("location") + if loc: + target = urljoin(url, loc) + # Guard against self-redirects + if target == url: + return None + try: + r2 = await self.client.head( + target, timeout=10, follow_redirects=False + ) + if 200 <= r2.status_code < 300: + return str(r2.url) + except Exception: + pass + return None + + return None + + except Exception as e: + self._log("debug", "HEAD {url} failed: {err}", + params={"url": url, "err": str(e)}, tag="URL_SEED") + return None + + # ─────────────────────────────── CC + async def _from_cc(self, domain: str, pattern: str, force: bool): + import re + digest = hashlib.md5(pattern.encode()).hexdigest()[:8] + + # ── normalise for CC (strip scheme, query, fragment) + raw = re.sub(r'^https?://', '', domain).split('#', + 1)[0].split('?', 1)[0].lstrip('.') + + # ── sanitize only for cache-file name + safe = re.sub('[/?#]+', '_', raw) + path = self.cache_dir / f"{self.index_id}_{safe}_{digest}.jsonl" + + if path.exists() and not force: + self._log("info", "Loading CC URLs for {domain} from cache: {path}", + params={"domain": domain, "path": path}, tag="URL_SEED") + async with aiofiles.open(path, "r") as fp: + async for line in fp: + url = line.strip() + if _match(url, pattern): + yield url + return + + # build CC glob – if a path is present keep it, else add trailing /* + glob = f"*.{raw}*" if '/' in raw else f"*.{raw}/*" + url = f"https://index.commoncrawl.org/{self.index_id}-index?url={quote(glob, safe='*')}&output=json" + + retries = (1, 3, 7) + self._log("info", "Fetching CC URLs for {domain} from Common Crawl index: {url}", + params={"domain": domain, "url": url}, tag="URL_SEED") + for i, d in enumerate(retries+(-1,)): # last -1 means don't retry + try: + async with self.client.stream("GET", url) as r: + r.raise_for_status() + async with aiofiles.open(path, "w") as fp: + async for line in r.aiter_lines(): + rec = json.loads(line) + u = rec["url"] + await fp.write(u+"\n") + if _match(u, pattern): + yield u + return + except httpx.HTTPStatusError as e: + if e.response.status_code == 503 and i < len(retries): + self._log("warning", "Common Crawl API returned 503 for {domain}. Retrying in {delay}s.", + params={"domain": domain, "delay": retries[i]}, tag="URL_SEED") + await asyncio.sleep(retries[i]) + continue + self._log("error", "HTTP error fetching CC index for {domain}: {error}", + params={"domain": domain, "error": str(e)}, tag="URL_SEED") + raise + except Exception as e: + self._log("error", "Error fetching CC index for {domain}: {error}", + params={"domain": domain, "error": str(e)}, tag="URL_SEED") + raise + + # ─────────────────────────────── Sitemaps + async def _from_sitemaps(self, domain: str, pattern: str, force: bool = False): + """ + Discover URLs from sitemaps with smart TTL-based caching. + + 1. Check cache validity (TTL + lastmod) + 2. If valid, yield from cache + 3. If invalid or force=True, fetch fresh and update cache + 4. FALLBACK: If anything fails, bypass cache and fetch directly + """ + # Get config values (passed via self during urls() call) + cache_ttl_hours = getattr(self, '_cache_ttl_hours', 24) + validate_lastmod = getattr(self, '_validate_sitemap_lastmod', True) + + # Cache file path (new format: .json instead of .jsonl) + host = re.sub(r'^https?://', '', domain).rstrip('/') + host_safe = re.sub('[/?#]+', '_', host) + digest = hashlib.md5(pattern.encode()).hexdigest()[:8] + cache_path = self.cache_dir / f"sitemap_{host_safe}_{digest}.json" + + # Check for old .jsonl format and delete it + old_cache_path = self.cache_dir / f"sitemap_{host_safe}_{digest}.jsonl" + if old_cache_path.exists(): + try: + old_cache_path.unlink() + self._log("info", "Deleted old cache format: {p}", + params={"p": str(old_cache_path)}, tag="URL_SEED") + except Exception: + pass + + # Step 1: Find sitemap URL and get lastmod (needed for validation) + sitemap_url = None + sitemap_lastmod = None + sitemap_content = None + + schemes = ('https', 'http') + for scheme in schemes: + for suffix in ("/sitemap.xml", "/sitemap_index.xml"): + sm = f"{scheme}://{host}{suffix}" + resolved = await self._resolve_head(sm) + if resolved: + sitemap_url = resolved + # Fetch sitemap content to get lastmod + try: + r = await self.client.get(sitemap_url, timeout=15, follow_redirects=True) + if 200 <= r.status_code < 300: + sitemap_content = r.content + sitemap_lastmod = _parse_sitemap_lastmod(sitemap_content) + except Exception: + pass + break + if sitemap_url: + break + + # Step 2: Check cache validity (skip if force=True) + if not force and cache_path.exists(): + if _is_cache_valid(cache_path, cache_ttl_hours, validate_lastmod, sitemap_lastmod): + self._log("info", "Loading sitemap URLs from valid cache: {p}", + params={"p": str(cache_path)}, tag="URL_SEED") + cached_urls = _read_cache(cache_path) + for url in cached_urls: + if _match(url, pattern): + yield url + return + else: + self._log("info", "Cache invalid/expired, refetching sitemap for {d}", + params={"d": domain}, tag="URL_SEED") + + # Step 3: Fetch fresh URLs + discovered_urls = [] + + if sitemap_url and sitemap_content: + self._log("info", "Found sitemap at {url}", params={"url": sitemap_url}, tag="URL_SEED") + + # Parse sitemap (reuse content we already fetched) + async for u in self._iter_sitemap_content(sitemap_url, sitemap_content): + discovered_urls.append(u) + if _match(u, pattern): + yield u + elif sitemap_url: + # We have a sitemap URL but no content (fetch failed earlier), try again + self._log("info", "Found sitemap at {url}", params={"url": sitemap_url}, tag="URL_SEED") + async for u in self._iter_sitemap(sitemap_url): + discovered_urls.append(u) + if _match(u, pattern): + yield u + else: + # Fallback: robots.txt + robots = f"https://{host}/robots.txt" + try: + r = await self.client.get(robots, timeout=10, follow_redirects=True) + if 200 <= r.status_code < 300: + sitemap_lines = [l.split(":", 1)[1].strip() + for l in r.text.splitlines() + if l.lower().startswith("sitemap:")] + for sm in sitemap_lines: + async for u in self._iter_sitemap(sm): + discovered_urls.append(u) + if _match(u, pattern): + yield u + else: + self._log("warning", "robots.txt unavailable for {d} HTTP{c}", + params={"d": domain, "c": r.status_code}, tag="URL_SEED") + return + except Exception as e: + self._log("warning", "Failed to fetch robots.txt for {d}: {e}", + params={"d": domain, "e": str(e)}, tag="URL_SEED") + return + + # Step 4: Write to cache (FALLBACK: if write fails, URLs still yielded above) + if discovered_urls: + _write_cache(cache_path, discovered_urls, sitemap_url or "", sitemap_lastmod) + self._log("info", "Cached {count} URLs for {d}", + params={"count": len(discovered_urls), "d": domain}, tag="URL_SEED") + + async def _iter_sitemap_content(self, url: str, content: bytes): + """Parse sitemap from already-fetched content.""" + data = gzip.decompress(content) if url.endswith(".gz") else content + base_url = url + + def _normalize_loc(raw: Optional[str]) -> Optional[str]: + if not raw: + return None + cleaned = raw.strip().replace("\u200b", "").replace("\ufeff", "") + normalized = urljoin(base_url, cleaned) + if not normalized: + return None + return normalized + + # Detect if this is a sitemap index + is_sitemap_index = False + sub_sitemaps = [] + regular_urls = [] + + if LXML: + try: + parser = etree.XMLParser(recover=True) + root = etree.fromstring(data, parser=parser) + sitemap_loc_nodes = root.xpath("//*[local-name()='sitemap']/*[local-name()='loc']") + url_loc_nodes = root.xpath("//*[local-name()='url']/*[local-name()='loc']") + + if sitemap_loc_nodes: + is_sitemap_index = True + for sitemap_elem in sitemap_loc_nodes: + loc = _normalize_loc(sitemap_elem.text) + if loc: + sub_sitemaps.append(loc) + + if not is_sitemap_index: + for loc_elem in url_loc_nodes: + loc = _normalize_loc(loc_elem.text) + if loc: + regular_urls.append(loc) + except Exception as e: + self._log("error", "LXML parsing error for sitemap {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return + else: + import xml.etree.ElementTree as ET + try: + root = ET.fromstring(data) + for elem in root.iter(): + if '}' in elem.tag: + elem.tag = elem.tag.split('}')[1] + + sitemaps = root.findall('.//sitemap') + url_entries = root.findall('.//url') + + if sitemaps: + is_sitemap_index = True + for sitemap in sitemaps: + loc_elem = sitemap.find('loc') + loc = _normalize_loc(loc_elem.text if loc_elem is not None else None) + if loc: + sub_sitemaps.append(loc) + + if not is_sitemap_index: + for url_elem in url_entries: + loc_elem = url_elem.find('loc') + loc = _normalize_loc(loc_elem.text if loc_elem is not None else None) + if loc: + regular_urls.append(loc) + except Exception as e: + self._log("error", "ElementTree parsing error for sitemap {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return + + # Process based on type + if is_sitemap_index and sub_sitemaps: + self._log("info", "Processing sitemap index with {count} sub-sitemaps", + params={"count": len(sub_sitemaps)}, tag="URL_SEED") + + queue_size = min(50000, len(sub_sitemaps) * 1000) + result_queue = asyncio.Queue(maxsize=queue_size) + completed_count = 0 + total_sitemaps = len(sub_sitemaps) + + async def process_subsitemap(sitemap_url: str): + try: + async for u in self._iter_sitemap(sitemap_url): + await result_queue.put(u) + except Exception as e: + self._log("error", "Error processing sub-sitemap {url}: {error}", + params={"url": sitemap_url, "error": str(e)}, tag="URL_SEED") + finally: + await result_queue.put(None) + + tasks = [asyncio.create_task(process_subsitemap(sm)) for sm in sub_sitemaps] + + while completed_count < total_sitemaps: + item = await result_queue.get() + if item is None: + completed_count += 1 + else: + yield item + + await asyncio.gather(*tasks, return_exceptions=True) + else: + for u in regular_urls: + yield u + + async def _iter_sitemap(self, url: str): + try: + r = await self.client.get(url, timeout=15, follow_redirects=True) + r.raise_for_status() + except httpx.HTTPStatusError as e: + self._log("warning", "Failed to fetch sitemap {url}: HTTP {status_code}", + params={"url": url, "status_code": e.response.status_code}, tag="URL_SEED") + return + except httpx.RequestError as e: + self._log("warning", "Network error fetching sitemap {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return + except Exception as e: + self._log("error", "Unexpected error fetching sitemap {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return + + data = gzip.decompress(r.content) if url.endswith(".gz") else r.content + base_url = str(r.url) + + def _normalize_loc(raw: Optional[str]) -> Optional[str]: + if not raw: + return None + cleaned = raw.strip().replace("\u200b", "").replace("\ufeff", "") + normalized = urljoin(base_url, cleaned) + if not normalized: + return None + return normalized + + # Detect if this is a sitemap index by checking for or presence of elements + is_sitemap_index = False + sub_sitemaps = [] + regular_urls = [] + + # Use lxml for XML parsing if available, as it's generally more robust + if LXML: + try: + # Use XML parser for sitemaps, not HTML parser + parser = etree.XMLParser(recover=True) + root = etree.fromstring(data, parser=parser) + # Namespace-agnostic lookups using local-name() so we honor custom or missing namespaces + sitemap_loc_nodes = root.xpath("//*[local-name()='sitemap']/*[local-name()='loc']") + url_loc_nodes = root.xpath("//*[local-name()='url']/*[local-name()='loc']") + + self._log( + "debug", + "Parsed sitemap {url}: {sitemap_count} sitemap entries, {url_count} url entries discovered", + params={ + "url": url, + "sitemap_count": len(sitemap_loc_nodes), + "url_count": len(url_loc_nodes), + }, + tag="URL_SEED", + ) + + # Check for sitemap index entries + if sitemap_loc_nodes: + is_sitemap_index = True + for sitemap_elem in sitemap_loc_nodes: + loc = _normalize_loc(sitemap_elem.text) + if loc: + sub_sitemaps.append(loc) + + # If not a sitemap index, get regular URLs + if not is_sitemap_index: + for loc_elem in url_loc_nodes: + loc = _normalize_loc(loc_elem.text) + if loc: + regular_urls.append(loc) + if not regular_urls: + self._log( + "warning", + "No entries found inside tags for sitemap {url}. The sitemap might be empty or use an unexpected structure.", + params={"url": url}, + tag="URL_SEED", + ) + except Exception as e: + self._log("error", "LXML parsing error for sitemap {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return + else: # Fallback to xml.etree.ElementTree + import xml.etree.ElementTree as ET + try: + # Parse the XML + root = ET.fromstring(data) + # Remove namespace from tags for easier processing + for elem in root.iter(): + if '}' in elem.tag: + elem.tag = elem.tag.split('}')[1] + + # Check for sitemap index entries + sitemaps = root.findall('.//sitemap') + url_entries = root.findall('.//url') + self._log( + "debug", + "ElementTree parsed sitemap {url}: {sitemap_count} sitemap entries, {url_count} url entries discovered", + params={ + "url": url, + "sitemap_count": len(sitemaps), + "url_count": len(url_entries), + }, + tag="URL_SEED", + ) + if sitemaps: + is_sitemap_index = True + for sitemap in sitemaps: + loc_elem = sitemap.find('loc') + loc = _normalize_loc(loc_elem.text if loc_elem is not None else None) + if loc: + sub_sitemaps.append(loc) + + # If not a sitemap index, get regular URLs + if not is_sitemap_index: + for url_elem in url_entries: + loc_elem = url_elem.find('loc') + loc = _normalize_loc(loc_elem.text if loc_elem is not None else None) + if loc: + regular_urls.append(loc) + if not regular_urls: + self._log( + "warning", + "No entries found inside tags for sitemap {url}. The sitemap might be empty or use an unexpected structure.", + params={"url": url}, + tag="URL_SEED", + ) + except Exception as e: + self._log("error", "ElementTree parsing error for sitemap {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return + + # Process based on type + if is_sitemap_index and sub_sitemaps: + self._log("info", "Processing sitemap index with {count} sub-sitemaps in parallel", + params={"count": len(sub_sitemaps)}, tag="URL_SEED") + + # Create a bounded queue for results to prevent RAM issues + # For sitemap indexes, use a larger queue as we expect many URLs + queue_size = min(50000, len(sub_sitemaps) * 1000) # Estimate 1000 URLs per sitemap + result_queue = asyncio.Queue(maxsize=queue_size) + completed_count = 0 + total_sitemaps = len(sub_sitemaps) + + async def process_subsitemap(sitemap_url: str): + try: + self._log( + "debug", "Processing sub-sitemap: {url}", params={"url": sitemap_url}, tag="URL_SEED") + # Recursively process sub-sitemap + async for u in self._iter_sitemap(sitemap_url): + await result_queue.put(u) # Will block if queue is full + except Exception as e: + self._log("error", "Error processing sub-sitemap {url}: {error}", + params={"url": sitemap_url, "error": str(e)}, tag="URL_SEED") + finally: + # Put sentinel to signal completion + await result_queue.put(None) + + # Start all tasks + tasks = [asyncio.create_task(process_subsitemap(sm)) + for sm in sub_sitemaps] + + # Yield results as they come in + while completed_count < total_sitemaps: + item = await result_queue.get() + if item is None: + completed_count += 1 + else: + yield item + + # Ensure all tasks are done + await asyncio.gather(*tasks, return_exceptions=True) + else: + # Regular sitemap - yield URLs directly + for u in regular_urls: + yield u + + # ─────────────────────────────── validate helpers + async def _validate(self, url: str, res_list: List[Dict[str, Any]], live: bool, + extract: bool, timeout: int, verbose: bool, query: Optional[str] = None, + score_threshold: Optional[float] = None, scoring_method: str = "bm25", + filter_nonsense: bool = True): + # Local verbose parameter for this function is used to decide if intermediate logs should be printed + # The main logger's verbose status should be controlled by the caller. + + # First check if this is a nonsense URL (if filtering is enabled) + if filter_nonsense and self._is_nonsense_url(url): + self._log("debug", "Filtered out nonsense URL: {url}", + params={"url": url}, tag="URL_SEED") + return + + cache_kind = "head" if extract else "live" + + # ---------- try cache ---------- + if not (hasattr(self, 'force') and self.force): + cached = await self._cache_get(cache_kind, url) + if cached: + res_list.append(cached) + return + + if extract: + self._log("debug", "Fetching head for {url}", params={ + "url": url}, tag="URL_SEED") + ok, html, final = await self._fetch_head(url, timeout) + status = "valid" if ok else "not_valid" + self._log("info" if ok else "warning", "HEAD {status} for {final_url}", + params={"status": status.upper(), "final_url": final or url}, tag="URL_SEED") + # head_data = _parse_head(html) if ok else {} + head_data = await asyncio.to_thread(_parse_head, html) if ok else {} + entry = { + "url": final or url, + "original_url": url, + "status": status, + "head_data": head_data, + } + + elif live: + self._log("debug", "Performing live check for {url}", params={ + "url": url}, tag="URL_SEED") + ok = await self._resolve_head(url) + status = "valid" if ok else "not_valid" + self._log("info" if ok else "warning", "LIVE CHECK {status} for {url}", + params={"status": status.upper(), "url": url}, tag="URL_SEED") + entry = {"url": url, "status": status, "head_data": {}} + + else: + entry = {"url": url, "status": "unknown", "head_data": {}} + + # Add entry to results (scoring will be done later) + if live or extract: + await self._cache_set(cache_kind, url, entry) + res_list.append(entry) + + async def _head_ok(self, url: str, timeout: int) -> bool: + try: + r = await self.client.head(url, timeout=timeout, + headers={"Range": "bytes=0-0", "Accept-Encoding": "identity"}) + r.raise_for_status() # Raise for bad status codes (4xx, 5xx) + return True + except httpx.RequestError as e: + self._log("debug", "HEAD check network error for {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return False + except httpx.HTTPStatusError as e: + self._log("debug", "HEAD check HTTP status error for {url}: {status_code}", + params={"url": url, "status_code": e.response.status_code}, tag="URL_SEED") + return False + except Exception as e: + self._log("error", "Unexpected error during HEAD check for {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return False + + async def _fetch_head( + self, + url: str, + timeout: int, + max_redirects: int = 5, + max_bytes: int = 65_536, # stop after 64 kB even if never comes + chunk_size: int = 4096, # how much we read per await + ): + for _ in range(max_redirects+1): + try: + # ask the first `max_bytes` and force plain text to avoid + # partial-gzip decode headaches + async with self.client.stream( + "GET", + url, + timeout=timeout, + headers={ + # "Range": f"bytes=0-{max_bytes-1}", # Dropped the Range header – no need now, and some servers ignore it. We still keep an upper‐bound max_bytes as a fail-safe. + "Accept-Encoding": "identity", + }, + follow_redirects=False, + ) as r: + + if r.status_code in (301, 302, 303, 307, 308): + location = r.headers.get("Location") + if location: + url = urljoin(url, location) + self._log("debug", "Redirecting from {original_url} to {new_url}", + params={"original_url": r.url, "new_url": url}, tag="URL_SEED") + continue + else: + self._log("warning", "Redirect status {status_code} but no Location header for {url}", + params={"status_code": r.status_code, "url": r.url}, tag="URL_SEED") + # Return original URL if no new location + return False, "", str(r.url) + + # For 2xx or other non-redirect codes, proceed to read content + # Only allow successful codes, or continue + if not (200 <= r.status_code < 400): + self._log("warning", "Non-success status {status_code} when fetching head for {url}", + params={"status_code": r.status_code, "url": r.url}, tag="URL_SEED") + return False, "", str(r.url) + + buf = bytearray() + async for chunk in r.aiter_bytes(chunk_size): + buf.extend(chunk) + low = buf.lower() + if b"" in low or len(buf) >= max_bytes: + await r.aclose() + break + + enc = r.headers.get("Content-Encoding", "").lower() + try: + if enc == "gzip" and buf[:2] == b"\x1f\x8b": + buf = gzip.decompress(buf) + elif enc == "br" and HAS_BROTLI and buf[:4] == b"\x8b\x6c\x0a\x1a": + buf = brotli.decompress(buf) + elif enc in {"gzip", "br"}: + # Header says “gzip” or “br” but payload is plain – ignore + self._log( + "debug", + "Skipping bogus {encoding} for {url}", + params={"encoding": enc, "url": r.url}, + tag="URL_SEED", + ) + except Exception as e: + self._log( + "warning", + "Decompression error for {url} ({encoding}): {error}", + params={"url": r.url, + "encoding": enc, "error": str(e)}, + tag="URL_SEED", + ) + # fall through with raw buf + + # Find the tag case-insensitively and decode + idx = buf.lower().find(b"") + if idx == -1: + self._log("debug", "No tag found in initial bytes of {url}", + params={"url": r.url}, tag="URL_SEED") + # If no is found, take a reasonable chunk or all if small + # Take max 10KB if no head tag + html_bytes = buf if len(buf) < 10240 else buf[:10240] + else: + html_bytes = buf[:idx+7] # Include tag + + try: + html = html_bytes.decode("utf-8", "replace") + except Exception as e: + self._log( + "warning", + "Failed to decode head content for {url}: {error}", + params={"url": r.url, "error": str(e)}, + tag="URL_SEED", + ) + html = html_bytes.decode("latin-1", "replace") + + # Return the actual URL after redirects + return True, html, str(r.url) + + except httpx.RequestError as e: + self._log("debug", "Fetch head network error for {url}: {error}", + params={"url": url, "error": str(e)}, tag="URL_SEED") + return False, "", url + + # If loop finishes without returning (e.g. too many redirects) + self._log("warning", "Exceeded max redirects ({max_redirects}) for {url}", + params={"max_redirects": max_redirects, "url": url}, tag="URL_SEED") + return False, "", url + + # ─────────────────────────────── BM25 scoring helpers + def _extract_text_context(self, head_data: Dict[str, Any]) -> str: + """Extract all relevant text from head metadata for scoring.""" + # Priority fields with their weights (for future enhancement) + text_parts = [] + + # Title + if head_data.get("title"): + text_parts.append(head_data["title"]) + + # Standard meta tags + meta = head_data.get("meta", {}) + for key in ["description", "keywords", "author", "subject", "summary", "abstract"]: + if meta.get(key): + text_parts.append(meta[key]) + + # Open Graph tags + for key in ["og:title", "og:description", "og:site_name", "article:tag"]: + if meta.get(key): + text_parts.append(meta[key]) + + # Twitter Card tags + for key in ["twitter:title", "twitter:description", "twitter:image:alt"]: + if meta.get(key): + text_parts.append(meta[key]) + + # Dublin Core tags + for key in ["dc.title", "dc.description", "dc.subject", "dc.creator"]: + if meta.get(key): + text_parts.append(meta[key]) + + # JSON-LD structured data + for jsonld in head_data.get("jsonld", []): + if isinstance(jsonld, dict): + # Extract common fields from JSON-LD + for field in ["name", "headline", "description", "abstract", "keywords"]: + if field in jsonld: + if isinstance(jsonld[field], str): + text_parts.append(jsonld[field]) + elif isinstance(jsonld[field], list): + text_parts.extend(str(item) + for item in jsonld[field] if item) + + # Handle @graph structures + if "@graph" in jsonld and isinstance(jsonld["@graph"], list): + for item in jsonld["@graph"]: + if isinstance(item, dict): + for field in ["name", "headline", "description"]: + if field in item and isinstance(item[field], str): + text_parts.append(item[field]) + + # Combine all text parts + return " ".join(filter(None, text_parts)) + + def _calculate_url_relevance_score(self, query: str, url: str) -> float: + """Calculate relevance score between query and URL using string matching.""" + # Normalize inputs + query_lower = query.lower() + url_lower = url.lower() + + # Extract URL components + from urllib.parse import urlparse + parsed = urlparse(url) + domain = parsed.netloc.replace('www.', '') + path = parsed.path.strip('/') + + # Create searchable text from URL + # Split domain by dots and path by slashes + domain_parts = domain.split('.') + path_parts = [p for p in path.split('/') if p] + + # Include query parameters if any + query_params = parsed.query + param_parts = [] + if query_params: + for param in query_params.split('&'): + if '=' in param: + key, value = param.split('=', 1) + param_parts.extend([key, value]) + + # Combine all parts + all_parts = domain_parts + path_parts + param_parts + + # Calculate scores + scores = [] + query_tokens = query_lower.split() + + # 1. Exact match in any part (highest score) + for part in all_parts: + part_lower = part.lower() + if query_lower in part_lower: + scores.append(1.0) + elif part_lower in query_lower: + scores.append(0.9) + + # 2. Token matching + for token in query_tokens: + token_scores = [] + for part in all_parts: + part_lower = part.lower() + if token in part_lower: + # Score based on how much of the part the token covers + coverage = len(token) / len(part_lower) + token_scores.append(0.7 * coverage) + elif part_lower in token: + coverage = len(part_lower) / len(token) + token_scores.append(0.6 * coverage) + + if token_scores: + scores.append(max(token_scores)) + + # 3. Character n-gram similarity (for fuzzy matching) + def get_ngrams(text, n=3): + return set(text[i:i+n] for i in range(len(text)-n+1)) + + # Combine all URL parts into one string for n-gram comparison + url_text = ' '.join(all_parts).lower() + if len(query_lower) >= 3 and len(url_text) >= 3: + query_ngrams = get_ngrams(query_lower) + url_ngrams = get_ngrams(url_text) + if query_ngrams and url_ngrams: + intersection = len(query_ngrams & url_ngrams) + union = len(query_ngrams | url_ngrams) + jaccard = intersection / union if union > 0 else 0 + scores.append(0.5 * jaccard) + + # Calculate final score + if not scores: + return 0.0 + + # Weighted average with bias towards higher scores + scores.sort(reverse=True) + weighted_score = 0 + total_weight = 0 + for i, score in enumerate(scores): + weight = 1 / (i + 1) # Higher weight for better matches + weighted_score += score * weight + total_weight += weight + + final_score = weighted_score / total_weight if total_weight > 0 else 0 + return min(final_score, 1.0) # Cap at 1.0 + + def _is_nonsense_url(self, url: str) -> bool: + """ + Check if URL is a utility/nonsense URL that shouldn't be crawled. + Returns True if the URL should be filtered out. + """ + url_lower = url.lower() + + # Extract path and filename + from urllib.parse import urlparse + parsed = urlparse(url) + path = parsed.path.lower() + + # 1. Robot and sitemap files + if path.endswith(('/robots.txt', '/sitemap.xml', '/sitemap_index.xml')): + return True + + # 2. Sitemap variations + if '/sitemap' in path and path.endswith(('.xml', '.xml.gz', '.txt')): + return True + + # 3. Common utility files + utility_files = [ + 'ads.txt', 'humans.txt', 'security.txt', '.well-known/security.txt', + 'crossdomain.xml', 'browserconfig.xml', 'manifest.json', + 'apple-app-site-association', '.well-known/apple-app-site-association', + 'favicon.ico', 'apple-touch-icon.png', 'android-chrome-192x192.png' + ] + if any(path.endswith(f'/{file}') for file in utility_files): + return True + + # # 4. Feed files + # if path.endswith(('.rss', '.atom', '/feed', '/rss', '/atom', '/feed.xml', '/rss.xml')): + # return True + + # # 5. API endpoints and data files + # api_patterns = ['/api/', '/v1/', '/v2/', '/v3/', '/graphql', '/.json', '/.xml'] + # if any(pattern in path for pattern in api_patterns): + # return True + + # # 6. Archive and download files + # download_extensions = [ + # '.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', + # '.exe', '.dmg', '.pkg', '.deb', '.rpm', + # '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', + # '.csv', '.tsv', '.sql', '.db', '.sqlite' + # ] + # if any(path.endswith(ext) for ext in download_extensions): + # return True + + # # 7. Media files (often not useful for text content) + # media_extensions = [ + # '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp', '.ico', + # '.mp4', '.avi', '.mov', '.wmv', '.flv', '.webm', + # '.mp3', '.wav', '.ogg', '.m4a', '.flac', + # '.woff', '.woff2', '.ttf', '.eot', '.otf' + # ] + # if any(path.endswith(ext) for ext in media_extensions): + # return True + + # # 8. Source code and config files + # code_extensions = [ + # '.js', '.css', '.scss', '.sass', '.less', + # '.map', '.min.js', '.min.css', + # '.py', '.rb', '.php', '.java', '.cpp', '.h', + # '.yaml', '.yml', '.toml', '.ini', '.conf', '.config' + # ] + # if any(path.endswith(ext) for ext in code_extensions): + # return True + + # 9. Hidden files and directories + path_parts = path.split('/') + if any(part.startswith('.') for part in path_parts if part): + return True + + # 10. Common non-content paths + non_content_paths = [ + '/wp-admin', '/wp-includes', '/wp-content/uploads', + '/admin', '/login', '/signin', '/signup', '/register', + '/checkout', '/cart', '/account', '/profile', + '/search', '/404', '/error', + '/.git', '/.svn', '/.hg', + '/cgi-bin', '/scripts', '/includes' + ] + if any(ncp in path for ncp in non_content_paths): + return True + + # 11. URL patterns that indicate non-content + if any(pattern in url_lower for pattern in ['?print=', '&print=', '/print/', '_print.']): + return True + + # 12. Very short paths (likely homepage redirects or errors) + if len(path.strip('/')) < 3 and path not in ['/', '/en', '/de', '/fr', '/es', '/it']: + return True + + return False + + def _calculate_bm25_score(self, query: str, documents: List[str]) -> List[float]: + """Calculate BM25 scores for documents against a query.""" + if not HAS_BM25: + self._log( + "warning", "rank_bm25 not installed. Returning zero scores.", tag="URL_SEED") + return [0.0] * len(documents) + + if not query or not documents: + return [0.0] * len(documents) + + # Tokenize query and documents (simple whitespace tokenization) + # For production, consider using a proper tokenizer + query_tokens = query.lower().split() + tokenized_docs = [doc.lower().split() for doc in documents] + + # Handle edge case where all documents are empty + if all(len(doc) == 0 for doc in tokenized_docs): + return [0.0] * len(documents) + + # Create BM25 instance and calculate scores + try: + from rank_bm25 import BM25Okapi + bm25 = BM25Okapi(tokenized_docs) + scores = bm25.get_scores(query_tokens) + + # Normalize scores to 0-1 range + # BM25 can return negative scores, so we need to handle the full range + if len(scores) == 0: + return [] + + min_score = min(scores) + max_score = max(scores) + + # If all scores are the same, return 0.5 for all + if max_score == min_score: + return [0.5] * len(scores) + + # Normalize to 0-1 range using min-max normalization + normalized_scores = [(score - min_score) / (max_score - min_score) for score in scores] + + return normalized_scores + except Exception as e: + self._log("error", "Error calculating BM25 scores: {error}", + params={"error": str(e)}, tag="URL_SEED") + return [0.0] * len(documents) + + # ─────────────────────────────── cleanup methods + async def close(self): + """Close the HTTP client if we own it.""" + if self._owns_client and self.client: + await self.client.aclose() + self._log("debug", "Closed HTTP client", tag="URL_SEED") + + async def __aenter__(self): + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() + return False + + # ─────────────────────────────── index helper + async def _latest_index(self) -> str: + if self.index_cache_path.exists() and (time.time()-self.index_cache_path.stat().st_mtime) < self.ttl.total_seconds(): + self._log("info", "Loading latest CC index from cache: {path}", + params={"path": self.index_cache_path}, tag="URL_SEED") + return self.index_cache_path.read_text().strip() + + self._log("info", "Fetching latest Common Crawl index from {url}", + params={"url": COLLINFO_URL}, tag="URL_SEED") + try: + async with httpx.AsyncClient() as c: + j = await c.get(COLLINFO_URL, timeout=10) + j.raise_for_status() # Raise an exception for bad status codes + idx = j.json()[0]["id"] + self.index_cache_path.write_text(idx) + self._log("success", "Successfully fetched and cached CC index: {index_id}", + params={"index_id": idx}, tag="URL_SEED") + return idx + except httpx.RequestError as e: + self._log("error", "Network error fetching CC index info: {error}", + params={"error": str(e)}, tag="URL_SEED") + raise + except httpx.HTTPStatusError as e: + self._log("error", "HTTP error fetching CC index info: {status_code}", + params={"status_code": e.response.status_code}, tag="URL_SEED") + raise + except Exception as e: + self._log("error", "Unexpected error fetching CC index info: {error}", + params={"error": str(e)}, tag="URL_SEED") + raise diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py new file mode 100644 index 0000000..8216d19 --- /dev/null +++ b/crawl4ai/async_webcrawler.py @@ -0,0 +1,1249 @@ +from .__version__ import __version__ as crawl4ai_version +import os +import re +import sys +import time +from pathlib import Path +from typing import Optional, List +import json +import asyncio + +# from contextlib import nullcontext, asynccontextmanager +from contextlib import asynccontextmanager +from .models import ( + CrawlResult, + MarkdownGenerationResult, + DispatchResult, + ScrapingResult, + CrawlResultContainer, + RunManyReturn +) +from .async_database import async_db_manager +from .chunking_strategy import * # noqa: F403 +from .chunking_strategy import IdentityChunking +from .content_filter_strategy import * # noqa: F403 +from .extraction_strategy import * # noqa: F403 +from .extraction_strategy import NoExtractionStrategy +from .async_crawler_strategy import ( + AsyncCrawlerStrategy, + AsyncPlaywrightCrawlerStrategy, + AsyncCrawlResponse, +) +from .cache_context import CacheMode, CacheContext +from .markdown_generation_strategy import ( + DefaultMarkdownGenerator, + MarkdownGenerationStrategy, +) +from .deep_crawling import DeepCrawlDecorator +from .async_logger import AsyncLogger, AsyncLoggerBase +from .async_configs import BrowserConfig, CrawlerRunConfig, ProxyConfig, SeedingConfig, DomainMapperConfig +from .async_dispatcher import * # noqa: F403 +from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter +from .async_url_seeder import AsyncUrlSeeder +from .domain_mapper import DomainMapper + +from .utils import ( + sanitize_input_encode, + InvalidCSSSelectorError, + fast_format_html, + get_error_context, + RobotsParser, + preprocess_html_for_schema, + compute_head_fingerprint, +) +from .cache_validator import CacheValidator, CacheValidationResult +from .antibot_detector import is_blocked + + +class AsyncWebCrawler: + """ + Asynchronous web crawler with flexible caching capabilities. + + There are two ways to use the crawler: + + 1. Using context manager (recommended for simple cases): + ```python + async with AsyncWebCrawler() as crawler: + result = await crawler.arun(url="https://example.com") + ``` + + 2. Using explicit lifecycle management (recommended for long-running applications): + ```python + crawler = AsyncWebCrawler() + await crawler.start() + + # Use the crawler multiple times + result1 = await crawler.arun(url="https://example.com") + result2 = await crawler.arun(url="https://another.com") + + await crawler.close() + ``` + + Attributes: + browser_config (BrowserConfig): Configuration object for browser settings. + crawler_strategy (AsyncCrawlerStrategy): Strategy for crawling web pages. + logger (AsyncLogger): Logger instance for recording events and errors. + crawl4ai_folder (str): Directory for storing cache. + base_directory (str): Base directory for storing cache. + ready (bool): Whether the crawler is ready for use. + + Methods: + start(): Start the crawler explicitly without using context manager. + close(): Close the crawler explicitly without using context manager. + arun(): Run the crawler for a single source: URL (web, local file, or raw HTML). + awarmup(): Perform warmup sequence. + arun_many(): Run the crawler for multiple sources. + aprocess_html(): Process HTML content. + + Typical Usage: + async with AsyncWebCrawler() as crawler: + result = await crawler.arun(url="https://example.com") + print(result.markdown) + + Using configuration: + browser_config = BrowserConfig(browser_type="chromium", headless=True) + async with AsyncWebCrawler(config=browser_config) as crawler: + crawler_config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS + ) + result = await crawler.arun(url="https://example.com", config=crawler_config) + print(result.markdown) + """ + + _domain_last_hit = {} + + def __init__( + self, + crawler_strategy: AsyncCrawlerStrategy = None, + config: BrowserConfig = None, + base_directory: str = str( + os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())), + thread_safe: bool = False, + logger: AsyncLoggerBase = None, + **kwargs, + ): + """ + Initialize the AsyncWebCrawler. + + Args: + crawler_strategy: Strategy for crawling web pages. Default AsyncPlaywrightCrawlerStrategy + config: Configuration object for browser settings. Default BrowserConfig() + base_directory: Base directory for storing cache + thread_safe: Whether to use thread-safe operations + **kwargs: Additional arguments for backwards compatibility + """ + # Handle browser configuration + browser_config = config or BrowserConfig() + + self.browser_config = browser_config + + # Initialize logger first since other components may need it + self.logger = logger or AsyncLogger( + log_file=os.path.join(base_directory, ".crawl4ai", "crawler.log"), + verbose=self.browser_config.verbose, + tag_width=10, + ) + + # Initialize crawler strategy + params = {k: v for k, v in kwargs.items() if k in [ + "browser_config", "logger"]} + self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy( + browser_config=browser_config, + logger=self.logger, + **params, # Pass remaining kwargs for backwards compatibility + ) + + # Thread safety setup + self._lock = asyncio.Lock() if thread_safe else None + + # Initialize directories + self.crawl4ai_folder = os.path.join(base_directory, ".crawl4ai") + os.makedirs(self.crawl4ai_folder, exist_ok=True) + os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True) + + # Initialize robots parser + self.robots_parser = RobotsParser() + + self.ready = False + + # Decorate arun method with deep crawling capabilities + self._deep_handler = DeepCrawlDecorator(self) + self.arun = self._deep_handler(self.arun) + + self.url_seeder: Optional[AsyncUrlSeeder] = None + self._domain_mapper: Optional[DomainMapper] = None + + async def start(self): + """ + Start the crawler explicitly without using context manager. + This is equivalent to using 'async with' but gives more control over the lifecycle. + Returns: + AsyncWebCrawler: The initialized crawler instance + """ + await self.crawler_strategy.__aenter__() + self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT") + self.ready = True + return self + + async def close(self): + """ + Close the crawler explicitly without using context manager. + This should be called when you're done with the crawler if you used start(). + + This method will: + 1. Clean up browser resources + 2. Close any open pages and contexts + """ + await self.crawler_strategy.__aexit__(None, None, None) + + async def __aenter__(self): + return await self.start() + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + @asynccontextmanager + async def nullcontext(self): + """异步空上下文管理器""" + yield + + async def arun( + self, + url: str, + config: CrawlerRunConfig = None, + **kwargs, + ) -> CrawlResultContainer: + """ + Runs the crawler for a single source: URL (web, local file, or raw HTML). + + Migration Guide: + Old way (deprecated): + result = await crawler.arun( + url="https://example.com", + word_count_threshold=200, + screenshot=True, + ... + ) + + New way (recommended): + config = CrawlerRunConfig( + word_count_threshold=200, + screenshot=True, + ... + ) + result = await crawler.arun(url="https://example.com", config=config) + + Args: + url: The URL to crawl (http://, https://, file://, or raw:) + config: Configuration object controlling crawl behavior + [other parameters maintained for backwards compatibility] + + Returns: + CrawlResultContainer: A single-result container that proxies + attribute access to the underlying CrawlResult for backwards + compatibility (e.g. result.markdown, result.html). + """ + # Auto-start if not ready + if not self.ready: + await self.start() + + config = config or CrawlerRunConfig() + if not isinstance(url, str) or not url: + raise ValueError( + "Invalid URL, make sure the URL is a non-empty string") + + async with self._lock or self.nullcontext(): + try: + self.logger.verbose = config.verbose + + # Default to ENABLED if no cache mode specified + if config.cache_mode is None: + config.cache_mode = CacheMode.ENABLED + + # Create cache context + cache_context = CacheContext(url, config.cache_mode, False) + + # Initialize processing variables + async_response: AsyncCrawlResponse = None + cached_result: CrawlResult = None + screenshot_data = None + pdf_data = None + extracted_content = None + start_time = time.perf_counter() + + # Try to get cached result if appropriate + if cache_context.should_read(): + cached_result = await async_db_manager.aget_cached_url(url) + + # Smart Cache: Validate cache freshness if enabled + if cached_result and config.check_cache_freshness: + cache_metadata = await async_db_manager.aget_cache_metadata(url) + if cache_metadata: + async with CacheValidator(timeout=config.cache_validation_timeout) as validator: + validation = await validator.validate( + url=url, + stored_etag=cache_metadata.get("etag"), + stored_last_modified=cache_metadata.get("last_modified"), + stored_head_fingerprint=cache_metadata.get("head_fingerprint"), + ) + + if validation.status == CacheValidationResult.FRESH: + cached_result.cache_status = "hit_validated" + self.logger.info( + message="Cache validated: {reason}", + tag="CACHE", + params={"reason": validation.reason} + ) + # Update metadata if we got new values + if validation.new_etag or validation.new_last_modified: + await async_db_manager.aupdate_cache_metadata( + url=url, + etag=validation.new_etag, + last_modified=validation.new_last_modified, + head_fingerprint=validation.new_head_fingerprint, + ) + elif validation.status == CacheValidationResult.ERROR: + cached_result.cache_status = "hit_fallback" + self.logger.warning( + message="Cache validation failed, using cached: {reason}", + tag="CACHE", + params={"reason": validation.reason} + ) + else: + # STALE or UNKNOWN - force recrawl + self.logger.info( + message="Cache stale: {reason}", + tag="CACHE", + params={"reason": validation.reason} + ) + cached_result = None + elif cached_result: + cached_result.cache_status = "hit" + + if cached_result: + html = sanitize_input_encode(cached_result.html) + extracted_content = sanitize_input_encode( + cached_result.extracted_content or "" + ) + extracted_content = ( + None + if not extracted_content or extracted_content == "[]" + else extracted_content + ) + # If screenshot is requested but its not in cache, then set cache_result to None + screenshot_data = cached_result.screenshot + pdf_data = cached_result.pdf + # if config.screenshot and not screenshot or config.pdf and not pdf: + if config.screenshot and not screenshot_data: + cached_result = None + + if config.pdf and not pdf_data: + cached_result = None + + self.logger.url_status( + url=cache_context.display_url, + success=bool(html), + timing=time.perf_counter() - start_time, + tag="FETCH", + ) + + # Update proxy configuration from rotation strategy if available + if config and config.proxy_rotation_strategy: + # Handle sticky sessions - use same proxy for all requests with same session_id + if config.proxy_session_id: + next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_proxy_for_session( + config.proxy_session_id, + ttl=config.proxy_session_ttl + ) + if next_proxy: + self.logger.info( + message="Using sticky proxy session: {session_id} -> {proxy}", + tag="PROXY", + params={ + "session_id": config.proxy_session_id, + "proxy": next_proxy.server + } + ) + config.proxy_config = next_proxy + else: + # Existing behavior: rotate on each request + next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_next_proxy() + if next_proxy: + self.logger.info( + message="Switch proxy: {proxy}", + tag="PROXY", + params={"proxy": next_proxy.server} + ) + config.proxy_config = next_proxy + + # Fetch fresh content if needed + if not cached_result or not html: + from urllib.parse import urlparse + + # Check robots.txt if enabled (once, before any attempts) + if config and config.check_robots_txt: + if not await self.robots_parser.can_fetch( + url, self.browser_config.user_agent + ): + return CrawlResult( + url=url, + html="", + success=False, + status_code=403, + error_message="Access denied by robots.txt", + response_headers={ + "X-Robots-Status": "Blocked by robots.txt" + }, + ) + + # --- Anti-bot retry setup --- + # raw: URLs contain caller-provided HTML (e.g. from cache), + # not content fetched from a web server. Anti-bot detection, + # proxy retries, and fallback fetching are meaningless here. + _is_raw_url = url.startswith("raw:") or url.startswith("raw://") + + _max_attempts = 1 + getattr(config, "max_retries", 0) + _proxy_list = config._get_proxy_list() + _original_proxy_config = config.proxy_config + _block_reason = "" + _done = False + crawl_result = None + _crawl_stats = { + "attempts": 0, + "retries": 0, + "proxies_used": [], + "fallback_fetch_used": False, + "resolved_by": None, + } + + for _attempt in range(_max_attempts): + if _done: + break + + if _attempt > 0: + _crawl_stats["retries"] = _attempt + self.logger.warning( + message="Anti-bot retry {attempt}/{max_retries} for {url} — {reason}", + tag="ANTIBOT", + params={ + "attempt": _attempt, + "max_retries": config.max_retries, + "url": url[:80], + "reason": _block_reason, + }, + ) + + for _p_idx, _proxy in enumerate(_proxy_list): + if _p_idx > 0 or _attempt > 0: + self.logger.info( + message="Trying proxy {idx}/{total}: {proxy}", + tag="ANTIBOT", + params={ + "idx": _p_idx + 1, + "total": len(_proxy_list), + "proxy": _proxy.server if _proxy else "direct", + }, + ) + + # Set the active proxy for this attempt + config.proxy_config = _proxy + _crawl_stats["attempts"] += 1 + + try: + t1 = time.perf_counter() + + if config.user_agent: + self.crawler_strategy.update_user_agent( + config.user_agent) + + async_response = await self.crawler_strategy.crawl( + url, config=config) + + html = sanitize_input_encode(async_response.html) + screenshot_data = async_response.screenshot + pdf_data = async_response.pdf_data + js_execution_result = async_response.js_execution_result + + self.logger.url_status( + url=cache_context.display_url, + success=bool(html), + timing=time.perf_counter() - t1, + tag="FETCH", + ) + + crawl_result = await self.aprocess_html( + url=url, html=html, + extracted_content=extracted_content, + config=config, + screenshot_data=screenshot_data, + pdf_data=pdf_data, + verbose=config.verbose, + is_raw_html=True if url.startswith("raw:") else False, + redirected_url=async_response.redirected_url, + original_scheme=urlparse(url).scheme, + **kwargs, + ) + + crawl_result.status_code = async_response.status_code + is_raw_url = url.startswith("raw:") or url.startswith("raw://") + crawl_result.redirected_url = async_response.redirected_url or (None if is_raw_url else url) + crawl_result.redirected_status_code = async_response.redirected_status_code + crawl_result.response_headers = async_response.response_headers + crawl_result.downloaded_files = async_response.downloaded_files + crawl_result.js_execution_result = js_execution_result + crawl_result.mhtml = async_response.mhtml_data + crawl_result.ssl_certificate = async_response.ssl_certificate + crawl_result.network_requests = async_response.network_requests + crawl_result.console_messages = async_response.console_messages + # Success when html is non-empty OR a binary + # download was retrieved (PDFs, archives etc. + # have empty html by design — file content is + # in downloaded_files). + crawl_result.success = bool(html) or bool(async_response.downloaded_files) + crawl_result.session_id = getattr(config, "session_id", None) + crawl_result.cache_status = "miss" + + # Check if blocked (skip for raw: URLs — + # caller-provided content, anti-bot N/A) + if _is_raw_url: + _blocked = False + _block_reason = "" + else: + _blocked, _block_reason = is_blocked( + async_response.status_code, html) + + _crawl_stats["proxies_used"].append({ + "proxy": _proxy.server if _proxy else None, + "status_code": async_response.status_code, + "blocked": _blocked, + "reason": _block_reason if _blocked else "", + }) + + if not _blocked: + _crawl_stats["resolved_by"] = "proxy" if _proxy else "direct" + _done = True + break # Success — exit proxy loop + + except Exception as _crawl_err: + _crawl_stats["proxies_used"].append({ + "proxy": _proxy.server if _proxy else None, + "status_code": None, + "blocked": True, + "reason": str(_crawl_err), + }) + self.logger.error_status( + url=url, + error=f"Proxy {_proxy.server if _proxy else 'direct'} failed: {_crawl_err}", + tag="ANTIBOT", + ) + _block_reason = str(_crawl_err) + # If this is the only proxy and only attempt, re-raise + # so the caller gets the real error (not a silent swallow). + # But if there are more proxies or retries to try, continue. + if len(_proxy_list) <= 1 and _max_attempts <= 1: + raise + + # Restore original proxy_config + config.proxy_config = _original_proxy_config + + # --- Fallback fetch function (last resort after all retries+proxies exhausted) --- + # Invoke fallback when: (a) crawl_result exists but is blocked, OR + # (b) crawl_result is None because all proxies threw exceptions (browser crash, timeout). + # Skip for raw: URLs — fallback expects a real URL, not raw HTML content. + _fallback_fn = getattr(config, "fallback_fetch_function", None) + if _fallback_fn and not _done and not _is_raw_url: + _needs_fallback = ( + crawl_result is None # All proxies threw exceptions + or is_blocked(crawl_result.status_code, crawl_result.html or "")[0] + ) + if _needs_fallback: + self.logger.warning( + message="All retries exhausted, invoking fallback_fetch_function for {url}", + tag="ANTIBOT", + params={"url": url[:80]}, + ) + _crawl_stats["fallback_fetch_used"] = True + try: + _fallback_html = await _fallback_fn(url) + if _fallback_html: + _sanitized_html = sanitize_input_encode(_fallback_html) + try: + crawl_result = await self.aprocess_html( + url=url, + html=_sanitized_html, + extracted_content=extracted_content, + config=config, + screenshot_data=None, + pdf_data=None, + verbose=config.verbose, + is_raw_html=True, + redirected_url=url, + original_scheme=urlparse(url).scheme, + **kwargs, + ) + except Exception as _proc_err: + # aprocess_html may fail if browser is dead (e.g., + # consent popup removal needs Page.evaluate). + # Fall back to a minimal result with raw HTML. + self.logger.warning( + message="Fallback HTML processing failed ({err}), using raw HTML", + tag="ANTIBOT", + params={"err": str(_proc_err)[:100]}, + ) + crawl_result = CrawlResult( + url=url, + html=_sanitized_html, + success=True, + status_code=200, + ) + crawl_result.success = True + crawl_result.status_code = 200 + crawl_result.session_id = getattr(config, "session_id", None) + crawl_result.cache_status = "miss" + _crawl_stats["resolved_by"] = "fallback_fetch" + except Exception as _fallback_err: + self.logger.error_status( + url=url, + error=f"Fallback fetch failed: {_fallback_err}", + tag="ANTIBOT", + ) + + # --- Mark blocked results as failed --- + # Skip re-check ONLY when fallback SUCCEEDED — the fallback result + # is authoritative and real pages may contain anti-bot script markers + # (e.g. PerimeterX JS on Walmart) that trigger false positives. + # When fallback was attempted but FAILED, we must still re-check + # because the result is from a blocked proxy attempt. + # Also skip for raw: URLs — caller-provided content, anti-bot N/A. + # Also skip for binary downloads (PDFs, archives, etc.) — content + # was delivered via downloaded_files, html is empty by design, + # and is_blocked() would misread "0 bytes html" as a block. + if crawl_result: + _fallback_succeeded = _crawl_stats.get("resolved_by") == "fallback_fetch" + # Skip the block check for binary downloads (PDFs, archives, + # etc.) — content was delivered via downloaded_files, html is + # empty by design, and is_blocked() would misread "0 bytes + # html" as a block. + _has_download = bool(getattr(crawl_result, "downloaded_files", None)) + if not _fallback_succeeded and not _is_raw_url and not _has_download: + _blocked, _block_reason = is_blocked( + crawl_result.status_code, crawl_result.html or "") + if _blocked: + crawl_result.success = False + crawl_result.error_message = f"Blocked by anti-bot protection: {_block_reason}" + crawl_result.crawl_stats = _crawl_stats + else: + # All proxies threw exceptions and fallback either wasn't + # configured or also failed. Build a minimal result so the + # caller gets crawl_stats instead of None. + crawl_result = CrawlResult( + url=url, + html="", + success=False, + status_code=None, + error_message=f"All proxies failed: {_block_reason}" if _block_reason else "All proxies failed", + ) + crawl_result.crawl_stats = _crawl_stats + + # Compute head fingerprint for cache validation + if crawl_result and crawl_result.html: + head_end = crawl_result.html.lower().find('') + if head_end != -1: + head_html = crawl_result.html[:head_end + 7] + crawl_result.head_fingerprint = compute_head_fingerprint(head_html) + + # Log failure reason before COMPLETE so users can see why it failed. + if crawl_result and not crawl_result.success and crawl_result.error_message: + self.logger.error_status( + url=cache_context.display_url, + error=crawl_result.error_message, + tag="ERROR", + ) + + self.logger.url_status( + url=cache_context.display_url, + success=crawl_result.success if crawl_result else False, + timing=time.perf_counter() - start_time, + tag="COMPLETE", + ) + + # Update cache if appropriate + if cache_context.should_write() and not bool(cached_result): + await async_db_manager.acache_url(crawl_result) + + return CrawlResultContainer(crawl_result) + + else: + self.logger.url_status( + url=cache_context.display_url, + success=True, + timing=time.perf_counter() - start_time, + tag="COMPLETE" + ) + # Same binary-download awareness as the live-fetch path + # — a cached PDF/archive should replay as success. + cached_result.success = bool(html) or bool(getattr(cached_result, "downloaded_files", None)) + cached_result.session_id = getattr( + config, "session_id", None) + # For raw: URLs, don't fall back to the raw HTML string as redirected_url + is_raw_url = url.startswith("raw:") or url.startswith("raw://") + cached_result.redirected_url = cached_result.redirected_url or (None if is_raw_url else url) + return CrawlResultContainer(cached_result) + + except Exception as e: + error_context = get_error_context(sys.exc_info()) + + error_message = ( + f"Unexpected error in _crawl_web at line {error_context['line_no']} " + f"in {error_context['function']} ({error_context['filename']}):\n" + f"Error: {str(e)}\n\n" + f"Code context:\n{error_context['code_context']}" + ) + + self.logger.error_status( + url=url, + error=error_message, + tag="ERROR", + ) + + return CrawlResultContainer( + CrawlResult( + url=url, html="", success=False, error_message=error_message + ) + ) + + async def aprocess_html( + self, + url: str, + html: str, + extracted_content: str, + config: CrawlerRunConfig, + screenshot_data: str, + pdf_data: str, + verbose: bool, + **kwargs, + ) -> CrawlResult: + """ + Process HTML content using the provided configuration. + + Args: + url: The URL being processed + html: Raw HTML content + extracted_content: Previously extracted content (if any) + config: Configuration object controlling processing behavior + screenshot_data: Screenshot data (if any) + pdf_data: PDF data (if any) + verbose: Whether to enable verbose logging + **kwargs: Additional parameters for backwards compatibility + + Returns: + CrawlResult: Processed result containing extracted and formatted content + """ + # === PREFETCH MODE SHORT-CIRCUIT === + if getattr(config, 'prefetch', False): + from .utils import quick_extract_links + + # Use base_url from config (for raw: URLs), redirected_url, or original url + effective_url = getattr(config, 'base_url', None) or kwargs.get('redirected_url') or url + links = quick_extract_links(html, effective_url) + + return CrawlResult( + url=url, + html=html, + success=True, + links=links, + status_code=kwargs.get('status_code'), + response_headers=kwargs.get('response_headers'), + redirected_url=kwargs.get('redirected_url'), + ssl_certificate=kwargs.get('ssl_certificate'), + # All other fields default to None + ) + # === END PREFETCH SHORT-CIRCUIT === + + cleaned_html = "" + try: + _url = url if not kwargs.get("is_raw_html", False) else "Raw HTML" + t1 = time.perf_counter() + + # Get scraping strategy and ensure it has a logger + scraping_strategy = config.scraping_strategy + if not scraping_strategy.logger: + scraping_strategy.logger = self.logger + + # Process HTML content + params = config.__dict__.copy() + params.pop("url", None) + # add keys from kwargs to params that doesn't exist in params + params.update({k: v for k, v in kwargs.items() + if k not in params.keys()}) + + ################################ + # Scraping Strategy Execution # + ################################ + result: ScrapingResult = scraping_strategy.scrap( + url, html, **params) + + if result is None: + raise ValueError( + f"Process HTML, Failed to extract content from the website: {url}" + ) + + except InvalidCSSSelectorError as e: + raise ValueError(str(e)) + except Exception as e: + raise ValueError( + f"Process HTML, Failed to extract content from the website: {url}, error: {str(e)}" + ) + + # Extract results - handle both dict and ScrapingResult + if isinstance(result, dict): + cleaned_html = sanitize_input_encode( + result.get("cleaned_html", "")) + media = result.get("media", {}) + tables = media.pop("tables", []) if isinstance(media, dict) else [] + links = result.get("links", {}) + metadata = result.get("metadata", {}) + else: + cleaned_html = sanitize_input_encode(result.cleaned_html) + # media = result.media.model_dump() + # tables = media.pop("tables", []) + # links = result.links.model_dump() + media = result.media.model_dump() if hasattr(result.media, 'model_dump') else result.media + tables = media.pop("tables", []) if isinstance(media, dict) else [] + links = result.links.model_dump() if hasattr(result.links, 'model_dump') else result.links + metadata = result.metadata + + fit_html = preprocess_html_for_schema(html_content=html, text_threshold= 500, max_size= 300_000) + + ################################ + # Generate Markdown # + ################################ + markdown_generator: Optional[MarkdownGenerationStrategy] = ( + config.markdown_generator or DefaultMarkdownGenerator() + ) + + # --- SELECT HTML SOURCE BASED ON CONTENT_SOURCE --- + # Get the desired source from the generator config, default to 'cleaned_html' + selected_html_source = getattr(markdown_generator, 'content_source', 'cleaned_html') + + # Define the source selection logic using dict dispatch + html_source_selector = { + "raw_html": lambda: html, # The original raw HTML + "cleaned_html": lambda: cleaned_html, # The HTML after scraping strategy + "fit_html": lambda: fit_html, # The HTML after preprocessing for schema + } + + markdown_input_html = cleaned_html # Default to cleaned_html + + try: + # Get the appropriate lambda function, default to returning cleaned_html if key not found + source_lambda = html_source_selector.get(selected_html_source, lambda: cleaned_html) + # Execute the lambda to get the selected HTML + markdown_input_html = source_lambda() + + # Log which source is being used (optional, but helpful for debugging) + # if self.logger and verbose: + # actual_source_used = selected_html_source if selected_html_source in html_source_selector else 'cleaned_html (default)' + # self.logger.debug(f"Using '{actual_source_used}' as source for Markdown generation for {url}", tag="MARKDOWN_SRC") + + except Exception as e: + # Handle potential errors, especially from preprocess_html_for_schema + if self.logger: + self.logger.warning( + f"Error getting/processing '{selected_html_source}' for markdown source: {e}. Falling back to cleaned_html.", + tag="MARKDOWN_SRC" + ) + # Ensure markdown_input_html is still the default cleaned_html in case of error + markdown_input_html = cleaned_html + # --- END: HTML SOURCE SELECTION --- + + # Uncomment if by default we want to use PruningContentFilter + # if not config.content_filter and not markdown_generator.content_filter: + # markdown_generator.content_filter = PruningContentFilter() + + # Extract from raw HTML before it gets stripped by cleaning. + # This ensures relative URLs resolve correctly even with cleaned_html. + base_url = params.get("base_url") or params.get("redirected_url") or url + base_tag_match = re.search(r']*href\s*=\s*["\']([^"\']+)["\']', html, re.IGNORECASE) + if base_tag_match: + base_url = base_tag_match.group(1) + + markdown_result: MarkdownGenerationResult = ( + markdown_generator.generate_markdown( + input_html=markdown_input_html, + base_url=base_url + # html2text_options=kwargs.get('html2text', {}) + ) + ) + + # Log processing completion — reflect actual content outcome + self.logger.url_status( + url=_url, + success=bool(cleaned_html), + timing=int((time.perf_counter() - t1) * 1000) / 1000, + tag="SCRAPE" + ) + # self.logger.info( + # message="{url:.50}... | Time: {timing}s", + # tag="SCRAPE", + # params={"url": _url, "timing": int((time.perf_counter() - t1) * 1000) / 1000}, + # ) + + ################################ + # Structured Content Extraction # + ################################ + if ( + not bool(extracted_content) + and config.extraction_strategy + and not isinstance(config.extraction_strategy, NoExtractionStrategy) + ): + t1 = time.perf_counter() + # Choose content based on input_format + content_format = config.extraction_strategy.input_format + if content_format == "fit_markdown" and not markdown_result.fit_markdown: + + self.logger.url_status( + url=_url, + success=bool(html), + timing=time.perf_counter() - t1, + tag="EXTRACT", + ) + content_format = "markdown" + + content = { + "markdown": markdown_result.raw_markdown, + "html": html, + "fit_html": fit_html, + "cleaned_html": cleaned_html, + "fit_markdown": markdown_result.fit_markdown, + }.get(content_format, markdown_result.raw_markdown) + + # Use IdentityChunking for HTML input, otherwise use provided chunking strategy + chunking = ( + IdentityChunking() + if content_format in ["html", "cleaned_html", "fit_html"] + else config.chunking_strategy + ) + sections = chunking.chunk(content) + # extracted_content = config.extraction_strategy.run(_url, sections) + + # Use async version if available for better parallelism + if hasattr(config.extraction_strategy, 'arun'): + extracted_content = await config.extraction_strategy.arun(_url, sections) + else: + # Fallback to sync version run in thread pool to avoid blocking + extracted_content = await asyncio.to_thread( + config.extraction_strategy.run, url, sections + ) + + extracted_content = json.dumps( + extracted_content, indent=4, default=str, ensure_ascii=False + ) + + # Log extraction completion + self.logger.url_status( + url=_url, + success=bool(html), + timing=time.perf_counter() - t1, + tag="EXTRACT", + ) + + # Apply HTML formatting if requested + if config.prettiify: + cleaned_html = fast_format_html(cleaned_html) + + # Return complete crawl result + return CrawlResult( + url=url, + html=html, + fit_html=fit_html, + cleaned_html=cleaned_html, + markdown=markdown_result, + media=media, + tables=tables, # NEW + links=links, + metadata=metadata, + screenshot=screenshot_data, + pdf=pdf_data, + extracted_content=extracted_content, + success=True, + error_message="", + ) + + async def arun_many( + self, + urls: List[str], + config: Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]] = None, + dispatcher: Optional[BaseDispatcher] = None, + # Legacy parameters maintained for backwards compatibility + # word_count_threshold=MIN_WORD_THRESHOLD, + # extraction_strategy: ExtractionStrategy = None, + # chunking_strategy: ChunkingStrategy = RegexChunking(), + # content_filter: RelevantContentFilter = None, + # cache_mode: Optional[CacheMode] = None, + # bypass_cache: bool = False, + # css_selector: str = None, + # screenshot: bool = False, + # pdf: bool = False, + # user_agent: str = None, + # verbose=True, + **kwargs, + ) -> RunManyReturn: + """ + Runs the crawler for multiple URLs concurrently using a configurable dispatcher strategy. + + Args: + urls: List of URLs to crawl + config: Configuration object(s) controlling crawl behavior. Can be: + - Single CrawlerRunConfig: Used for all URLs + - List[CrawlerRunConfig]: Configs with url_matcher for URL-specific settings + dispatcher: The dispatcher strategy instance to use. Defaults to MemoryAdaptiveDispatcher + [other parameters maintained for backwards compatibility] + + Returns: + Union[List[CrawlResult], AsyncGenerator[CrawlResult, None]]: + Either a list of all results or an async generator yielding results + + Examples: + + # Batch processing (default) + results = await crawler.arun_many( + urls=["https://example1.com", "https://example2.com"], + config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS) + ) + for result in results: + print(f"Processed {result.url}: {len(result.markdown)} chars") + + # Streaming results + async for result in await crawler.arun_many( + urls=["https://example1.com", "https://example2.com"], + config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS, stream=True), + ): + print(f"Processed {result.url}: {len(result.markdown)} chars") + """ + config = config or CrawlerRunConfig() + + # When deep_crawl_strategy is set, bypass the dispatcher and call + # arun() directly for each URL. The DeepCrawlDecorator on arun() + # will invoke the strategy and return List[CrawlResult]. The + # dispatcher cannot handle that return type (it expects a single + # CrawlResult), so we must handle it here. + primary_cfg = config[0] if isinstance(config, list) else config + if getattr(primary_cfg, "deep_crawl_strategy", None): + if primary_cfg.stream: + async def _deep_crawl_stream(): + for url in urls: + result = await self.arun(url, config=primary_cfg) + if isinstance(result, list): + for r in result: + yield r + else: + async for r in result: + yield r + return _deep_crawl_stream() + else: + all_results = [] + for url in urls: + result = await self.arun(url, config=primary_cfg) + if isinstance(result, list): + all_results.extend(result) + else: + all_results.append(result) + return all_results + + if dispatcher is None: + primary_cfg = config[0] if isinstance(config, list) else config + mean_delay = getattr(primary_cfg, "mean_delay", 0.1) + max_range = getattr(primary_cfg, "max_range", 0.3) + max_session_permit = max(1, int(getattr(primary_cfg, "semaphore_count", 10) or 10)) + dispatcher = MemoryAdaptiveDispatcher( + max_session_permit=max_session_permit, + rate_limiter=RateLimiter( + base_delay=(mean_delay, mean_delay + max_range), + max_delay=60.0, + max_retries=3, + ), + ) + + def transform_result(task_result): + return ( + setattr( + task_result.result, + "dispatch_result", + DispatchResult( + task_id=task_result.task_id, + memory_usage=task_result.memory_usage, + peak_memory=task_result.peak_memory, + start_time=task_result.start_time, + end_time=task_result.end_time, + error_message=task_result.error_message, + ), + ) + or task_result.result + ) + + # Handle stream setting - use first config's stream setting if config is a list + if isinstance(config, list): + stream = config[0].stream if config else False + primary_config = config[0] if config else None + else: + stream = config.stream + primary_config = config + + # Helper to release sticky session if auto_release is enabled + async def maybe_release_session(): + if (primary_config and + primary_config.proxy_session_id and + primary_config.proxy_session_auto_release and + primary_config.proxy_rotation_strategy): + await primary_config.proxy_rotation_strategy.release_session( + primary_config.proxy_session_id + ) + self.logger.info( + message="Auto-released proxy session: {session_id}", + tag="PROXY", + params={"session_id": primary_config.proxy_session_id} + ) + + if stream: + async def result_transformer(): + try: + async for task_result in dispatcher.run_urls_stream( + crawler=self, urls=urls, config=config + ): + yield transform_result(task_result) + finally: + # Auto-release session after streaming completes + await maybe_release_session() + + return result_transformer() + else: + try: + _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config) + return [transform_result(res) for res in _results] + finally: + # Auto-release session after batch completes + await maybe_release_session() + + async def aseed_urls( + self, + domain_or_domains: Union[str, List[str]], + config: Optional[SeedingConfig] = None, + **kwargs + ) -> Union[List[str], Dict[str, List[Union[str, Dict[str, Any]]]]]: + """ + Discovers, filters, and optionally validates URLs for a given domain(s) + using sitemaps and Common Crawl archives. + + Args: + domain_or_domains: A single domain string (e.g., "iana.org") or a list of domains. + config: A SeedingConfig object to control the seeding process. + Parameters passed directly via kwargs will override those in 'config'. + **kwargs: Additional parameters (e.g., `source`, `live_check`, `extract_head`, + `pattern`, `concurrency`, `hits_per_sec`, `force_refresh`, `verbose`) + that will be used to construct or update the SeedingConfig. + + Returns: + If `extract_head` is False: + - For a single domain: `List[str]` of discovered URLs. + - For multiple domains: `Dict[str, List[str]]` mapping each domain to its URLs. + If `extract_head` is True: + - For a single domain: `List[Dict[str, Any]]` where each dict contains 'url' + and 'head_data' (parsed metadata). + - For multiple domains: `Dict[str, List[Dict[str, Any]]]` mapping each domain + to a list of URL data dictionaries. + + Raises: + ValueError: If `domain_or_domains` is not a string or a list of strings. + Exception: Any underlying exceptions from AsyncUrlSeeder or network operations. + + Example: + >>> # Discover URLs from sitemap with live check for 'example.com' + >>> result = await crawler.aseed_urls("example.com", source="sitemap", live_check=True, hits_per_sec=10) + + >>> # Discover URLs from Common Crawl, extract head data for 'example.com' and 'python.org' + >>> multi_domain_result = await crawler.aseed_urls( + >>> ["example.com", "python.org"], + >>> source="cc", extract_head=True, concurrency=200, hits_per_sec=50 + >>> ) + """ + # Initialize AsyncUrlSeeder here if it hasn't been already + if not self.url_seeder: + # Pass the crawler's base_directory for seeder's cache management + # Pass the crawler's logger for consistent logging + self.url_seeder = AsyncUrlSeeder( + base_directory=self.crawl4ai_folder, + logger=self.logger + ) + + # Merge config object with direct kwargs, giving kwargs precedence + seeding_config = config.clone(**kwargs) if config else SeedingConfig.from_kwargs(kwargs) + + # Ensure base_directory is set for the seeder's cache + seeding_config.base_directory = seeding_config.base_directory or self.crawl4ai_folder + # Ensure the seeder uses the crawler's logger (if not already set) + if not self.url_seeder.logger: + self.url_seeder.logger = self.logger + + # Pass verbose setting if explicitly provided in SeedingConfig or kwargs + if seeding_config.verbose is not None: + self.url_seeder.logger.verbose = seeding_config.verbose + else: # Default to crawler's verbose setting + self.url_seeder.logger.verbose = self.logger.verbose + + + if isinstance(domain_or_domains, str): + self.logger.info( + message="Starting URL seeding for domain: {domain}", + tag="SEED", + params={"domain": domain_or_domains} + ) + return await self.url_seeder.urls( + domain_or_domains, + seeding_config + ) + elif isinstance(domain_or_domains, (list, tuple)): + self.logger.info( + message="Starting URL seeding for {count} domains", + tag="SEED", + params={"count": len(domain_or_domains)} + ) + # AsyncUrlSeeder.many_urls directly accepts a list of domains and individual params. + return await self.url_seeder.many_urls( + domain_or_domains, + seeding_config + ) + else: + raise ValueError("`domain_or_domains` must be a string or a list of strings.") + + async def amap_domain( + self, + domain: str, + config: Optional[DomainMapperConfig] = None, + **kwargs, + ) -> List[Dict[str, Any]]: + """ + Discover all URLs under a domain without deep crawling. + + Uses DomainMapper to combine sitemap, Common Crawl, Wayback Machine, + certificate transparency, path probing, robots.txt mining, feed discovery, + and homepage link extraction. + + Args: + domain: Domain to map (e.g., "example.com") + config: DomainMapperConfig object. kwargs override config fields. + + Returns: + List of discovered URL dicts with metadata. + """ + if not self._domain_mapper: + self._domain_mapper = DomainMapper( + logger=self.logger, + base_directory=self.crawl4ai_folder, + ) + + mapper_config = config.clone(**kwargs) if config and kwargs else ( + config or DomainMapperConfig(**kwargs) if kwargs else DomainMapperConfig() + ) + + return await self._domain_mapper.scan(domain, mapper_config) \ No newline at end of file diff --git a/crawl4ai/browser_adapter.py b/crawl4ai/browser_adapter.py new file mode 100644 index 0000000..2f4c158 --- /dev/null +++ b/crawl4ai/browser_adapter.py @@ -0,0 +1,413 @@ +# browser_adapter.py +""" +Browser adapter for Crawl4AI to support both Playwright and undetected browsers +with minimal changes to existing codebase. +""" + +from abc import ABC, abstractmethod +from typing import List, Dict, Any, Optional, Callable +import time +import json + +# Import both, but use conditionally +try: + from playwright.async_api import Page +except ImportError: + Page = Any + +try: + from patchright.async_api import Page as UndetectedPage +except ImportError: + UndetectedPage = Any + + +class BrowserAdapter(ABC): + """Abstract adapter for browser-specific operations""" + + @abstractmethod + async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any: + """Execute JavaScript in the page""" + pass + + @abstractmethod + async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]: + """Setup console message capturing, returns handler function if needed""" + pass + + @abstractmethod + async def setup_error_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]: + """Setup error capturing, returns handler function if needed""" + pass + + @abstractmethod + async def retrieve_console_messages(self, page: Page) -> List[Dict]: + """Retrieve captured console messages (for undetected browsers)""" + pass + + @abstractmethod + async def cleanup_console_capture(self, page: Page, handle_console: Optional[Callable], handle_error: Optional[Callable]): + """Clean up console event listeners""" + pass + + @abstractmethod + def get_imports(self) -> tuple: + """Get the appropriate imports for this adapter""" + pass + + +class PlaywrightAdapter(BrowserAdapter): + """Adapter for standard Playwright""" + + async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any: + """Standard Playwright evaluate""" + if arg is not None: + return await page.evaluate(expression, arg) + return await page.evaluate(expression) + + async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]: + """Setup console capture using Playwright's event system""" + def handle_console_capture(msg): + try: + message_type = "unknown" + try: + message_type = msg.type + except: + pass + + message_text = "unknown" + try: + message_text = msg.text + except: + pass + + entry = { + "type": message_type, + "text": message_text, + "timestamp": time.time() + } + + captured_console.append(entry) + + except Exception as e: + captured_console.append({ + "type": "console_capture_error", + "error": str(e), + "timestamp": time.time() + }) + + page.on("console", handle_console_capture) + return handle_console_capture + + async def setup_error_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]: + """Setup error capture using Playwright's event system""" + def handle_pageerror_capture(err): + try: + error_message = "Unknown error" + try: + error_message = err.message + except: + pass + + error_stack = "" + try: + error_stack = err.stack + except: + pass + + captured_console.append({ + "type": "error", + "text": error_message, + "stack": error_stack, + "timestamp": time.time() + }) + except Exception as e: + captured_console.append({ + "type": "pageerror_capture_error", + "error": str(e), + "timestamp": time.time() + }) + + page.on("pageerror", handle_pageerror_capture) + return handle_pageerror_capture + + async def retrieve_console_messages(self, page: Page) -> List[Dict]: + """Not needed for Playwright - messages are captured via events""" + return [] + + async def cleanup_console_capture(self, page: Page, handle_console: Optional[Callable], handle_error: Optional[Callable]): + """Remove event listeners""" + if handle_console: + page.remove_listener("console", handle_console) + if handle_error: + page.remove_listener("pageerror", handle_error) + + def get_imports(self) -> tuple: + """Return Playwright imports""" + from playwright.async_api import Page, Error + from playwright.async_api import TimeoutError as PlaywrightTimeoutError + return Page, Error, PlaywrightTimeoutError + + +class StealthAdapter(BrowserAdapter): + """Adapter for Playwright with stealth features using playwright_stealth""" + + def __init__(self): + self._console_script_injected = {} + self._stealth = None + self._stealth_available = self._check_stealth_availability() + + def _check_stealth_availability(self) -> bool: + """Check if playwright_stealth is importable and instantiate the Stealth helper.""" + try: + from playwright_stealth import Stealth + except ImportError: + return False + self._stealth = Stealth() + return True + + async def apply_stealth(self, page: Page): + """Apply stealth to a page if available""" + if not (self._stealth_available and self._stealth): + return + try: + await self._stealth.apply_stealth_async(page) + except Exception: + # Fail silently or log error depending on requirements + pass + + async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any: + """Standard Playwright evaluate with stealth applied""" + if arg is not None: + return await page.evaluate(expression, arg) + return await page.evaluate(expression) + + async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]: + """Setup console capture using Playwright's event system with stealth""" + # Apply stealth to the page first + await self.apply_stealth(page) + + def handle_console_capture(msg): + try: + message_type = "unknown" + try: + message_type = msg.type + except: + pass + + message_text = "unknown" + try: + message_text = msg.text + except: + pass + + entry = { + "type": message_type, + "text": message_text, + "timestamp": time.time() + } + + captured_console.append(entry) + + except Exception as e: + captured_console.append({ + "type": "console_capture_error", + "error": str(e), + "timestamp": time.time() + }) + + page.on("console", handle_console_capture) + return handle_console_capture + + async def setup_error_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]: + """Setup error capture using Playwright's event system""" + def handle_pageerror_capture(err): + try: + error_message = "Unknown error" + try: + error_message = err.message + except: + pass + + error_stack = "" + try: + error_stack = err.stack + except: + pass + + captured_console.append({ + "type": "error", + "text": error_message, + "stack": error_stack, + "timestamp": time.time() + }) + except Exception as e: + captured_console.append({ + "type": "pageerror_capture_error", + "error": str(e), + "timestamp": time.time() + }) + + page.on("pageerror", handle_pageerror_capture) + return handle_pageerror_capture + + async def retrieve_console_messages(self, page: Page) -> List[Dict]: + """Not needed for Playwright - messages are captured via events""" + return [] + + async def cleanup_console_capture(self, page: Page, handle_console: Optional[Callable], handle_error: Optional[Callable]): + """Remove event listeners""" + if handle_console: + page.remove_listener("console", handle_console) + if handle_error: + page.remove_listener("pageerror", handle_error) + + def get_imports(self) -> tuple: + """Return Playwright imports""" + from playwright.async_api import Page, Error + from playwright.async_api import TimeoutError as PlaywrightTimeoutError + return Page, Error, PlaywrightTimeoutError + + +class UndetectedAdapter(BrowserAdapter): + """Adapter for undetected browser automation with stealth features""" + + def __init__(self): + self._console_script_injected = {} + + async def evaluate(self, page: UndetectedPage, expression: str, arg: Any = None) -> Any: + """Undetected browser evaluate with isolated context""" + # For most evaluations, use isolated context for stealth + # Only use non-isolated when we need to access our injected console capture + isolated = not ( + "__console" in expression or + "__captured" in expression or + "__error" in expression or + "window.__" in expression + ) + + if arg is not None: + return await page.evaluate(expression, arg, isolated_context=isolated) + return await page.evaluate(expression, isolated_context=isolated) + + async def setup_console_capture(self, page: UndetectedPage, captured_console: List[Dict]) -> Optional[Callable]: + """Setup console capture using JavaScript injection for undetected browsers""" + if not self._console_script_injected.get(page, False): + await page.add_init_script(""" + // Initialize console capture + window.__capturedConsole = []; + window.__capturedErrors = []; + + // Store original console methods + const originalConsole = {}; + ['log', 'info', 'warn', 'error', 'debug'].forEach(method => { + originalConsole[method] = console[method]; + console[method] = function(...args) { + try { + window.__capturedConsole.push({ + type: method, + text: args.map(arg => { + try { + if (typeof arg === 'object') { + return JSON.stringify(arg); + } + return String(arg); + } catch (e) { + return '[Object]'; + } + }).join(' '), + timestamp: Date.now() + }); + } catch (e) { + // Fail silently to avoid detection + } + + // Call original method + originalConsole[method].apply(console, args); + }; + }); + """) + self._console_script_injected[page] = True + + return None # No handler function needed for undetected browser + + async def setup_error_capture(self, page: UndetectedPage, captured_console: List[Dict]) -> Optional[Callable]: + """Setup error capture using JavaScript injection for undetected browsers""" + if not self._console_script_injected.get(page, False): + await page.add_init_script(""" + // Capture errors + window.addEventListener('error', (event) => { + try { + window.__capturedErrors.push({ + type: 'error', + text: event.message, + stack: event.error ? event.error.stack : '', + filename: event.filename, + lineno: event.lineno, + colno: event.colno, + timestamp: Date.now() + }); + } catch (e) { + // Fail silently + } + }); + + // Capture unhandled promise rejections + window.addEventListener('unhandledrejection', (event) => { + try { + window.__capturedErrors.push({ + type: 'unhandledrejection', + text: event.reason ? String(event.reason) : 'Unhandled Promise Rejection', + stack: event.reason && event.reason.stack ? event.reason.stack : '', + timestamp: Date.now() + }); + } catch (e) { + // Fail silently + } + }); + """) + self._console_script_injected[page] = True + + return None # No handler function needed for undetected browser + + async def retrieve_console_messages(self, page: UndetectedPage) -> List[Dict]: + """Retrieve captured console messages and errors from the page""" + messages = [] + + try: + # Get console messages + console_messages = await page.evaluate( + "() => { const msgs = window.__capturedConsole || []; window.__capturedConsole = []; return msgs; }", + isolated_context=False + ) + messages.extend(console_messages) + + # Get errors + errors = await page.evaluate( + "() => { const errs = window.__capturedErrors || []; window.__capturedErrors = []; return errs; }", + isolated_context=False + ) + messages.extend(errors) + + # Convert timestamps from JS to Python format + for msg in messages: + if 'timestamp' in msg and isinstance(msg['timestamp'], (int, float)): + msg['timestamp'] = msg['timestamp'] / 1000.0 # Convert from ms to seconds + + except Exception: + # If retrieval fails, return empty list + pass + + return messages + + async def cleanup_console_capture(self, page: UndetectedPage, handle_console: Optional[Callable], handle_error: Optional[Callable]): + """Clean up for undetected browser - retrieve final messages""" + # For undetected browser, we don't have event listeners to remove + # but we should retrieve any final messages + final_messages = await self.retrieve_console_messages(page) + return final_messages + + def get_imports(self) -> tuple: + """Return undetected browser imports""" + from patchright.async_api import Page, Error + from patchright.async_api import TimeoutError as PlaywrightTimeoutError + return Page, Error, PlaywrightTimeoutError diff --git a/crawl4ai/browser_manager.py b/crawl4ai/browser_manager.py new file mode 100644 index 0000000..d7b8009 --- /dev/null +++ b/crawl4ai/browser_manager.py @@ -0,0 +1,2096 @@ +import asyncio +import time +from typing import Dict, List, Optional, Tuple +import os +import sys +import shutil +import tempfile +import psutil +import signal +import subprocess +import shlex +from playwright.async_api import BrowserContext +import hashlib +from .js_snippet import load_js_script +from .config import DOWNLOAD_PAGE_TIMEOUT +from .async_configs import BrowserConfig, CrawlerRunConfig +from .utils import get_chromium_path +import warnings + + +BROWSER_DISABLE_OPTIONS = [ + "--disable-background-networking", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-breakpad", + "--disable-client-side-phishing-detection", + "--disable-component-extensions-with-background-pages", + "--disable-default-apps", + "--disable-extensions", + "--disable-features=TranslateUI", + "--disable-hang-monitor", + "--disable-ipc-flooding-protection", + "--disable-popup-blocking", + "--disable-prompt-on-repost", + "--disable-sync", + "--force-color-profile=srgb", + "--metrics-recording-only", + "--no-first-run", + "--password-store=basic", + "--use-mock-keychain", +] + + +class ManagedBrowser: + """ + Manages the browser process and context. This class allows to connect to the browser using CDP protocol. + + Attributes: + browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". + Default: "chromium". + user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a + temporary directory may be used. Default: None. + headless (bool): Whether to run the browser in headless mode (no visible GUI). + Default: True. + browser_process (subprocess.Popen): The process object for the browser. + temp_dir (str): Temporary directory for user data if not provided. + debugging_port (int): Port for debugging the browser. + host (str): Host for debugging the browser. + + Methods: + start(): Starts the browser process and returns the CDP endpoint URL. + _get_browser_path(): Returns the browser executable path based on OS and browser type. + _get_browser_args(): Returns browser-specific command line arguments. + _get_user_data_dir(): Returns the user data directory path. + _cleanup(): Terminates the browser process and removes the temporary directory. + create_profile(): Static method to create a user profile by launching a browser for user interaction. + """ + + @staticmethod + def build_browser_flags(config: BrowserConfig) -> List[str]: + """Common CLI flags for launching Chromium""" + flags = [ + "--no-sandbox", + "--disable-dev-shm-usage", + "--no-first-run", + "--no-default-browser-check", + "--disable-infobars", + "--window-position=0,0", + "--ignore-certificate-errors", + "--ignore-certificate-errors-spki-list", + "--disable-blink-features=AutomationControlled", + "--window-position=400,0", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--force-color-profile=srgb", + "--mute-audio", + "--disable-background-timer-throttling", + # Memory-saving flags: disable unused Chrome features + "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider", + "--disable-component-update", + "--disable-domain-reliability", + ] + # GPU flags disable WebGL which anti-bot sensors detect as headless. + # Keep WebGL working (via SwiftShader) when stealth mode is active. + if not config.enable_stealth: + flags.extend([ + "--disable-gpu", + "--disable-gpu-compositing", + "--disable-software-rasterizer", + ]) + if config.memory_saving_mode: + flags.extend([ + "--aggressive-cache-discard", + '--js-flags=--max-old-space-size=512', + ]) + if config.light_mode: + flags.extend(BROWSER_DISABLE_OPTIONS) + if config.text_mode: + flags.extend([ + "--blink-settings=imagesEnabled=false", + "--disable-remote-fonts", + "--disable-images", + "--disable-javascript", + "--disable-software-rasterizer", + "--disable-dev-shm-usage", + ]) + # proxy support — only pass server URL, never credentials. + # Chromium's --proxy-server flag silently ignores inline user:pass@. + # Auth credentials are handled at the Playwright context level instead. + if config.proxy: + flags.append(f"--proxy-server={config.proxy}") + elif config.proxy_config: + flags.append(f"--proxy-server={config.proxy_config.server}") + # dedupe + return list(dict.fromkeys(flags)) + + browser_type: str + user_data_dir: str + headless: bool + browser_process: subprocess.Popen + temp_dir: str + debugging_port: int + host: str + + def __init__( + self, + browser_type: str = "chromium", + user_data_dir: Optional[str] = None, + headless: bool = False, + logger=None, + host: str = "localhost", + debugging_port: int = 9222, + cdp_url: Optional[str] = None, + browser_config: Optional[BrowserConfig] = None, + ): + """ + Initialize the ManagedBrowser instance. + + Args: + browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". + Default: "chromium". + user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a + temporary directory may be used. Default: None. + headless (bool): Whether to run the browser in headless mode (no visible GUI). + Default: True. + logger (logging.Logger): Logger instance for logging messages. Default: None. + host (str): Host for debugging the browser. Default: "localhost". + debugging_port (int): Port for debugging the browser. Default: 9222. + cdp_url (str or None): CDP URL to connect to the browser. Default: None. + browser_config (BrowserConfig): Configuration object containing all browser settings. Default: None. + """ + self.browser_type = browser_config.browser_type + self.user_data_dir = browser_config.user_data_dir + self.headless = browser_config.headless + self.browser_process = None + self.temp_dir = None + self.debugging_port = browser_config.debugging_port + self.host = browser_config.host + self.logger = logger + self.shutting_down = False + self.cdp_url = browser_config.cdp_url + self.browser_config = browser_config + + async def start(self) -> str: + """ + Starts the browser process or returns CDP endpoint URL. + If cdp_url is provided, returns it directly. + If user_data_dir is not provided for local browser, creates a temporary directory. + + Returns: + str: CDP endpoint URL + """ + # If CDP URL provided, just return it + if self.cdp_url: + return self.cdp_url + + # Create temp dir if needed + if not self.user_data_dir: + self.temp_dir = tempfile.mkdtemp(prefix="browser-profile-") + self.user_data_dir = self.temp_dir + + # Get browser path and args based on OS and browser type + # browser_path = self._get_browser_path() + args = await self._get_browser_args() + + if self.browser_config.extra_args: + args.extend(self.browser_config.extra_args) + + + # ── make sure no old Chromium instance is owning the same port/profile ── + try: + if sys.platform == "win32": + if psutil is None: + raise RuntimeError("psutil not available, cannot clean old browser") + for p in psutil.process_iter(["pid", "name", "cmdline"]): + cl = " ".join(p.info.get("cmdline") or []) + if ( + f"--remote-debugging-port={self.debugging_port}" in cl + and f"--user-data-dir={self.user_data_dir}" in cl + ): + p.kill() + p.wait(timeout=5) + else: # macOS / Linux + # kill any process listening on the same debugging port + try: + pids = ( + subprocess.check_output( + shlex.split(f"lsof -t -i:{self.debugging_port}"), + stderr=subprocess.DEVNULL, + ) + .decode() + .strip() + .splitlines() + ) + except (FileNotFoundError, subprocess.CalledProcessError): + pids = [] + for pid in pids: + try: + os.kill(int(pid), signal.SIGTERM) + except ProcessLookupError: + pass + + # remove Chromium singleton locks, or new launch exits with + # “Opening in existing browser session.” + for f in ("SingletonLock", "SingletonSocket", "SingletonCookie"): + fp = os.path.join(self.user_data_dir, f) + if os.path.exists(fp): + os.remove(fp) + except Exception as _e: + # non-fatal — we'll try to start anyway, but log what happened + self.logger.warning(f"pre-launch cleanup failed: {_e}", tag="BROWSER") + + + # Start browser process + try: + # Use DETACHED_PROCESS flag on Windows to fully detach the process + # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group + if sys.platform == "win32": + self.browser_process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP + ) + else: + self.browser_process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setpgrp # Start in a new process group + ) + + # If verbose is True print args used to run the process + if self.logger and self.browser_config.verbose: + self.logger.debug( + f"Starting browser with args: {' '.join(args)}", + tag="BROWSER" + ) + + # We'll monitor for a short time to make sure it starts properly, but won't keep monitoring + await asyncio.sleep(0.5) # Give browser time to start + await self._initial_startup_check() + await asyncio.sleep(2) # Give browser time to start + return f"http://{self.host}:{self.debugging_port}" + except Exception as e: + await self.cleanup() + raise Exception(f"Failed to start browser: {e}") + + async def _initial_startup_check(self): + """ + Perform a quick check to make sure the browser started successfully. + This only runs once at startup rather than continuously monitoring. + """ + if not self.browser_process: + return + + # Check that process started without immediate termination + await asyncio.sleep(0.5) + if self.browser_process.poll() is not None: + # Process already terminated + stdout, stderr = b"", b"" + try: + stdout, stderr = self.browser_process.communicate(timeout=0.5) + except subprocess.TimeoutExpired: + pass + + self.logger.error( + message="Browser process terminated during startup | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}", + tag="ERROR", + params={ + "code": self.browser_process.returncode, + "stdout": stdout.decode() if stdout else "", + "stderr": stderr.decode() if stderr else "", + }, + ) + + async def _monitor_browser_process(self): + """ + Monitor the browser process for unexpected termination. + + How it works: + 1. Read stdout and stderr from the browser process. + 2. If the process has terminated, log the error message and terminate the browser. + 3. If the shutting_down flag is set, log the normal termination message. + 4. If any other error occurs, log the error message. + + Note: This method should be called in a separate task to avoid blocking the main event loop. + This is DEPRECATED and should not be used for builtin browsers that need to outlive the Python process. + """ + if self.browser_process: + try: + stdout, stderr = await asyncio.gather( + asyncio.to_thread(self.browser_process.stdout.read), + asyncio.to_thread(self.browser_process.stderr.read), + ) + + # Check shutting_down flag BEFORE logging anything + if self.browser_process.poll() is not None: + if not self.shutting_down: + self.logger.error( + message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}", + tag="ERROR", + params={ + "code": self.browser_process.returncode, + "stdout": stdout.decode(), + "stderr": stderr.decode(), + }, + ) + await self.cleanup() + else: + self.logger.info( + message="Browser process terminated normally | Code: {code}", + tag="INFO", + params={"code": self.browser_process.returncode}, + ) + except Exception as e: + if not self.shutting_down: + self.logger.error( + message="Error monitoring browser process: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + + def _get_browser_path_WIP(self) -> str: + """Returns the browser executable path based on OS and browser type""" + if sys.platform == "darwin": # macOS + paths = { + "chromium": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "firefox": "/Applications/Firefox.app/Contents/MacOS/firefox", + "webkit": "/Applications/Safari.app/Contents/MacOS/Safari", + } + elif sys.platform == "win32": # Windows + paths = { + "chromium": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "firefox": "C:\\Program Files\\Mozilla Firefox\\firefox.exe", + "webkit": None, # WebKit not supported on Windows + } + else: # Linux + paths = { + "chromium": "google-chrome", + "firefox": "firefox", + "webkit": None, # WebKit not supported on Linux + } + + return paths.get(self.browser_type) + + async def _get_browser_path(self) -> str: + browser_path = await get_chromium_path(self.browser_type) + return browser_path + + async def _get_browser_args(self) -> List[str]: + """Returns full CLI args for launching the browser""" + base = [await self._get_browser_path()] + if self.browser_type == "chromium": + flags = [ + f"--remote-debugging-port={self.debugging_port}", + f"--user-data-dir={self.user_data_dir}", + ] + if self.headless: + flags.append("--headless=new") + # Add viewport flag if specified in config + if self.browser_config.viewport_height and self.browser_config.viewport_width: + flags.append(f"--window-size={self.browser_config.viewport_width},{self.browser_config.viewport_height}") + # merge common launch flags + flags.extend(self.build_browser_flags(self.browser_config)) + elif self.browser_type == "firefox": + flags = [ + "--remote-debugging-port", + str(self.debugging_port), + "--profile", + self.user_data_dir, + ] + if self.headless: + flags.append("--headless") + else: + raise NotImplementedError(f"Browser type {self.browser_type} not supported") + return base + flags + + async def cleanup(self): + """Cleanup browser process and temporary directory""" + # Set shutting_down flag BEFORE any termination actions + self.shutting_down = True + + if self.browser_process: + try: + # For builtin browsers that should persist, we should check if it's a detached process + # Only terminate if we have proper control over the process + if not self.browser_process.poll(): + # Process is still running + self.browser_process.terminate() + # Wait for process to end gracefully + for _ in range(10): # 10 attempts, 100ms each + if self.browser_process.poll() is not None: + break + await asyncio.sleep(0.1) + + # Force kill if still running + if self.browser_process.poll() is None: + if sys.platform == "win32": + # On Windows, use taskkill /T to kill the entire process tree + try: + subprocess.run(["taskkill", "/F", "/T", "/PID", str(self.browser_process.pid)]) + except Exception: + self.browser_process.kill() + else: + # On Unix, kill entire process group to reap child processes + try: + os.killpg(os.getpgid(self.browser_process.pid), signal.SIGKILL) + except (ProcessLookupError, OSError): + pass + await asyncio.sleep(0.1) # Brief wait for kill to take effect + + except Exception as e: + self.logger.error( + message="Error terminating browser: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + + if self.temp_dir and os.path.exists(self.temp_dir): + try: + shutil.rmtree(self.temp_dir) + except Exception as e: + self.logger.error( + message="Error removing temporary directory: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + + # These methods have been moved to BrowserProfiler class + @staticmethod + async def create_profile(browser_config=None, profile_name=None, logger=None): + """ + This method has been moved to the BrowserProfiler class. + + Creates a browser profile by launching a browser for interactive user setup + and waits until the user closes it. The profile is stored in a directory that + can be used later with BrowserConfig.user_data_dir. + + Please use BrowserProfiler.create_profile() instead. + + Example: + ```python + from crawl4ai.browser_profiler import BrowserProfiler + + profiler = BrowserProfiler() + profile_path = await profiler.create_profile(profile_name="my-login-profile") + ``` + """ + from .browser_profiler import BrowserProfiler + + # Create a BrowserProfiler instance and delegate to it + profiler = BrowserProfiler(logger=logger) + return await profiler.create_profile(profile_name=profile_name, browser_config=browser_config) + + @staticmethod + def list_profiles(): + """ + This method has been moved to the BrowserProfiler class. + + Lists all available browser profiles in the Crawl4AI profiles directory. + + Please use BrowserProfiler.list_profiles() instead. + + Example: + ```python + from crawl4ai.browser_profiler import BrowserProfiler + + profiler = BrowserProfiler() + profiles = profiler.list_profiles() + ``` + """ + from .browser_profiler import BrowserProfiler + + # Create a BrowserProfiler instance and delegate to it + profiler = BrowserProfiler() + return profiler.list_profiles() + + @staticmethod + def delete_profile(profile_name_or_path): + """ + This method has been moved to the BrowserProfiler class. + + Delete a browser profile by name or path. + + Please use BrowserProfiler.delete_profile() instead. + + Example: + ```python + from crawl4ai.browser_profiler import BrowserProfiler + + profiler = BrowserProfiler() + success = profiler.delete_profile("my-profile") + ``` + """ + from .browser_profiler import BrowserProfiler + + # Create a BrowserProfiler instance and delegate to it + profiler = BrowserProfiler() + return profiler.delete_profile(profile_name_or_path) + + +async def clone_runtime_state( + src: BrowserContext, + dst: BrowserContext, + crawlerRunConfig: CrawlerRunConfig | None = None, + browserConfig: BrowserConfig | None = None, +) -> None: + """ + Bring everything that *can* be changed at runtime from `src` → `dst`. + + 1. Cookies + 2. localStorage (and sessionStorage, same API) + 3. Extra headers, permissions, geolocation if supplied in configs + """ + + # ── 1. cookies ──────────────────────────────────────────────────────────── + cookies = await src.cookies() + if cookies: + await dst.add_cookies(cookies) + + # ── 2. localStorage / sessionStorage ────────────────────────────────────── + state = await src.storage_state() + for origin in state.get("origins", []): + url = origin["origin"] + kvs = origin.get("localStorage", []) + if not kvs: + continue + + page = dst.pages[0] if dst.pages else await dst.new_page() + await page.goto(url, wait_until="domcontentloaded") + for k, v in kvs: + await page.evaluate("(k,v)=>localStorage.setItem(k,v)", k, v) + + # ── 3. runtime-mutable extras from configs ──────────────────────────────── + # headers + if browserConfig and browserConfig.headers: + await dst.set_extra_http_headers(browserConfig.headers) + + # geolocation + if crawlerRunConfig and crawlerRunConfig.geolocation: + await dst.grant_permissions(["geolocation"]) + await dst.set_geolocation( + { + "latitude": crawlerRunConfig.geolocation.latitude, + "longitude": crawlerRunConfig.geolocation.longitude, + "accuracy": crawlerRunConfig.geolocation.accuracy, + } + ) + + return dst + + + +class _CDPConnectionCache: + """ + Class-level cache for Playwright + CDP browser connections. + + When enabled via BrowserConfig(cache_cdp_connection=True), multiple + BrowserManager instances connecting to the same cdp_url will share + a single Playwright subprocess and CDP WebSocket. Reference-counted; + the connection is closed when the last user releases it. + """ + + _cache: Dict[str, Tuple] = {} # cdp_url -> (playwright, browser, ref_count) + _lock: Optional[asyncio.Lock] = None # lazy-init to avoid event loop issues + _lock_loop: Optional[asyncio.AbstractEventLoop] = None + + @classmethod + def _get_lock(cls) -> asyncio.Lock: + loop = asyncio.get_running_loop() + if cls._lock is None or cls._lock_loop is not loop: + cls._lock = asyncio.Lock() + cls._lock_loop = loop + return cls._lock + + @classmethod + async def acquire(cls, cdp_url: str, use_undetected: bool = False): + """Get or create a cached (playwright, browser) for this cdp_url.""" + async with cls._get_lock(): + if cdp_url in cls._cache: + pw, browser, count = cls._cache[cdp_url] + if browser.is_connected(): + cls._cache[cdp_url] = (pw, browser, count + 1) + return pw, browser + # Stale connection — clean up and fall through to create new + try: + await pw.stop() + except Exception: + pass + del cls._cache[cdp_url] + + # Create new connection + if use_undetected: + from patchright.async_api import async_playwright + else: + from playwright.async_api import async_playwright + pw = await async_playwright().start() + browser = await pw.chromium.connect_over_cdp(cdp_url) + cls._cache[cdp_url] = (pw, browser, 1) + return pw, browser + + @classmethod + async def release(cls, cdp_url: str): + """Decrement ref count; close connection when last user releases.""" + async with cls._get_lock(): + if cdp_url not in cls._cache: + return + pw, browser, count = cls._cache[cdp_url] + if count <= 1: + try: + await browser.close() + except Exception: + pass + try: + await pw.stop() + except Exception: + pass + del cls._cache[cdp_url] + else: + cls._cache[cdp_url] = (pw, browser, count - 1) + + @classmethod + async def close_all(cls): + """Force-close all cached connections. Call on application shutdown.""" + async with cls._get_lock(): + for cdp_url in list(cls._cache.keys()): + pw, browser, _ = cls._cache[cdp_url] + try: + await browser.close() + except Exception: + pass + try: + await pw.stop() + except Exception: + pass + cls._cache.clear() + + +class BrowserManager: + """ + Manages the browser instance and context. + + Attributes: + config (BrowserConfig): Configuration object containing all browser settings + logger: Logger instance for recording events and errors + browser (Browser): The browser instance + default_context (BrowserContext): The default browser context + managed_browser (ManagedBrowser): The managed browser instance + playwright (Playwright): The Playwright instance + sessions (dict): Dictionary to store session information + session_ttl (int): Session timeout in seconds + """ + + _playwright_instance = None + + # Class-level tracking of pages in use, keyed by browser endpoint (CDP URL or instance id) + # This ensures multiple BrowserManager instances connecting to the same browser + # share the same page tracking, preventing race conditions. + _global_pages_in_use: dict = {} # endpoint_key -> set of pages + _global_pages_lock: asyncio.Lock = None # Initialized lazily + + @classmethod + def _get_global_lock(cls) -> asyncio.Lock: + """Get or create the global pages lock (lazy initialization for async context).""" + if cls._global_pages_lock is None: + cls._global_pages_lock = asyncio.Lock() + return cls._global_pages_lock + + @classmethod + async def get_playwright(cls, use_undetected: bool = False): + if use_undetected: + from patchright.async_api import async_playwright + else: + from playwright.async_api import async_playwright + cls._playwright_instance = await async_playwright().start() + return cls._playwright_instance + + def __init__(self, browser_config: BrowserConfig, logger=None, use_undetected: bool = False): + """ + Initialize the BrowserManager with a browser configuration. + + Args: + browser_config (BrowserConfig): Configuration object containing all browser settings + logger: Logger instance for recording events and errors + use_undetected (bool): Whether to use undetected browser (Patchright) + """ + self.config: BrowserConfig = browser_config + self.logger = logger + self.use_undetected = use_undetected + + # Browser state + self.browser = None + self.default_context = None + self.managed_browser = None + self.playwright = None + self._using_cached_cdp = False + self._launched_persistent = False # True when using launch_persistent_context + + # Session management + self.sessions = {} + self.session_ttl = 1800 # 30 minutes + + # Keep track of contexts by a "config signature," so each unique config reuses a single context + self.contexts_by_config = {} + self._contexts_lock = asyncio.Lock() + + # Context lifecycle tracking for LRU eviction + self._context_refcounts = {} # sig -> int (active crawls using this context) + self._context_last_used = {} # sig -> float (monotonic timestamp for LRU) + self._page_to_sig = {} # page -> sig (for decrement lookup on release) + self._max_contexts = 20 # LRU eviction threshold + + # Serialize context.new_page() across concurrent tasks to avoid races + # when using a shared persistent context (context.pages may be empty + # for all racers). Prevents 'Target page/context closed' errors. + self._page_lock = asyncio.Lock() + + # Browser endpoint key for global page tracking (set after browser starts) + self._browser_endpoint_key: Optional[str] = None + + # Browser recycling state (version-based approach) + self._pages_served = 0 + self._browser_version = 1 # included in signature, bump to create new browser + self._pending_cleanup = {} # old_sig -> {"browser": browser, "contexts": [...], "done": Event} + self._pending_cleanup_lock = asyncio.Lock() + self._max_pending_browsers = 3 # safety cap — block if too many draining + self._cleanup_slot_available = asyncio.Event() + self._cleanup_slot_available.set() # starts open + + # Stealth adapter for stealth mode + self._stealth_adapter = None + if self.config.enable_stealth and not self.use_undetected: + from .browser_adapter import StealthAdapter + self._stealth_adapter = StealthAdapter() + + # Initialize ManagedBrowser if needed + if self.config.use_managed_browser: + self.managed_browser = ManagedBrowser( + browser_type=self.config.browser_type, + user_data_dir=self.config.user_data_dir, + headless=self.config.headless, + logger=self.logger, + debugging_port=self.config.debugging_port, + cdp_url=self.config.cdp_url, + browser_config=self.config, + ) + + async def start(self): + """ + Start the browser instance and set up the default context. + + How it works: + 1. Check if Playwright is already initialized. + 2. If not, initialize Playwright. + 3. If managed browser is used, start it and connect to the CDP endpoint. + 4. If managed browser is not used, launch the browser and set up the default context. + + Note: This method should be called in a separate task to avoid blocking the main event loop. + """ + if self.playwright is not None: + await self.close() + + # Use cached CDP connection if enabled and cdp_url is set + if self.config.cache_cdp_connection and self.config.cdp_url: + self._using_cached_cdp = True + self.config.use_managed_browser = True + self.playwright, self.browser = await _CDPConnectionCache.acquire( + self.config.cdp_url, self.use_undetected + ) + else: + self._using_cached_cdp = False + if self.use_undetected: + from patchright.async_api import async_playwright + else: + from playwright.async_api import async_playwright + + # Initialize playwright + self.playwright = await async_playwright().start() + + # ── Persistent context via Playwright's native API ────────────── + # When use_persistent_context is set and we're not connecting to an + # external CDP endpoint, use launch_persistent_context() instead of + # subprocess + CDP. This properly supports proxy authentication + # (server + username + password) which the --proxy-server CLI flag + # cannot handle. + if ( + self.config.use_persistent_context + and not self.config.cdp_url + and not self._using_cached_cdp + ): + # Collect stealth / optimization CLI flags, excluding ones that + # launch_persistent_context handles via keyword arguments. + _skip_prefixes = ( + "--proxy-server", + "--remote-debugging-port", + "--user-data-dir", + "--headless", + "--window-size", + ) + cli_args = [ + flag + for flag in ManagedBrowser.build_browser_flags(self.config) + if not flag.startswith(_skip_prefixes) + ] + if self.config.extra_args: + cli_args.extend(self.config.extra_args) + + launch_kwargs = { + "headless": self.config.headless, + "args": list(dict.fromkeys(cli_args)), # dedupe + "viewport": { + "width": self.config.viewport_width, + "height": self.config.viewport_height, + }, + "user_agent": self.config.user_agent or None, + "ignore_https_errors": self.config.ignore_https_errors, + "accept_downloads": self.config.accept_downloads, + } + + if self.config.proxy_config: + launch_kwargs["proxy"] = { + "server": self.config.proxy_config.server, + "username": self.config.proxy_config.username, + "password": self.config.proxy_config.password, + } + + if self.config.storage_state: + launch_kwargs["storage_state"] = self.config.storage_state + + user_data_dir = self.config.user_data_dir or tempfile.mkdtemp( + prefix="crawl4ai-persistent-" + ) + + self.default_context = ( + await self.playwright.chromium.launch_persistent_context( + user_data_dir, **launch_kwargs + ) + ) + self.browser = None # persistent context has no separate Browser + self._launched_persistent = True + + await self.setup_context(self.default_context) + + # Set the browser endpoint key for global page tracking + self._browser_endpoint_key = self._compute_browser_endpoint_key() + if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: + BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() + return + + if self.config.cdp_url or self.config.use_managed_browser: + self.config.use_managed_browser = True + + if not self._using_cached_cdp: + cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url + + # Add CDP endpoint verification before connecting + if not await self._verify_cdp_ready(cdp_url): + raise Exception(f"CDP endpoint at {cdp_url} is not ready after startup") + + self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url) + + contexts = self.browser.contexts + + # If browser_context_id is provided, we're using a pre-created context + if self.config.browser_context_id: + if self.logger: + self.logger.debug( + f"Using pre-existing browser context: {self.config.browser_context_id}", + tag="BROWSER" + ) + # When connecting to a pre-created context, it should be in contexts + if contexts: + self.default_context = contexts[0] + if self.logger: + self.logger.debug( + f"Found {len(contexts)} existing context(s), using first one", + tag="BROWSER" + ) + else: + # Context was created but not yet visible - wait a bit + await asyncio.sleep(0.2) + contexts = self.browser.contexts + if contexts: + self.default_context = contexts[0] + else: + # Still no contexts - this shouldn't happen with pre-created context + if self.logger: + self.logger.warning( + "Pre-created context not found, creating new one", + tag="BROWSER" + ) + self.default_context = await self.create_browser_context() + elif contexts: + self.default_context = contexts[0] + else: + self.default_context = await self.create_browser_context() + await self.setup_context(self.default_context) + else: + browser_args = self._build_browser_args() + + # Launch appropriate browser type + if self.config.browser_type == "firefox": + self.browser = await self.playwright.firefox.launch(**browser_args) + elif self.config.browser_type == "webkit": + self.browser = await self.playwright.webkit.launch(**browser_args) + else: + self.browser = await self.playwright.chromium.launch(**browser_args) + + self.default_context = self.browser + + # Set the browser endpoint key for global page tracking + self._browser_endpoint_key = self._compute_browser_endpoint_key() + # Initialize global tracking set for this endpoint if needed + if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: + BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() + + def _compute_browser_endpoint_key(self) -> str: + """ + Compute a unique key identifying this browser connection. + + For CDP connections, uses the normalized CDP URL so all BrowserManager + instances connecting to the same browser share page tracking. + For standalone browsers, uses instance id since each is independent. + + Returns: + str: Unique identifier for this browser connection + """ + # For CDP connections, use the CDP URL as the key (normalized) + if self.config.cdp_url: + return self._normalize_cdp_url(self.config.cdp_url) + + # For managed browsers, use the CDP URL/port that was assigned + if self.managed_browser: + # Use debugging port as the key since it uniquely identifies the browser + port = getattr(self.managed_browser, 'debugging_port', None) + host = getattr(self.managed_browser, 'host', 'localhost') + if port: + return f"cdp:http://{host}:{port}" + + # For standalone browsers, use instance id (no sharing needed) + return f"instance:{id(self)}" + + def _normalize_cdp_url(self, cdp_url: str) -> str: + """ + Normalize a CDP URL to a canonical form for consistent tracking. + + Handles various formats: + - http://localhost:9222 + - ws://localhost:9222/devtools/browser/xxx + - http://localhost:9222?browser_id=xxx + + Returns: + str: Normalized CDP key in format "cdp:http://host:port" + """ + from urllib.parse import urlparse + + parsed = urlparse(cdp_url) + host = parsed.hostname or 'localhost' + port = parsed.port or 9222 + + return f"cdp:http://{host}:{port}" + + def _get_pages_in_use(self) -> set: + """Get the set of pages currently in use for this browser.""" + if self._browser_endpoint_key and self._browser_endpoint_key in BrowserManager._global_pages_in_use: + return BrowserManager._global_pages_in_use[self._browser_endpoint_key] + # Fallback: shouldn't happen, but return empty set + return set() + + def _mark_page_in_use(self, page) -> None: + """Mark a page as in use.""" + if self._browser_endpoint_key: + if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: + BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() + BrowserManager._global_pages_in_use[self._browser_endpoint_key].add(page) + + def _release_page_from_use(self, page) -> None: + """Release a page from the in-use tracking.""" + if self._browser_endpoint_key and self._browser_endpoint_key in BrowserManager._global_pages_in_use: + BrowserManager._global_pages_in_use[self._browser_endpoint_key].discard(page) + + async def _verify_cdp_ready(self, cdp_url: str) -> bool: + """Verify CDP endpoint is ready with exponential backoff. + + Supports multiple URL formats: + - HTTP URLs: http://localhost:9222 + - HTTP URLs with query params: http://localhost:9222?browser_id=XXX + - WebSocket URLs: ws://localhost:9222/devtools/browser/XXX + """ + import aiohttp + from urllib.parse import urlparse, urlunparse + + # If WebSocket URL, Playwright handles connection directly - skip HTTP verification + if cdp_url.startswith(('ws://', 'wss://')): + self.logger.debug(f"WebSocket CDP URL provided, skipping HTTP verification", tag="BROWSER") + return True + + # Parse HTTP URL and properly construct /json/version endpoint + parsed = urlparse(cdp_url) + # Build URL with /json/version path, preserving query params + verify_url = urlunparse(( + parsed.scheme, + parsed.netloc, + '/json/version', # Always use this path for verification + '', # params + parsed.query, # preserve query string + '' # fragment + )) + + self.logger.debug(f"Starting CDP verification for {verify_url}", tag="BROWSER") + for attempt in range(5): + try: + async with aiohttp.ClientSession() as session: + async with session.get(verify_url, timeout=aiohttp.ClientTimeout(total=2)) as response: + if response.status == 200: + self.logger.debug(f"CDP endpoint ready after {attempt + 1} attempts", tag="BROWSER") + return True + except Exception as e: + self.logger.debug(f"CDP check attempt {attempt + 1} failed: {e}", tag="BROWSER") + delay = 0.5 * (1.4 ** attempt) + self.logger.debug(f"Waiting {delay:.2f}s before next CDP check...", tag="BROWSER") + await asyncio.sleep(delay) + self.logger.debug(f"CDP verification failed after 5 attempts", tag="BROWSER") + return False + + def _build_browser_args(self) -> dict: + """Build browser launch arguments from config.""" + args = [ + "--disable-gpu", + "--disable-gpu-compositing", + "--disable-software-rasterizer", + "--no-sandbox", + "--disable-dev-shm-usage", + "--no-first-run", + "--no-default-browser-check", + "--disable-infobars", + "--window-position=0,0", + "--ignore-certificate-errors", + "--ignore-certificate-errors-spki-list", + "--disable-blink-features=AutomationControlled", + "--window-position=400,0", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--force-color-profile=srgb", + "--mute-audio", + "--disable-background-timer-throttling", + # Memory-saving flags: disable unused Chrome features + "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider", + "--disable-component-update", + "--disable-domain-reliability", + # "--single-process", + f"--window-size={self.config.viewport_width},{self.config.viewport_height}", + ] + + if self.config.memory_saving_mode: + args.extend([ + "--aggressive-cache-discard", + '--js-flags=--max-old-space-size=512', + ]) + + if self.config.light_mode: + args.extend(BROWSER_DISABLE_OPTIONS) + + if self.config.text_mode: + args.extend( + [ + "--blink-settings=imagesEnabled=false", + "--disable-remote-fonts", + "--disable-images", + "--disable-javascript", + "--disable-software-rasterizer", + "--disable-dev-shm-usage", + ] + ) + + if self.config.extra_args: + args.extend(self.config.extra_args) + + # Deduplicate args + args = list(dict.fromkeys(args)) + + browser_args = {"headless": self.config.headless, "args": args} + + # On Windows, passing channel='chromium' (the default) causes Playwright + # to look for a system Chrome installation instead of using the bundled + # ms-playwright binary. This makes Chrome exit immediately with code 0, + # resulting in TargetClosedError. Skip the default channel. + if self.config.chrome_channel and self.config.chrome_channel != "chromium": + browser_args["channel"] = self.config.chrome_channel + + if self.config.accept_downloads: + browser_args["downloads_path"] = self.config.downloads_path or os.path.join( + os.getcwd(), "downloads" + ) + os.makedirs(browser_args["downloads_path"], exist_ok=True) + + if self.config.proxy: + warnings.warn( + "BrowserConfig.proxy is deprecated and ignored. Use proxy_config instead.", + DeprecationWarning, + ) + if self.config.proxy_config: + from playwright.async_api import ProxySettings + + proxy_settings = ProxySettings( + server=self.config.proxy_config.server, + username=self.config.proxy_config.username, + password=self.config.proxy_config.password, + ) + browser_args["proxy"] = proxy_settings + + return browser_args + + async def setup_context( + self, + context: BrowserContext, + crawlerRunConfig: CrawlerRunConfig = None, + is_default=False, + ): + """ + Set up a browser context with the configured options. + + How it works: + 1. Set extra HTTP headers if provided. + 2. Add cookies if provided. + 3. Load storage state if provided. + 4. Accept downloads if enabled. + 5. Set default timeouts for navigation and download. + 6. Set user agent if provided. + 7. Set browser hints if provided. + 8. Set proxy if provided. + 9. Set downloads path if provided. + 10. Set storage state if provided. + 11. Set cache if provided. + 12. Set extra HTTP headers if provided. + 13. Add cookies if provided. + 14. Set default timeouts for navigation and download if enabled. + 15. Set user agent if provided. + 16. Set browser hints if provided. + + Args: + context (BrowserContext): The browser context to set up + crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings + is_default (bool): Flag indicating if this is the default context + Returns: + None + """ + if self.config.headers: + await context.set_extra_http_headers(self.config.headers) + + if self.config.cookies: + await context.add_cookies(self.config.cookies) + + if self.config.storage_state: + await context.storage_state(path=None) + + if self.config.accept_downloads: + context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT) + context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT) + if self.config.downloads_path: + context._impl_obj._options["accept_downloads"] = True + context._impl_obj._options[ + "downloads_path" + ] = self.config.downloads_path + + # Handle user agent and browser hints + if self.config.user_agent: + combined_headers = { + "User-Agent": self.config.user_agent, + "sec-ch-ua": self.config.browser_hint, + } + combined_headers.update(self.config.headers) + await context.set_extra_http_headers(combined_headers) + + # Add default cookie (skip for raw:/file:// URLs which are not valid cookie URLs) + cookie_url = None + if crawlerRunConfig and crawlerRunConfig.url: + url = crawlerRunConfig.url + # Only set cookie for http/https URLs + if url.startswith(("http://", "https://")): + cookie_url = url + elif crawlerRunConfig.base_url and crawlerRunConfig.base_url.startswith(("http://", "https://")): + # Use base_url as fallback for raw:/file:// URLs + cookie_url = crawlerRunConfig.base_url + + if cookie_url: + await context.add_cookies( + [ + { + "name": "cookiesEnabled", + "value": "true", + "url": cookie_url, + } + ] + ) + + # Handle navigator overrides + if crawlerRunConfig: + if ( + crawlerRunConfig.override_navigator + or crawlerRunConfig.simulate_user + or crawlerRunConfig.magic + ): + await context.add_init_script(load_js_script("navigator_overrider")) + context._crawl4ai_nav_overrider_injected = True + + # Force-open closed shadow roots when flatten_shadow_dom is enabled + if crawlerRunConfig and crawlerRunConfig.flatten_shadow_dom: + await context.add_init_script(""" + const _origAttachShadow = Element.prototype.attachShadow; + Element.prototype.attachShadow = function(init) { + return _origAttachShadow.call(this, {...init, mode: 'open'}); + }; + """) + context._crawl4ai_shadow_dom_injected = True + + # Apply custom init_scripts from BrowserConfig (for stealth evasions, etc.) + if self.config.init_scripts: + for script in self.config.init_scripts: + await context.add_init_script(script) + + async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None): + """ + Creates and returns a new browser context with configured settings. + Applies text-only mode settings if text_mode is enabled in config. + + Returns: + Context: Browser context object with the specified configurations + """ + if self.browser is None: + if self._launched_persistent: + raise RuntimeError( + "Cannot create new browser contexts when using " + "use_persistent_context=True. Persistent context uses a " + "single shared context." + ) + raise RuntimeError( + "Browser is not available. It may have been closed, crashed, " + "or not yet started. Ensure the browser is running before " + "creating new contexts." + ) + # Base settings + user_agent = self.config.headers.get("User-Agent", self.config.user_agent) + viewport_settings = { + "width": self.config.viewport_width, + "height": self.config.viewport_height, + } + proxy_settings = {"server": self.config.proxy} if self.config.proxy else None + + # CSS extensions (blocked separately via avoid_css flag) + css_extensions = ["css", "less", "scss", "sass"] + + # Static resource extensions (blocked when text_mode is enabled) + static_extensions = [ + # Images + "jpg", "jpeg", "png", "gif", "webp", "svg", "ico", "bmp", "tiff", "psd", + # Fonts + "woff", "woff2", "ttf", "otf", "eot", + # Media + "mp4", "webm", "ogg", "avi", "mov", "wmv", "flv", "m4v", + "mp3", "wav", "aac", "m4a", "opus", "flac", + # Documents + "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", + # Archives + "zip", "rar", "7z", "tar", "gz", + # Scripts and data + "xml", "swf", "wasm", + ] + + # Ad and tracker domain patterns (curated from uBlock/EasyList sources) + ad_tracker_patterns = [ + "**/google-analytics.com/**", + "**/googletagmanager.com/**", + "**/googlesyndication.com/**", + "**/doubleclick.net/**", + "**/adservice.google.com/**", + "**/adsystem.com/**", + "**/adzerk.net/**", + "**/adnxs.com/**", + "**/ads.linkedin.com/**", + "**/facebook.net/**", + "**/analytics.twitter.com/**", + "**/ads-twitter.com/**", + "**/hotjar.com/**", + "**/clarity.ms/**", + "**/scorecardresearch.com/**", + "**/pixel.wp.com/**", + "**/amazon-adsystem.com/**", + "**/mixpanel.com/**", + "**/segment.com/**", + ] + + # Common context settings + context_settings = { + "user_agent": user_agent, + "viewport": viewport_settings, + "proxy": proxy_settings, + "accept_downloads": self.config.accept_downloads, + "storage_state": self.config.storage_state, + "ignore_https_errors": self.config.ignore_https_errors, + "device_scale_factor": self.config.device_scale_factor, + "java_script_enabled": self.config.java_script_enabled, + } + + if crawlerRunConfig: + # Check if there is value for crawlerRunConfig.proxy_config set add that to context + if crawlerRunConfig.proxy_config: + from playwright.async_api import ProxySettings + proxy_settings = ProxySettings( + server=crawlerRunConfig.proxy_config.server, + username=crawlerRunConfig.proxy_config.username, + password=crawlerRunConfig.proxy_config.password, + ) + context_settings["proxy"] = proxy_settings + + if self.config.text_mode: + text_mode_settings = { + "has_touch": False, + "is_mobile": False, + } + # Update context settings with text mode settings + context_settings.update(text_mode_settings) + + # inject locale / tz / geo if user provided them + if crawlerRunConfig: + if crawlerRunConfig.locale: + context_settings["locale"] = crawlerRunConfig.locale + if crawlerRunConfig.timezone_id: + context_settings["timezone_id"] = crawlerRunConfig.timezone_id + if crawlerRunConfig.geolocation: + context_settings["geolocation"] = { + "latitude": crawlerRunConfig.geolocation.latitude, + "longitude": crawlerRunConfig.geolocation.longitude, + "accuracy": crawlerRunConfig.geolocation.accuracy, + } + # ensure geolocation permission + perms = context_settings.get("permissions", []) + perms.append("geolocation") + context_settings["permissions"] = perms + + # Create and return the context with all settings + context = await self.browser.new_context(**context_settings) + + # Build dynamic blocking list based on config flags + to_block = [] + if self.config.avoid_css: + to_block.extend(css_extensions) + if self.config.text_mode: + to_block.extend(static_extensions) + + if to_block: + for ext in to_block: + await context.route(f"**/*.{ext}", lambda route: route.abort()) + + if self.config.avoid_ads: + for pattern in ad_tracker_patterns: + await context.route(pattern, lambda route: route.abort()) + + return context + + def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str: + """ + Hash ONLY the CrawlerRunConfig fields that affect browser context + creation (create_browser_context) or context setup (setup_context). + + Whitelist approach: fields like css_selector, word_count_threshold, + screenshot, verbose, etc. do NOT cause a new context to be created. + """ + import json + + sig_dict = {} + + # Fields that flow into create_browser_context() + pc = crawlerRunConfig.proxy_config + if pc is not None: + sig_dict["proxy_config"] = { + "server": getattr(pc, "server", None), + "username": getattr(pc, "username", None), + "password": getattr(pc, "password", None), + } + else: + sig_dict["proxy_config"] = None + + sig_dict["locale"] = crawlerRunConfig.locale + sig_dict["timezone_id"] = crawlerRunConfig.timezone_id + + geo = crawlerRunConfig.geolocation + if geo is not None: + sig_dict["geolocation"] = { + "latitude": geo.latitude, + "longitude": geo.longitude, + "accuracy": geo.accuracy, + } + else: + sig_dict["geolocation"] = None + + # Fields that flow into setup_context() as init scripts + sig_dict["override_navigator"] = crawlerRunConfig.override_navigator + sig_dict["simulate_user"] = crawlerRunConfig.simulate_user + sig_dict["magic"] = crawlerRunConfig.magic + + # Browser version — bumped on recycle to force new browser instance + sig_dict["_browser_version"] = self._browser_version + + signature_json = json.dumps(sig_dict, sort_keys=True, default=str) + return hashlib.sha256(signature_json.encode("utf-8")).hexdigest() + + def _evict_lru_context_locked(self): + """ + If contexts exceed the limit, find the least-recently-used context + with zero active crawls and remove it from all tracking dicts. + + MUST be called while holding self._contexts_lock. + + Returns the BrowserContext to close (caller closes it OUTSIDE the + lock), or None if no eviction is needed or possible. + """ + if len(self.contexts_by_config) <= self._max_contexts: + return None + + # Sort candidates by last-used timestamp (oldest first) + candidates = sorted( + self._context_last_used.items(), + key=lambda item: item[1], + ) + for evict_sig, _ in candidates: + if self._context_refcounts.get(evict_sig, 0) == 0: + ctx = self.contexts_by_config.pop(evict_sig, None) + self._context_refcounts.pop(evict_sig, None) + self._context_last_used.pop(evict_sig, None) + # Clean up stale page->sig mappings for evicted context + stale_pages = [ + p for p, s in self._page_to_sig.items() if s == evict_sig + ] + for p in stale_pages: + del self._page_to_sig[p] + return ctx + + # All contexts are in active use — cannot evict + return None + + async def _apply_stealth_to_page(self, page): + """Apply stealth to a page if stealth mode is enabled""" + if self._stealth_adapter: + try: + await self._stealth_adapter.apply_stealth(page) + except Exception as e: + if self.logger: + self.logger.warning( + message="Failed to apply stealth to page: {error}", + tag="STEALTH", + params={"error": str(e)} + ) + + async def _get_page_by_target_id(self, context: BrowserContext, target_id: str): + """ + Get an existing page by its CDP target ID. + + This is used when connecting to a pre-created browser context with an existing page. + Playwright may not immediately see targets created via raw CDP commands, so we + use CDP to get all targets and find the matching one. + + Args: + context: The browser context to search in + target_id: The CDP target ID to find + + Returns: + Page object if found, None otherwise + """ + try: + # First check if Playwright already sees the page + for page in context.pages: + # Playwright's internal target ID might match + if hasattr(page, '_impl_obj') and hasattr(page._impl_obj, '_target_id'): + if page._impl_obj._target_id == target_id: + return page + + # If not found, try using CDP to get targets + if hasattr(self.browser, '_impl_obj') and hasattr(self.browser._impl_obj, '_connection'): + cdp_session = await context.new_cdp_session(context.pages[0] if context.pages else None) + if cdp_session: + try: + result = await cdp_session.send("Target.getTargets") + targets = result.get("targetInfos", []) + for target in targets: + if target.get("targetId") == target_id: + # Found the target - if it's a page type, we can use it + if target.get("type") == "page": + # The page exists, let Playwright discover it + await asyncio.sleep(0.1) + # Refresh pages list + if context.pages: + return context.pages[0] + finally: + await cdp_session.detach() + + # Fallback: if there are any pages now, return the first one + if context.pages: + return context.pages[0] + + return None + except Exception as e: + if self.logger: + self.logger.warning( + message="Failed to get page by target ID: {error}", + tag="BROWSER", + params={"error": str(e)} + ) + return None + + async def get_page(self, crawlerRunConfig: CrawlerRunConfig): + """ + Get a page for the given session ID, creating a new one if needed. + + Args: + crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings + + Returns: + (page, context): The Page and its BrowserContext + """ + self._cleanup_expired_sessions() + + # If a session_id is provided and we already have it, reuse that page + context + if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions: + context, page, _ = self.sessions[crawlerRunConfig.session_id] + # Update last-used timestamp + self.sessions[crawlerRunConfig.session_id] = (context, page, time.time()) + return page, context + + # If using a managed browser, just grab the shared default_context + if self.config.use_managed_browser: + # If create_isolated_context is True, create isolated contexts for concurrent crawls + # Uses the same caching mechanism as non-CDP mode: cache context by config signature, + # but always create a new page. This prevents navigation conflicts while allowing + # context reuse for multiple URLs with the same config (e.g., batch/deep crawls). + if self.config.create_isolated_context: + config_signature = self._make_config_signature(crawlerRunConfig) + to_close = None + + async with self._contexts_lock: + if config_signature in self.contexts_by_config: + context = self.contexts_by_config[config_signature] + else: + context = await self.create_browser_context(crawlerRunConfig) + await self.setup_context(context, crawlerRunConfig) + self.contexts_by_config[config_signature] = context + self._context_refcounts[config_signature] = 0 + to_close = self._evict_lru_context_locked() + + # Increment refcount INSIDE lock before releasing + self._context_refcounts[config_signature] = ( + self._context_refcounts.get(config_signature, 0) + 1 + ) + self._context_last_used[config_signature] = time.monotonic() + + # Close evicted context OUTSIDE lock + if to_close is not None: + try: + await to_close.close() + except Exception: + pass + + # Always create a new page for each crawl (isolation for navigation) + try: + page = await context.new_page() + except Exception: + async with self._contexts_lock: + if config_signature in self._context_refcounts: + self._context_refcounts[config_signature] = max( + 0, self._context_refcounts[config_signature] - 1 + ) + raise + await self._apply_stealth_to_page(page) + self._page_to_sig[page] = config_signature + elif self.config.storage_state: + tmp_context = await self.create_browser_context(crawlerRunConfig) + ctx = self.default_context # default context, one window only + ctx = await clone_runtime_state(tmp_context, ctx, crawlerRunConfig, self.config) + # Close the temporary context — only needed as a clone source + try: + await tmp_context.close() + except Exception: + pass + context = ctx # so (page, context) return value is correct + # Avoid concurrent new_page on shared persistent context + # See GH-1198: context.pages can be empty under races + async with self._page_lock: + page = await ctx.new_page() + await self._apply_stealth_to_page(page) + else: + context = self.default_context + + # Handle pre-existing target case (for reconnecting to specific CDP targets) + if self.config.browser_context_id and self.config.target_id: + page = await self._get_page_by_target_id(context, self.config.target_id) + if not page: + async with self._page_lock: + page = await context.new_page() + self._mark_page_in_use(page) + await self._apply_stealth_to_page(page) + else: + # Mark pre-existing target as in use + self._mark_page_in_use(page) + else: + # For CDP connections (external browser), multiple Playwright connections + # create separate browser/context objects. Page reuse across connections + # isn't reliable because each connection sees different page objects. + # Always create new pages for CDP to avoid cross-connection race conditions. + if self.config.cdp_url and not self.config.use_managed_browser: + async with self._page_lock: + page = await context.new_page() + self._mark_page_in_use(page) + await self._apply_stealth_to_page(page) + else: + # For managed browsers (single process), page reuse is safe. + # Use lock to safely check for available pages and track usage. + # This prevents race conditions when multiple crawls run concurrently. + async with BrowserManager._get_global_lock(): + pages = context.pages + pages_in_use = self._get_pages_in_use() + # Find first available page (exists and not currently in use) + available_page = next( + (p for p in pages if p not in pages_in_use), + None + ) + if available_page: + page = available_page + else: + # No available pages - create a new one + page = await context.new_page() + await self._apply_stealth_to_page(page) + # Mark page as in use (global tracking) + self._mark_page_in_use(page) + else: + # Otherwise, check if we have an existing context for this config + config_signature = self._make_config_signature(crawlerRunConfig) + to_close = None + + async with self._contexts_lock: + if config_signature in self.contexts_by_config: + context = self.contexts_by_config[config_signature] + else: + # Create and setup a new context + context = await self.create_browser_context(crawlerRunConfig) + await self.setup_context(context, crawlerRunConfig) + self.contexts_by_config[config_signature] = context + self._context_refcounts[config_signature] = 0 + to_close = self._evict_lru_context_locked() + + # Increment refcount INSIDE lock before releasing + self._context_refcounts[config_signature] = ( + self._context_refcounts.get(config_signature, 0) + 1 + ) + self._context_last_used[config_signature] = time.monotonic() + + # Close evicted context OUTSIDE lock + if to_close is not None: + try: + await to_close.close() + except Exception: + pass + + # Create a new page from the chosen context + try: + page = await context.new_page() + except Exception: + async with self._contexts_lock: + if config_signature in self._context_refcounts: + self._context_refcounts[config_signature] = max( + 0, self._context_refcounts[config_signature] - 1 + ) + raise + await self._apply_stealth_to_page(page) + self._page_to_sig[page] = config_signature + + # If a session_id is specified, store this session so we can reuse later + if crawlerRunConfig.session_id: + self.sessions[crawlerRunConfig.session_id] = (context, page, time.time()) + + self._pages_served += 1 + + # Check if browser recycle threshold is hit — bump version for next requests + # This happens AFTER incrementing counter so concurrent requests see correct count + await self._maybe_bump_browser_version() + + return page, context + + async def kill_session(self, session_id: str): + """ + Kill a browser session and clean up resources. + + Args: + session_id (str): The session ID to kill. + """ + if session_id in self.sessions: + context, page, _ = self.sessions[session_id] + self._release_page_from_use(page) + # Decrement context refcount for the session's page + should_close_context = False + async with self._contexts_lock: + sig = self._page_to_sig.pop(page, None) + if sig is not None and sig in self._context_refcounts: + self._context_refcounts[sig] = max( + 0, self._context_refcounts[sig] - 1 + ) + # Only close the context if no other pages are using it + # (refcount dropped to 0) AND we own the context (not managed) + if not self.config.use_managed_browser: + if self._context_refcounts.get(sig, 0) == 0: + self.contexts_by_config.pop(sig, None) + self._context_refcounts.pop(sig, None) + self._context_last_used.pop(sig, None) + should_close_context = True + await page.close() + if should_close_context: + await context.close() + del self.sessions[session_id] + + def release_page(self, page): + """ + Release a page from the in-use tracking set (global tracking). + Sync variant — does NOT decrement context refcount. + """ + self._release_page_from_use(page) + + async def release_page_with_context(self, page): + """ + Release a page and decrement its context's refcount under the lock. + + Should be called from the async crawl finally block instead of + release_page() so the context lifecycle is properly tracked. + """ + self._release_page_from_use(page) + sig = None + refcount = -1 + async with self._contexts_lock: + sig = self._page_to_sig.pop(page, None) + if sig is not None and sig in self._context_refcounts: + self._context_refcounts[sig] = max( + 0, self._context_refcounts[sig] - 1 + ) + refcount = self._context_refcounts[sig] + + # Check if this signature belongs to an old browser waiting to be cleaned up + if sig is not None and refcount == 0: + await self._maybe_cleanup_old_browser(sig) + + def _should_recycle(self) -> bool: + """Check if page threshold reached for browser recycling.""" + limit = self.config.max_pages_before_recycle + if limit <= 0: + return False + return self._pages_served >= limit + + async def _maybe_bump_browser_version(self): + """Bump browser version if threshold reached, moving old browser to pending cleanup. + + New requests automatically get a new browser (via new signature). + Old browser drains naturally and gets cleaned up when refcount hits 0. + """ + if not self._should_recycle(): + return + + # Safety cap: wait if too many old browsers are draining + while True: + async with self._pending_cleanup_lock: + # Re-check threshold under lock (another request may have bumped already) + if not self._should_recycle(): + return + + # Check safety cap + if len(self._pending_cleanup) >= self._max_pending_browsers: + if self.logger: + self.logger.debug( + message="Waiting for old browser to drain (pending: {count})", + tag="BROWSER", + params={"count": len(self._pending_cleanup)}, + ) + self._cleanup_slot_available.clear() + # Release lock and wait + else: + # We have a slot — do the bump inside this lock hold + old_version = self._browser_version + active_sigs = [] + idle_sigs = [] + async with self._contexts_lock: + for sig in list(self._context_refcounts.keys()): + if self._context_refcounts.get(sig, 0) > 0: + active_sigs.append(sig) + else: + idle_sigs.append(sig) + + if self.logger: + self.logger.info( + message="Bumping browser version {old} -> {new} after {count} pages ({active} active, {idle} idle sigs)", + tag="BROWSER", + params={ + "old": old_version, + "new": old_version + 1, + "count": self._pages_served, + "active": len(active_sigs), + "idle": len(idle_sigs), + }, + ) + + # Only add sigs with active crawls to pending cleanup. + # Sigs with refcount 0 are cleaned up immediately below + # to avoid them being stuck in _pending_cleanup forever + # (no future release would trigger their cleanup). + done_event = asyncio.Event() + for sig in active_sigs: + self._pending_cleanup[sig] = { + "version": old_version, + "done": done_event, + } + + # Bump version — new get_page() calls will create new contexts + self._browser_version += 1 + self._pages_served = 0 + + # Clean up idle sigs immediately (outside pending_cleanup_lock below) + break # exit while loop to do cleanup outside locks + + # Safety cap path: wait for a cleanup slot, then retry. + # Timeout prevents permanent deadlock if stuck entries never drain. + try: + await asyncio.wait_for( + self._cleanup_slot_available.wait(), timeout=30.0 + ) + except asyncio.TimeoutError: + # Force-clean any pending entries that have refcount 0 + # (they're stuck and will never drain naturally) + async with self._pending_cleanup_lock: + stuck_sigs = [ + s for s in list(self._pending_cleanup.keys()) + if self._context_refcounts.get(s, 0) == 0 + ] + for sig in stuck_sigs: + self._pending_cleanup.pop(sig, None) + if stuck_sigs: + if self.logger: + self.logger.warning( + message="Force-cleaned {count} stuck pending entries after timeout", + tag="BROWSER", + params={"count": len(stuck_sigs)}, + ) + # Clean up the stuck contexts + for sig in stuck_sigs: + async with self._contexts_lock: + context = self.contexts_by_config.pop(sig, None) + self._context_refcounts.pop(sig, None) + self._context_last_used.pop(sig, None) + if context is not None: + try: + await context.close() + except Exception: + pass + if len(self._pending_cleanup) < self._max_pending_browsers: + self._cleanup_slot_available.set() + + # Reached via break — clean up idle sigs immediately (outside locks) + for sig in idle_sigs: + async with self._contexts_lock: + context = self.contexts_by_config.pop(sig, None) + self._context_refcounts.pop(sig, None) + self._context_last_used.pop(sig, None) + if context is not None: + try: + await context.close() + except Exception: + pass + if idle_sigs and self.logger: + self.logger.debug( + message="Immediately cleaned up {count} idle contexts from version {version}", + tag="BROWSER", + params={"count": len(idle_sigs), "version": old_version}, + ) + + async def _maybe_cleanup_old_browser(self, sig: str): + """Clean up an old browser's context if its refcount hit 0 and it's pending cleanup.""" + async with self._pending_cleanup_lock: + if sig not in self._pending_cleanup: + return # Not an old browser signature + + cleanup_info = self._pending_cleanup.pop(sig) + old_version = cleanup_info["version"] + + if self.logger: + self.logger.debug( + message="Cleaning up context from browser version {version} (sig: {sig})", + tag="BROWSER", + params={"version": old_version, "sig": sig[:12]}, + ) + + # Remove context from tracking + async with self._contexts_lock: + context = self.contexts_by_config.pop(sig, None) + self._context_refcounts.pop(sig, None) + self._context_last_used.pop(sig, None) + + # Close context outside locks + if context is not None: + try: + await context.close() + except Exception: + pass + + # Check if any signatures from this old version remain + remaining_old = [ + s for s, info in self._pending_cleanup.items() + if info["version"] == old_version + ] + + if not remaining_old: + if self.logger: + self.logger.info( + message="All contexts from browser version {version} cleaned up", + tag="BROWSER", + params={"version": old_version}, + ) + + # Open a cleanup slot if we're below the cap + if len(self._pending_cleanup) < self._max_pending_browsers: + self._cleanup_slot_available.set() + + def _cleanup_expired_sessions(self): + """Clean up expired sessions based on TTL.""" + current_time = time.time() + expired_sessions = [ + sid + for sid, (_, _, last_used) in self.sessions.items() + if current_time - last_used > self.session_ttl + ] + for sid in expired_sessions: + asyncio.create_task(self.kill_session(sid)) + + async def close(self): + """Close all browser resources and clean up.""" + # Cached CDP path: only clean up this instance's sessions/contexts, + # then release the shared connection reference. + if self._using_cached_cdp: + session_ids = list(self.sessions.keys()) + for session_id in session_ids: + await self.kill_session(session_id) + for ctx in list(self.contexts_by_config.values()): + try: + await ctx.close() + except Exception: + pass + self.contexts_by_config.clear() + self._context_refcounts.clear() + self._context_last_used.clear() + self._page_to_sig.clear() + await _CDPConnectionCache.release(self.config.cdp_url) + self.browser = None + self.playwright = None + self._using_cached_cdp = False + return + + if self.config.cdp_url: + # When using external CDP, we don't own the browser process. + # If cdp_cleanup_on_close is True, properly disconnect from the browser + # and clean up Playwright resources. This frees the browser for other clients. + if self.config.cdp_cleanup_on_close: + # First close all sessions (pages) + session_ids = list(self.sessions.keys()) + for session_id in session_ids: + await self.kill_session(session_id) + + # Close all contexts we created + for ctx in list(self.contexts_by_config.values()): + try: + await ctx.close() + except Exception: + pass + self.contexts_by_config.clear() + self._context_refcounts.clear() + self._context_last_used.clear() + self._page_to_sig.clear() + + # Disconnect from browser (doesn't terminate it, just releases connection) + if self.browser: + try: + await self.browser.close() + except Exception as e: + if self.logger: + self.logger.debug( + message="Error disconnecting from CDP browser: {error}", + tag="BROWSER", + params={"error": str(e)} + ) + self.browser = None + # Allow time for CDP connection to fully release before another client connects + if self.config.cdp_close_delay > 0: + await asyncio.sleep(self.config.cdp_close_delay) + + # Stop Playwright instance to prevent memory leaks + if self.playwright: + await self.playwright.stop() + self.playwright = None + return + + # ── Persistent context launched via launch_persistent_context ── + if self._launched_persistent: + session_ids = list(self.sessions.keys()) + for session_id in session_ids: + await self.kill_session(session_id) + for ctx in list(self.contexts_by_config.values()): + try: + await ctx.close() + except Exception: + pass + self.contexts_by_config.clear() + self._context_refcounts.clear() + self._context_last_used.clear() + self._page_to_sig.clear() + + # Closing the persistent context also terminates the browser + if self.default_context: + try: + await self.default_context.close() + except Exception: + pass + self.default_context = None + + if self.playwright: + await self.playwright.stop() + self.playwright = None + self._launched_persistent = False + return + + if self.config.sleep_on_close: + await asyncio.sleep(0.5) + + session_ids = list(self.sessions.keys()) + for session_id in session_ids: + await self.kill_session(session_id) + + # Now close all contexts we created. This reclaims memory from ephemeral contexts. + for ctx in list(self.contexts_by_config.values()): + try: + await ctx.close() + except Exception as e: + self.logger.error( + message="Error closing context: {error}", + tag="ERROR", + params={"error": str(e)} + ) + self.contexts_by_config.clear() + self._context_refcounts.clear() + self._context_last_used.clear() + self._page_to_sig.clear() + + if self.browser: + await self.browser.close() + self.browser = None + + if self.managed_browser: + await asyncio.sleep(0.5) + await self.managed_browser.cleanup() + self.managed_browser = None + + if self.playwright: + await self.playwright.stop() + self.playwright = None diff --git a/crawl4ai/browser_profiler.py b/crawl4ai/browser_profiler.py new file mode 100644 index 0000000..c944a59 --- /dev/null +++ b/crawl4ai/browser_profiler.py @@ -0,0 +1,1403 @@ +""" +Browser Profiler Module + +This module provides a dedicated class for managing browser profiles +that can be used for identity-based crawling with Crawl4AI. +""" + +import os +import asyncio +import signal +import sys +import datetime +import uuid +import shutil +import json +import subprocess +import time +from enum import Enum +from pathlib import Path +from typing import List, Dict, Optional, Any, Set +from rich.console import Console + +from .async_configs import BrowserConfig +from .browser_manager import ManagedBrowser +from .async_logger import AsyncLogger, AsyncLoggerBase, LogColor +from .utils import get_home_folder + + +class ShrinkLevel(str, Enum): + """Profile shrink aggressiveness levels.""" + NONE = "none" # Keep everything + LIGHT = "light" # Remove caches only + MEDIUM = "medium" # Caches + history/favicons + AGGRESSIVE = "aggressive" # Auth only (recommended) + MINIMAL = "minimal" # Cookies + localStorage only + + +# Whitelist: what to KEEP at each level (everything else gets deleted) +# Note: "Cookies" can be at root (older Chrome) or in Network/ (Chrome 96+) +# storage_state.json is Playwright's portable cookie format (unencrypted) +# It MUST be kept in all levels for cross-machine profile portability +KEEP_PATTERNS: Dict[ShrinkLevel, Set[str]] = { + ShrinkLevel.NONE: {"*"}, + ShrinkLevel.LIGHT: { + "Network", "Cookies", "Local Storage", "Session Storage", "IndexedDB", + "Preferences", "Secure Preferences", "Login Data", "Login Data For Account", + "Web Data", "History", "History-journal", "Visited Links", "Bookmarks", + "TransportSecurity", "Trust Tokens", "storage_state.json", + }, + ShrinkLevel.MEDIUM: { + "Network", "Cookies", "Local Storage", "Session Storage", "IndexedDB", + "Preferences", "Secure Preferences", "Login Data", "Login Data For Account", + "Web Data", "TransportSecurity", "storage_state.json", + }, + ShrinkLevel.AGGRESSIVE: {"Network", "Cookies", "Local Storage", "IndexedDB", "Preferences", "storage_state.json"}, + ShrinkLevel.MINIMAL: {"Network", "Cookies", "Local Storage", "storage_state.json"}, +} + + +def _get_size(path: Path) -> int: + """Get total size in bytes. Works for files and directories.""" + if path.is_file(): + return path.stat().st_size + total = 0 + try: + for f in path.rglob("*"): + if f.is_file(): + total += f.stat().st_size + except (PermissionError, OSError): + pass + return total + + +def _format_size(n: int) -> str: + """Format bytes as human-readable string.""" + for u in ("B", "KB", "MB", "GB"): + if n < 1024: + return f"{n:.1f} {u}" + n /= 1024 + return f"{n:.1f} TB" + + +def shrink_profile( + profile_path: str, + level: ShrinkLevel = ShrinkLevel.AGGRESSIVE, + dry_run: bool = False +) -> Dict[str, Any]: + """ + Shrink a Chrome profile to reduce storage while preserving auth data. + + Args: + profile_path: Path to profile directory + level: How aggressively to shrink (LIGHT/MEDIUM/AGGRESSIVE/MINIMAL) + dry_run: If True, only report what would be removed + + Returns: + Dict with 'removed', 'kept', 'bytes_freed', 'size_before', 'size_after', 'errors' + """ + if level == ShrinkLevel.NONE: + return {"removed": [], "kept": [], "bytes_freed": 0, "errors": []} + + profile = Path(profile_path) + if not profile.exists() or not profile.is_dir(): + raise ValueError(f"Profile not found: {profile_path}") + + # Chrome profiles may have data in Default/ subdirectory + target = profile / "Default" if (profile / "Default").is_dir() else profile + + keep = KEEP_PATTERNS[level] + result = {"removed": [], "kept": [], "bytes_freed": 0, "errors": [], "size_before": _get_size(profile)} + + for item in target.iterdir(): + name = item.name + # Check if item matches any keep pattern + if any(name == p or name.startswith(p) for p in keep): + result["kept"].append(name) + else: + size = _get_size(item) + if not dry_run: + try: + shutil.rmtree(item) if item.is_dir() else item.unlink() + result["removed"].append(name) + result["bytes_freed"] += size + except Exception as e: + result["errors"].append(f"{name}: {e}") + else: + result["removed"].append(name) + result["bytes_freed"] += size + + result["size_after"] = _get_size(profile) if not dry_run else None + return result + + +class BrowserProfiler: + """ + A dedicated class for managing browser profiles for Crawl4AI. + + The BrowserProfiler allows you to: + - Create browser profiles interactively + - List available profiles + - Delete profiles when no longer needed + - Get profile paths for use in BrowserConfig + + Profiles are stored by default in ~/.crawl4ai/profiles/ + """ + + def __init__(self, logger: Optional[AsyncLoggerBase] = None): + """ + Initialize the BrowserProfiler. + + Args: + logger (AsyncLoggerBase, optional): Logger for outputting messages. + If None, a default AsyncLogger will be created. + """ + # Initialize rich console for colorful input prompts + self.console = Console() + + # Create a logger if not provided + if logger is None: + self.logger = AsyncLogger(verbose=True) + elif not isinstance(logger, AsyncLoggerBase): + self.logger = AsyncLogger(verbose=True) + else: + self.logger = logger + + # Ensure profiles directory exists + self.profiles_dir = os.path.join(get_home_folder(), "profiles") + os.makedirs(self.profiles_dir, exist_ok=True) + + # Builtin browser config file + self.builtin_browser_dir = os.path.join(get_home_folder(), "builtin-browser") + self.builtin_config_file = os.path.join(self.builtin_browser_dir, "browser_config.json") + os.makedirs(self.builtin_browser_dir, exist_ok=True) + + def _is_windows(self) -> bool: + """Check if running on Windows platform.""" + return sys.platform.startswith('win') or sys.platform == 'cygwin' + + def _is_macos(self) -> bool: + """Check if running on macOS platform.""" + return sys.platform == 'darwin' + + def _is_linux(self) -> bool: + """Check if running on Linux platform.""" + return sys.platform.startswith('linux') + + def _get_quit_message(self, tag: str) -> str: + """Get appropriate quit message based on context.""" + if tag == "PROFILE": + return "Closing browser and saving profile..." + elif tag == "CDP": + return "Closing browser..." + else: + return "Closing browser..." + + async def _listen_windows(self, user_done_event, check_browser_process, tag: str): + """Windows-specific keyboard listener using msvcrt.""" + try: + import msvcrt + except ImportError: + raise ImportError("msvcrt module not available on this platform") + + while True: + try: + # Check for keyboard input + if msvcrt.kbhit(): + raw = msvcrt.getch() + + # Handle Unicode decoding more robustly + key = None + try: + key = raw.decode("utf-8") + except UnicodeDecodeError: + try: + # Try different encodings + key = raw.decode("latin1") + except UnicodeDecodeError: + # Skip if we can't decode + continue + + # Validate key + if not key or len(key) != 1: + continue + + # Check for printable characters only + if not key.isprintable(): + continue + + # Check for quit command + if key.lower() == "q": + self.logger.info( + self._get_quit_message(tag), + tag=tag, + base_color=LogColor.GREEN + ) + user_done_event.set() + return + + # Check if browser process ended + if await check_browser_process(): + return + + # Small delay to prevent busy waiting + await asyncio.sleep(0.1) + + except Exception as e: + self.logger.warning(f"Error in Windows keyboard listener: {e}", tag=tag) + # Continue trying instead of failing completely + await asyncio.sleep(0.1) + continue + + async def _listen_unix(self, user_done_event: asyncio.Event, check_browser_process, tag: str): + """Unix/Linux/macOS keyboard listener using termios and select.""" + try: + import termios + import tty + import select + except ImportError: + raise ImportError("termios/tty/select modules not available on this platform") + + # Get stdin file descriptor + try: + fd = sys.stdin.fileno() + except (AttributeError, OSError): + raise ImportError("stdin is not a terminal") + + # Save original terminal settings + old_settings = None + try: + old_settings = termios.tcgetattr(fd) + except termios.error as e: + raise ImportError(f"Cannot get terminal attributes: {e}") + + try: + # Switch to non-canonical mode (cbreak mode) + tty.setcbreak(fd) + + while True: + try: + # Use select to check if input is available (non-blocking) + # Timeout of 0.5 seconds to periodically check browser process + readable, _, _ = select.select([sys.stdin], [], [], 0.5) + + if readable: + # Read one character + key = sys.stdin.read(1) + + if key and key.lower() == "q": + self.logger.info( + self._get_quit_message(tag), + tag=tag, + base_color=LogColor.GREEN + ) + user_done_event.set() + return + + # Check if browser process ended + if await check_browser_process(): + return + + # Small delay to prevent busy waiting + await asyncio.sleep(0.1) + + except (KeyboardInterrupt, EOFError): + # Handle Ctrl+C or EOF gracefully + self.logger.info("Keyboard interrupt received", tag=tag) + user_done_event.set() + return + except Exception as e: + self.logger.warning(f"Error in Unix keyboard listener: {e}", tag=tag) + await asyncio.sleep(0.1) + continue + + finally: + # Always restore terminal settings + if old_settings is not None: + try: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + except Exception as e: + self.logger.error(f"Failed to restore terminal settings: {e}", tag=tag) + + async def _listen_fallback(self, user_done_event: asyncio.Event, check_browser_process, tag: str): + """Fallback keyboard listener using simple input() method.""" + self.logger.info("Using fallback input mode. Type 'q' and press Enter to quit.", tag=tag) + + # Run input in a separate thread to avoid blocking + import threading + import queue + + input_queue = queue.Queue() + + def input_thread(): + """Thread function to handle input.""" + try: + while not user_done_event.is_set(): + try: + # Use input() with a prompt + user_input = input("Press 'q' + Enter to quit: ").strip().lower() + input_queue.put(user_input) + if user_input == 'q': + break + except (EOFError, KeyboardInterrupt): + input_queue.put('q') + break + except Exception as e: + self.logger.warning(f"Error in input thread: {e}", tag=tag) + break + except Exception as e: + self.logger.error(f"Input thread failed: {e}", tag=tag) + + # Start input thread + thread = threading.Thread(target=input_thread, daemon=True) + thread.start() + + try: + while not user_done_event.is_set(): + # Check for user input + try: + user_input = input_queue.get_nowait() + if user_input == 'q': + self.logger.info( + self._get_quit_message(tag), + tag=tag, + base_color=LogColor.GREEN + ) + user_done_event.set() + return + except queue.Empty: + pass + + # Check if browser process ended + if await check_browser_process(): + return + + # Small delay + await asyncio.sleep(0.5) + + except Exception as e: + self.logger.error(f"Fallback listener failed: {e}", tag=tag) + user_done_event.set() + + async def create_profile( + self, + profile_name: Optional[str] = None, + browser_config: Optional[BrowserConfig] = None, + shrink_level: ShrinkLevel = ShrinkLevel.NONE, + ) -> Optional[str]: + """ + Creates a browser profile by launching a browser for interactive user setup + and waits until the user closes it. The profile is stored in a directory that + can be used later with BrowserConfig.user_data_dir. + + Args: + profile_name (str, optional): Name for the profile directory. + If None, a name is generated based on timestamp. + browser_config (BrowserConfig, optional): Configuration for the browser. + If None, a default configuration is used with headless=False. + shrink_level (ShrinkLevel, optional): Optionally shrink profile after creation. + Default is NONE (no shrinking). + + Returns: + str: Path to the created profile directory, or None if creation failed + + Example: + ```python + profiler = BrowserProfiler() + + # Create a profile interactively + profile_path = await profiler.create_profile( + profile_name="my-login-profile" + ) + + # Use the profile in a crawler + browser_config = BrowserConfig( + headless=True, + use_managed_browser=True, + user_data_dir=profile_path + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + # The crawler will now use your profile with all your cookies and login state + result = await crawler.arun("https://example.com/dashboard") + ``` + """ + # Create default browser config if none provided + # IMPORTANT: We disable cookie encryption so profiles can be transferred + # between machines (e.g., local -> cloud). Without this, Chrome encrypts + # cookies with OS keychain which isn't portable. + portable_profile_args = [ + "--password-store=basic", # Linux: use basic store, not gnome-keyring + "--use-mock-keychain", # macOS: use mock keychain, not real one + ] + + if browser_config is None: + from .async_configs import BrowserConfig + browser_config = BrowserConfig( + browser_type="chromium", + headless=False, # Must be visible for user interaction + verbose=True, + extra_args=portable_profile_args, + ) + else: + # Ensure headless is False for user interaction + browser_config.headless = False + # Add portable profile args + if browser_config.extra_args: + browser_config.extra_args.extend(portable_profile_args) + else: + browser_config.extra_args = portable_profile_args + + # Generate profile name if not provided + if not profile_name: + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + profile_name = f"profile_{timestamp}_{uuid.uuid4().hex[:6]}" + + # Sanitize profile name (replace spaces and special chars) + profile_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in profile_name) + + # Set user data directory + profile_path = os.path.join(self.profiles_dir, profile_name) + os.makedirs(profile_path, exist_ok=True) + + # Print instructions for the user with rich formatting + border = f"{'='*80}" + self.logger.info("{border}", tag="PROFILE", params={"border": f"\n{border}"}, colors={"border": LogColor.CYAN}) + self.logger.info("Creating browser profile: {profile_name}", tag="PROFILE", params={"profile_name": profile_name}, colors={"profile_name": LogColor.GREEN}) + self.logger.info("Profile directory: {profile_path}", tag="PROFILE", params={"profile_path": profile_path}, colors={"profile_path": LogColor.YELLOW}) + + self.logger.info("\nInstructions:", tag="PROFILE") + self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE") + self.logger.info("{segment}, configure settings, etc. as needed.", tag="PROFILE", params={"segment": "2. Log in to websites"}, colors={"segment": LogColor.CYAN}) + self.logger.info("3. When you're done, {segment} to close the browser.", tag="PROFILE", params={"segment": "press 'q' in this terminal"}, colors={"segment": LogColor.YELLOW}) + self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE") + self.logger.info("{border}", tag="PROFILE", params={"border": f"{border}\n"}, colors={"border": LogColor.CYAN}) + + browser_config.headless = False + browser_config.user_data_dir = profile_path + + + # Create managed browser instance + managed_browser = ManagedBrowser( + browser_config=browser_config, + # user_data_dir=profile_path, + # headless=False, # Must be visible + logger=self.logger, + # debugging_port=browser_config.debugging_port + ) + + # Set up signal handlers to ensure cleanup on interrupt + original_sigint = signal.getsignal(signal.SIGINT) + original_sigterm = signal.getsignal(signal.SIGTERM) + + # Define cleanup handler for signals + async def cleanup_handler(sig, frame): + self.logger.warning("\nCleaning up browser process...", tag="PROFILE") + await managed_browser.cleanup() + # Restore original signal handlers + signal.signal(signal.SIGINT, original_sigint) + signal.signal(signal.SIGTERM, original_sigterm) + if sig == signal.SIGINT: + self.logger.error("Profile creation interrupted. Profile may be incomplete.", tag="PROFILE") + sys.exit(1) + + # Set signal handlers + def sigint_handler(sig, frame): + asyncio.create_task(cleanup_handler(sig, frame)) + + signal.signal(signal.SIGINT, sigint_handler) + signal.signal(signal.SIGTERM, sigint_handler) + + # Event to signal when user is done with the browser + user_done_event = asyncio.Event() + + # Run keyboard input loop in a separate task + async def listen_for_quit_command(): + """Cross-platform keyboard listener that waits for 'q' key press.""" + # First output the prompt + self.logger.info( + "Press {segment} when you've finished using the browser...", + tag="PROFILE", + params={"segment": "'q'"}, colors={"segment": LogColor.YELLOW}, + base_color=LogColor.CYAN + ) + + async def check_browser_process(): + """Check if browser process is still running.""" + if ( + managed_browser.browser_process + and managed_browser.browser_process.poll() is not None + ): + self.logger.info( + "Browser already closed. Ending input listener.", tag="PROFILE" + ) + user_done_event.set() + return True + return False + + # Try platform-specific implementations with fallback + try: + if self._is_windows(): + await self._listen_windows(user_done_event, check_browser_process, "PROFILE") + else: + await self._listen_unix(user_done_event, check_browser_process, "PROFILE") + except Exception as e: + self.logger.warning(f"Platform-specific keyboard listener failed: {e}", tag="PROFILE") + self.logger.info("Falling back to simple input mode...", tag="PROFILE") + await self._listen_fallback(user_done_event, check_browser_process, "PROFILE") + + try: + from playwright.async_api import async_playwright + + # Start the browser + # await managed_browser.start() + # 1. ── Start the browser ───────────────────────────────────────── + cdp_url = await managed_browser.start() + + # 2. ── Attach Playwright to that running Chrome ────────────────── + pw = await async_playwright().start() + browser = await pw.chromium.connect_over_cdp(cdp_url) + # Grab the existing default context (there is always one) + context = browser.contexts[0] + + # Check if browser started successfully + browser_process = managed_browser.browser_process + if not browser_process: + self.logger.error("Failed to start browser process.", tag="PROFILE") + return None + + self.logger.info("Browser launched. Waiting for you to finish...", tag="PROFILE") + + # Start listening for keyboard input + listener_task = asyncio.create_task(listen_for_quit_command()) + + # Wait for either the user to press 'q' or for the browser process to exit naturally + while not user_done_event.is_set() and browser_process.poll() is None: + await asyncio.sleep(0.5) + + # Cancel the listener task if it's still running + if not listener_task.done(): + listener_task.cancel() + try: + await listener_task + except asyncio.CancelledError: + pass + + # 3. ── Persist storage state *before* we kill Chrome ───────────── + state_file = os.path.join(profile_path, "storage_state.json") + try: + await context.storage_state(path=state_file) + self.logger.info(f"[PROFILE].i storage_state saved → {state_file}", tag="PROFILE") + except Exception as e: + self.logger.warning(f"[PROFILE].w failed to save storage_state: {e}", tag="PROFILE") + + # 4. ── Close everything cleanly ────────────────────────────────── + await browser.close() + await pw.stop() + + # If the browser is still running and the user pressed 'q', terminate it + if browser_process.poll() is None and user_done_event.is_set(): + self.logger.info("Terminating browser process...", tag="PROFILE") + await managed_browser.cleanup() + + self.logger.success(f"Browser closed. Profile saved at: {profile_path}", tag="PROFILE") + + except Exception as e: + self.logger.error(f"Error creating profile: {e!s}", tag="PROFILE") + await managed_browser.cleanup() + return None + finally: + # Restore original signal handlers + signal.signal(signal.SIGINT, original_sigint) + signal.signal(signal.SIGTERM, original_sigterm) + + # Make sure browser is fully cleaned up + await managed_browser.cleanup() + + # Shrink profile if requested + if shrink_level != ShrinkLevel.NONE and profile_path: + self.logger.info(f"Shrinking profile with level: {shrink_level.value}", tag="PROFILE") + self.shrink(profile_path, shrink_level) + + return profile_path + + def list_profiles(self) -> List[Dict[str, Any]]: + """ + Lists all available browser profiles in the Crawl4AI profiles directory. + + Returns: + list: A list of dictionaries containing profile information: + [{"name": "profile_name", "path": "/path/to/profile", "created": datetime, "type": "chromium|firefox"}] + + Example: + ```python + profiler = BrowserProfiler() + + # List all available profiles + profiles = profiler.list_profiles() + + for profile in profiles: + print(f"Profile: {profile['name']}") + print(f" Path: {profile['path']}") + print(f" Created: {profile['created']}") + print(f" Browser type: {profile['type']}") + ``` + """ + if not os.path.exists(self.profiles_dir): + return [] + + profiles = [] + + for name in os.listdir(self.profiles_dir): + profile_path = os.path.join(self.profiles_dir, name) + + # Skip if not a directory + if not os.path.isdir(profile_path): + continue + + # Check if this looks like a valid browser profile + # For Chromium: Look for Preferences file + # For Firefox: Look for prefs.js file + is_valid = False + + if os.path.exists(os.path.join(profile_path, "Preferences")) or \ + os.path.exists(os.path.join(profile_path, "Default", "Preferences")): + is_valid = "chromium" + elif os.path.exists(os.path.join(profile_path, "prefs.js")): + is_valid = "firefox" + + if is_valid: + # Get creation time + created = datetime.datetime.fromtimestamp( + os.path.getctime(profile_path) + ) + + profiles.append({ + "name": name, + "path": profile_path, + "created": created, + "type": is_valid + }) + + # Sort by creation time, newest first + profiles.sort(key=lambda x: x["created"], reverse=True) + + return profiles + + def get_profile_path(self, profile_name: str) -> Optional[str]: + """ + Get the full path to a profile by name. + + Args: + profile_name (str): Name of the profile (not the full path) + + Returns: + str: Full path to the profile directory, or None if not found + + Example: + ```python + profiler = BrowserProfiler() + + path = profiler.get_profile_path("my-profile") + if path: + print(f"Profile path: {path}") + else: + print("Profile not found") + ``` + """ + profile_path = os.path.join(self.profiles_dir, profile_name) + + # Check if path exists and is a valid profile + if not os.path.isdir(profile_path): + # Chrck if profile_name itself is full path + if os.path.isabs(profile_name): + profile_path = profile_name + else: + return None + + # Look for profile indicators + is_profile = ( + os.path.exists(os.path.join(profile_path, "Preferences")) or + os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or + os.path.exists(os.path.join(profile_path, "prefs.js")) + ) + + if not is_profile: + return None # Not a valid browser profile + + return profile_path + + def delete_profile(self, profile_name_or_path: str) -> bool: + """ + Delete a browser profile by name or path. + + Args: + profile_name_or_path (str): Name of the profile or full path to profile directory + + Returns: + bool: True if the profile was deleted successfully, False otherwise + + Example: + ```python + profiler = BrowserProfiler() + + # Delete by name + success = profiler.delete_profile("my-profile") + + # Delete by path + success = profiler.delete_profile("/path/to/.crawl4ai/profiles/my-profile") + ``` + """ + # Determine if input is a name or a path + if os.path.isabs(profile_name_or_path): + # Full path provided + profile_path = profile_name_or_path + else: + # Just a name provided, construct path + profile_path = os.path.join(self.profiles_dir, profile_name_or_path) + + # Check if path exists and is a valid profile + if not os.path.isdir(profile_path): + return False + + # Look for profile indicators + is_profile = ( + os.path.exists(os.path.join(profile_path, "Preferences")) or + os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or + os.path.exists(os.path.join(profile_path, "prefs.js")) + ) + + if not is_profile: + return False # Not a valid browser profile + + # Delete the profile directory + try: + shutil.rmtree(profile_path) + return True + except Exception: + return False + + def shrink( + self, + profile_name_or_path: str, + level: ShrinkLevel = ShrinkLevel.AGGRESSIVE, + dry_run: bool = False + ) -> Dict[str, Any]: + """ + Shrink a profile to reduce storage while preserving authentication data. + + Args: + profile_name_or_path: Profile name or full path + level: LIGHT, MEDIUM, AGGRESSIVE (default), or MINIMAL + dry_run: If True, only preview what would be removed + + Returns: + Dict with 'removed', 'kept', 'bytes_freed', 'size_before', 'size_after', 'errors' + """ + # Resolve path + if os.path.isabs(profile_name_or_path): + profile_path = profile_name_or_path + else: + profile_path = os.path.join(self.profiles_dir, profile_name_or_path) + + if not os.path.isdir(profile_path): + raise ValueError(f"Profile not found: {profile_name_or_path}") + + result = shrink_profile(profile_path, level, dry_run) + + action = "Would free" if dry_run else "Freed" + self.logger.info( + f"{action} {_format_size(result['bytes_freed'])} " + f"({len(result['removed'])} items removed, {len(result['kept'])} kept)", + tag="SHRINK" + ) + + return result + + async def interactive_manager(self, crawl_callback=None): + """ + Launch an interactive profile management console. + + Args: + crawl_callback (callable, optional): Function to call when selecting option to use + a profile for crawling. It will be called with (profile_path, url). + + Example: + ```python + profiler = BrowserProfiler() + + # Define a custom crawl function + async def my_crawl_function(profile_path, url): + print(f"Crawling {url} with profile {profile_path}") + # Implement your crawling logic here + + # Start interactive manager + await profiler.interactive_manager(crawl_callback=my_crawl_function) + ``` + """ + while True: + self.logger.info("\nProfile Management Options:", tag="MENU") + self.logger.info("1. Create a new profile", tag="MENU", base_color=LogColor.GREEN) + self.logger.info("2. List available profiles", tag="MENU", base_color=LogColor.YELLOW) + self.logger.info("3. Delete a profile", tag="MENU", base_color=LogColor.RED) + + # Only show crawl option if callback provided + if crawl_callback: + self.logger.info("4. Use a profile to crawl a website", tag="MENU", base_color=LogColor.CYAN) + self.logger.info("5. Exit", tag="MENU", base_color=LogColor.MAGENTA) + exit_option = "5" + else: + self.logger.info("4. Exit", tag="MENU", base_color=LogColor.MAGENTA) + exit_option = "4" + + self.logger.info(f"\n[cyan]Enter your choice (1-{exit_option}): [/cyan]", end="") + choice = input() + + if choice == "1": + # Create new profile + self.console.print("[green]Enter a name for the new profile (or press Enter for auto-generated name): [/green]", end="") + name = input() + await self.create_profile(name or None) + + elif choice == "2": + # List profiles + profiles = self.list_profiles() + + if not profiles: + self.logger.warning(" No profiles found. Create one first with option 1.", tag="PROFILES") + continue + + # Print profile information + self.logger.info("\nAvailable profiles:", tag="PROFILES") + for i, profile in enumerate(profiles): + self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES") + self.logger.info(f" Path: {profile['path']}", tag="PROFILES", base_color=LogColor.YELLOW) + self.logger.info(f" Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES") + self.logger.info(f" Browser type: {profile['type']}", tag="PROFILES") + self.logger.info("", tag="PROFILES") # Empty line for spacing + + elif choice == "3": + # Delete profile + profiles = self.list_profiles() + if not profiles: + self.logger.warning("No profiles found to delete", tag="PROFILES") + continue + + # Display numbered list + self.logger.info("\nAvailable profiles:", tag="PROFILES", base_color=LogColor.YELLOW) + for i, profile in enumerate(profiles): + self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES") + + # Get profile to delete + self.console.print("[red]Enter the number of the profile to delete (or 'c' to cancel): [/red]", end="") + profile_idx = input() + if profile_idx.lower() == 'c': + continue + + try: + idx = int(profile_idx) - 1 + if 0 <= idx < len(profiles): + profile_name = profiles[idx]["name"] + self.logger.info(f"Deleting profile: [yellow]{profile_name}[/yellow]", tag="PROFILES") + + # Confirm deletion + self.console.print("[red]Are you sure you want to delete this profile? (y/n): [/red]", end="") + confirm = input() + if confirm.lower() == 'y': + success = self.delete_profile(profiles[idx]["path"]) + + if success: + self.logger.success(f"Profile {profile_name} deleted successfully", tag="PROFILES") + else: + self.logger.error(f"Failed to delete profile {profile_name}", tag="PROFILES") + else: + self.logger.error("Invalid profile number", tag="PROFILES") + except ValueError: + self.logger.error("Please enter a valid number", tag="PROFILES") + + elif choice == "4" and crawl_callback: + # Use profile to crawl a site + profiles = self.list_profiles() + if not profiles: + self.logger.warning("No profiles found. Create one first.", tag="PROFILES") + continue + + # Display numbered list + self.logger.info("\nAvailable profiles:", tag="PROFILES", base_color=LogColor.YELLOW) + for i, profile in enumerate(profiles): + self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES") + + # Get profile to use + self.console.print("[cyan]Enter the number of the profile to use (or 'c' to cancel): [/cyan]", end="") + profile_idx = input() + if profile_idx.lower() == 'c': + continue + + try: + idx = int(profile_idx) - 1 + if 0 <= idx < len(profiles): + profile_path = profiles[idx]["path"] + self.console.print("[cyan]Enter the URL to crawl: [/cyan]", end="") + url = input() + if url: + # Call the provided crawl callback + await crawl_callback(profile_path, url) + else: + self.logger.error("No URL provided", tag="CRAWL") + else: + self.logger.error("Invalid profile number", tag="PROFILES") + except ValueError: + self.logger.error("Please enter a valid number", tag="PROFILES") + + elif choice == exit_option: + # Exit + self.logger.info("Exiting profile management", tag="MENU") + break + + else: + self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU") + + async def launch_standalone_browser(self, + browser_type: str = "chromium", + user_data_dir: Optional[str] = None, + debugging_port: int = 9222, + headless: bool = False, + save_as_builtin: bool = False) -> Optional[str]: + """ + Launch a standalone browser with CDP debugging enabled and keep it running + until the user presses 'q'. Returns and displays the CDP URL. + + Args: + browser_type (str): Type of browser to launch ('chromium' or 'firefox') + user_data_dir (str, optional): Path to user profile directory + debugging_port (int): Port to use for CDP debugging + headless (bool): Whether to run in headless mode + + Returns: + str: CDP URL for the browser, or None if launch failed + + Example: + ```python + profiler = BrowserProfiler() + cdp_url = await profiler.launch_standalone_browser( + user_data_dir="/path/to/profile", + debugging_port=9222 + ) + # Use cdp_url to connect to the browser + ``` + """ + # Use the provided directory if specified, otherwise create a temporary directory + if user_data_dir: + # Directory is provided directly, ensure it exists + profile_path = user_data_dir + os.makedirs(profile_path, exist_ok=True) + else: + # Create a temporary profile directory + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + profile_name = f"temp_{timestamp}_{uuid.uuid4().hex[:6]}" + profile_path = os.path.join(self.profiles_dir, profile_name) + os.makedirs(profile_path, exist_ok=True) + + # Print initial information + border = f"{'='*80}" + self.logger.info("{border}", tag="CDP", params={"border": border}, colors={"border": LogColor.CYAN}) + self.logger.info("Launching standalone browser with CDP debugging", tag="CDP") + self.logger.info("Browser type: {browser_type}", tag="CDP", params={"browser_type": browser_type}, colors={"browser_type": LogColor.CYAN}) + self.logger.info("Profile path: {profile_path}", tag="CDP", params={"profile_path": profile_path}, colors={"profile_path": LogColor.YELLOW}) + self.logger.info(f"Debugging port: {debugging_port}", tag="CDP") + self.logger.info(f"Headless mode: {headless}", tag="CDP") + + # create browser config + browser_config = BrowserConfig( + browser_type=browser_type, + headless=headless, + user_data_dir=profile_path, + debugging_port=debugging_port, + verbose=True + ) + + # Create managed browser instance + managed_browser = ManagedBrowser( + browser_config=browser_config, + user_data_dir=profile_path, + headless=headless, + logger=self.logger, + debugging_port=debugging_port + ) + + # Set up signal handlers to ensure cleanup on interrupt + original_sigint = signal.getsignal(signal.SIGINT) + original_sigterm = signal.getsignal(signal.SIGTERM) + + # Define cleanup handler for signals + async def cleanup_handler(sig, frame): + self.logger.warning("\nCleaning up browser process...", tag="CDP") + await managed_browser.cleanup() + # Restore original signal handlers + signal.signal(signal.SIGINT, original_sigint) + signal.signal(signal.SIGTERM, original_sigterm) + if sig == signal.SIGINT: + self.logger.error("Browser terminated by user.", tag="CDP") + sys.exit(1) + + # Set signal handlers + def sigint_handler(sig, frame): + asyncio.create_task(cleanup_handler(sig, frame)) + + signal.signal(signal.SIGINT, sigint_handler) + signal.signal(signal.SIGTERM, sigint_handler) + + # Event to signal when user wants to exit + user_done_event = asyncio.Event() + + # Run keyboard input loop in a separate task + async def listen_for_quit_command(): + """Cross-platform keyboard listener that waits for 'q' key press.""" + # First output the prompt + self.logger.info( + "Press {segment} to stop the browser and exit...", + tag="CDP", + params={"segment": "'q'"}, colors={"segment": LogColor.YELLOW}, + base_color=LogColor.CYAN + ) + + async def check_browser_process(): + """Check if browser process is still running.""" + if managed_browser.browser_process and managed_browser.browser_process.poll() is not None: + self.logger.info("Browser already closed. Ending input listener.", tag="CDP") + user_done_event.set() + return True + return False + + # Try platform-specific implementations with fallback + try: + if self._is_windows(): + await self._listen_windows(user_done_event, check_browser_process, "CDP") + else: + await self._listen_unix(user_done_event, check_browser_process, "CDP") + except Exception as e: + self.logger.warning(f"Platform-specific keyboard listener failed: {e}", tag="CDP") + self.logger.info("Falling back to simple input mode...", tag="CDP") + await self._listen_fallback(user_done_event, check_browser_process, "CDP") + + # Function to retrieve and display CDP JSON config + async def get_cdp_json(port): + import aiohttp + cdp_url = f"http://localhost:{port}" + json_url = f"{cdp_url}/json/version" + + try: + async with aiohttp.ClientSession() as session: + # Try multiple times in case the browser is still starting up + for _ in range(10): + try: + async with session.get(json_url) as response: + if response.status == 200: + data = await response.json() + return cdp_url, data + except Exception: + pass + + await asyncio.sleep(0.5) + + return cdp_url, None + except Exception as e: + self.logger.error(f"Error fetching CDP JSON: {str(e)}", tag="CDP") + return cdp_url, None + + cdp_url = None + config_json = None + + try: + # Start the browser + await managed_browser.start() + + # Check if browser started successfully + browser_process = managed_browser.browser_process + if not browser_process: + self.logger.error("Failed to start browser process.", tag="CDP") + return None + + self.logger.info("Browser launched successfully. Retrieving CDP information...", tag="CDP") + + # Get CDP URL and JSON config + cdp_url, config_json = await get_cdp_json(debugging_port) + + if cdp_url: + self.logger.success(f"CDP URL: {cdp_url}", tag="CDP") + + if config_json: + # Display relevant CDP information + self.logger.info(f"Browser: {config_json.get('Browser', 'Unknown')}", tag="CDP", colors={"Browser": LogColor.CYAN}) + self.logger.info(f"Protocol Version: {config_json.get('Protocol-Version', 'Unknown')}", tag="CDP", colors={"Protocol-Version": LogColor.CYAN}) + if 'webSocketDebuggerUrl' in config_json: + self.logger.info("WebSocket URL: {webSocketDebuggerUrl}", tag="CDP", params={"webSocketDebuggerUrl": config_json['webSocketDebuggerUrl']}, colors={"webSocketDebuggerUrl": LogColor.GREEN}) + else: + self.logger.warning("Could not retrieve CDP configuration JSON", tag="CDP") + else: + self.logger.error(f"Failed to get CDP URL on port {debugging_port}", tag="CDP") + await managed_browser.cleanup() + return None + + # Start listening for keyboard input + listener_task = asyncio.create_task(listen_for_quit_command()) + + # Wait for the user to press 'q' or for the browser process to exit naturally + while not user_done_event.is_set() and browser_process.poll() is None: + await asyncio.sleep(0.5) + + # Cancel the listener task if it's still running + if not listener_task.done(): + listener_task.cancel() + try: + await listener_task + except asyncio.CancelledError: + pass + + # If the browser is still running and the user pressed 'q', terminate it + if browser_process.poll() is None and user_done_event.is_set(): + self.logger.info("Terminating browser process...", tag="CDP") + await managed_browser.cleanup() + + self.logger.success("Browser closed.", tag="CDP") + + except Exception as e: + self.logger.error(f"Error launching standalone browser: {str(e)}", tag="CDP") + await managed_browser.cleanup() + return None + finally: + # Restore original signal handlers + signal.signal(signal.SIGINT, original_sigint) + signal.signal(signal.SIGTERM, original_sigterm) + + # Make sure browser is fully cleaned up + await managed_browser.cleanup() + + # Return the CDP URL + return cdp_url + + async def launch_builtin_browser(self, + browser_type: str = "chromium", + debugging_port: int = 9222, + headless: bool = True) -> Optional[str]: + """ + Launch a browser in the background for use as the builtin browser. + + Args: + browser_type (str): Type of browser to launch ('chromium' or 'firefox') + debugging_port (int): Port to use for CDP debugging + headless (bool): Whether to run in headless mode + + Returns: + str: CDP URL for the browser, or None if launch failed + """ + # Check if there's an existing browser still running + browser_info = self.get_builtin_browser_info() + if browser_info and self._is_browser_running(browser_info.get('pid')): + self.logger.info("Builtin browser is already running", tag="BUILTIN") + return browser_info.get('cdp_url') + + # Create a user data directory for the builtin browser + user_data_dir = os.path.join(self.builtin_browser_dir, "user_data") + os.makedirs(user_data_dir, exist_ok=True) + + # Create managed browser instance + managed_browser = ManagedBrowser( + browser_type=browser_type, + user_data_dir=user_data_dir, + headless=headless, + logger=self.logger, + debugging_port=debugging_port + ) + + try: + # Start the browser + await managed_browser.start() + + # Check if browser started successfully + browser_process = managed_browser.browser_process + if not browser_process: + self.logger.error("Failed to start browser process.", tag="BUILTIN") + return None + + # Get CDP URL + cdp_url = f"http://localhost:{debugging_port}" + + # Try to verify browser is responsive by fetching version info + import aiohttp + json_url = f"{cdp_url}/json/version" + config_json = None + + try: + async with aiohttp.ClientSession() as session: + for _ in range(10): # Try multiple times + try: + async with session.get(json_url) as response: + if response.status == 200: + config_json = await response.json() + break + except Exception: + pass + await asyncio.sleep(0.5) + except Exception as e: + self.logger.warning(f"Could not verify browser: {str(e)}", tag="BUILTIN") + + # Save browser info + browser_info = { + 'pid': browser_process.pid, + 'cdp_url': cdp_url, + 'user_data_dir': user_data_dir, + 'browser_type': browser_type, + 'debugging_port': debugging_port, + 'start_time': time.time(), + 'config': config_json + } + + with open(self.builtin_config_file, 'w') as f: + json.dump(browser_info, f, indent=2) + + # Detach from the browser process - don't keep any references + # This is important to allow the Python script to exit while the browser continues running + # We'll just record the PID and other info, and the browser will run independently + managed_browser.browser_process = None + + self.logger.success(f"Builtin browser launched at CDP URL: {cdp_url}", tag="BUILTIN") + return cdp_url + + except Exception as e: + self.logger.error(f"Error launching builtin browser: {str(e)}", tag="BUILTIN") + if managed_browser: + await managed_browser.cleanup() + return None + + def get_builtin_browser_info(self) -> Optional[Dict[str, Any]]: + """ + Get information about the builtin browser. + + Returns: + dict: Browser information or None if no builtin browser is configured + """ + if not os.path.exists(self.builtin_config_file): + return None + + try: + with open(self.builtin_config_file, 'r') as f: + browser_info = json.load(f) + + # Check if the browser is still running + if not self._is_browser_running(browser_info.get('pid')): + self.logger.warning("Builtin browser is not running", tag="BUILTIN") + return None + + return browser_info + except Exception as e: + self.logger.error(f"Error reading builtin browser config: {str(e)}", tag="BUILTIN") + return None + + def _is_browser_running(self, pid: Optional[int]) -> bool: + """Check if a process with the given PID is running""" + if not pid: + return False + + try: + # Check if the process exists + if sys.platform == "win32": + process = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], + capture_output=True, text=True) + return str(pid) in process.stdout + else: + # Unix-like systems + os.kill(pid, 0) # This doesn't actually kill the process, just checks if it exists + return True + except (ProcessLookupError, PermissionError, OSError): + return False + + async def kill_builtin_browser(self) -> bool: + """ + Kill the builtin browser if it's running. + + Returns: + bool: True if the browser was killed, False otherwise + """ + browser_info = self.get_builtin_browser_info() + if not browser_info: + self.logger.warning("No builtin browser found", tag="BUILTIN") + return False + + pid = browser_info.get('pid') + if not pid: + return False + + try: + if sys.platform == "win32": + subprocess.run(["taskkill", "/F", "/PID", str(pid)], check=True) + else: + os.kill(pid, signal.SIGTERM) + # Wait for termination + for _ in range(5): + if not self._is_browser_running(pid): + break + await asyncio.sleep(0.5) + else: + # Force kill if still running + os.kill(pid, signal.SIGKILL) + + # Remove config file + if os.path.exists(self.builtin_config_file): + os.unlink(self.builtin_config_file) + + self.logger.success("Builtin browser terminated", tag="BUILTIN") + return True + except Exception as e: + self.logger.error(f"Error killing builtin browser: {str(e)}", tag="BUILTIN") + return False + + async def get_builtin_browser_status(self) -> Dict[str, Any]: + """ + Get status information about the builtin browser. + + Returns: + dict: Status information with running, cdp_url, and info fields + """ + browser_info = self.get_builtin_browser_info() + + if not browser_info: + return { + 'running': False, + 'cdp_url': None, + 'info': None + } + + return { + 'running': True, + 'cdp_url': browser_info.get('cdp_url'), + 'info': browser_info + } + + +if __name__ == "__main__": + # Example usage + profiler = BrowserProfiler() + + # Create a new profile + import os + from pathlib import Path + home_dir = Path.home() + profile_path = asyncio.run(profiler.create_profile( str(home_dir / ".crawl4ai/profiles/test-profile"))) + + + + # Launch a standalone browser + asyncio.run(profiler.launch_standalone_browser()) + + # List profiles + profiles = profiler.list_profiles() + for profile in profiles: + print(f"Profile: {profile['name']}, Path: {profile['path']}") + + # Delete a profile + success = profiler.delete_profile("my-profile") + if success: + print("Profile deleted successfully") + else: + print("Failed to delete profile") \ No newline at end of file diff --git a/crawl4ai/cache_context.py b/crawl4ai/cache_context.py new file mode 100644 index 0000000..75914b5 --- /dev/null +++ b/crawl4ai/cache_context.py @@ -0,0 +1,117 @@ +from enum import Enum + + +class CacheMode(Enum): + """ + Defines the caching behavior for web crawling operations. + + Modes: + - ENABLED: Normal caching behavior (read and write) + - DISABLED: No caching at all + - READ_ONLY: Only read from cache, don't write + - WRITE_ONLY: Only write to cache, don't read + - BYPASS: Bypass cache for this operation + """ + + ENABLED = "enabled" + DISABLED = "disabled" + READ_ONLY = "read_only" + WRITE_ONLY = "write_only" + BYPASS = "bypass" + + +class CacheContext: + """ + Encapsulates cache-related decisions and URL handling. + + This class centralizes all cache-related logic and URL type checking, + making the caching behavior more predictable and maintainable. + + Attributes: + url (str): The URL being processed. + cache_mode (CacheMode): The cache mode for the current operation. + always_bypass (bool): If True, bypasses caching for this operation. + is_cacheable (bool): True if the URL is cacheable, False otherwise. + is_web_url (bool): True if the URL is a web URL, False otherwise. + is_local_file (bool): True if the URL is a local file, False otherwise. + is_raw_html (bool): True if the URL is raw HTML, False otherwise. + _url_display (str): The display name for the URL (web, local file, or raw HTML). + """ + + def __init__(self, url: str, cache_mode: CacheMode, always_bypass: bool = False): + """ + Initializes the CacheContext with the provided URL and cache mode. + + Args: + url (str): The URL being processed. + cache_mode (CacheMode): The cache mode for the current operation. + always_bypass (bool): If True, bypasses caching for this operation. + """ + self.url = url + self.cache_mode = cache_mode + self.always_bypass = always_bypass + self.is_cacheable = url.startswith(("http://", "https://", "file://")) + self.is_web_url = url.startswith(("http://", "https://")) + self.is_local_file = url.startswith("file://") + self.is_raw_html = url.startswith("raw:") + self._url_display = url if not self.is_raw_html else "Raw HTML" + + def should_read(self) -> bool: + """ + Determines if cache should be read based on context. + + How it works: + 1. If always_bypass is True or is_cacheable is False, return False. + 2. If cache_mode is ENABLED or READ_ONLY, return True. + + Returns: + bool: True if cache should be read, False otherwise. + """ + if self.always_bypass or not self.is_cacheable: + return False + return self.cache_mode in [CacheMode.ENABLED, CacheMode.READ_ONLY] + + def should_write(self) -> bool: + """ + Determines if cache should be written based on context. + + How it works: + 1. If always_bypass is True or is_cacheable is False, return False. + 2. If cache_mode is ENABLED or WRITE_ONLY, return True. + + Returns: + bool: True if cache should be written, False otherwise. + """ + if self.always_bypass or not self.is_cacheable: + return False + return self.cache_mode in [CacheMode.ENABLED, CacheMode.WRITE_ONLY] + + @property + def display_url(self) -> str: + """Returns the URL in display format.""" + return self._url_display + + +def _legacy_to_cache_mode( + disable_cache: bool = False, + bypass_cache: bool = False, + no_cache_read: bool = False, + no_cache_write: bool = False, +) -> CacheMode: + """ + Converts legacy cache parameters to the new CacheMode enum. + + This is an internal function to help transition from the old boolean flags + to the new CacheMode system. + """ + if disable_cache: + return CacheMode.DISABLED + if bypass_cache: + return CacheMode.BYPASS + if no_cache_read and no_cache_write: + return CacheMode.DISABLED + if no_cache_read: + return CacheMode.WRITE_ONLY + if no_cache_write: + return CacheMode.READ_ONLY + return CacheMode.ENABLED diff --git a/crawl4ai/cache_validator.py b/crawl4ai/cache_validator.py new file mode 100644 index 0000000..b4b170c --- /dev/null +++ b/crawl4ai/cache_validator.py @@ -0,0 +1,270 @@ +""" +Cache validation using HTTP conditional requests and head fingerprinting. + +Uses httpx for fast, lightweight HTTP requests (no browser needed). +This module enables smart cache validation to avoid unnecessary full browser crawls +when content hasn't changed. + +Validation Strategy: +1. Send HEAD request with If-None-Match / If-Modified-Since headers +2. If server returns 304 Not Modified → cache is FRESH +3. If server returns 200 → fetch and compare fingerprint +4. If fingerprint matches → cache is FRESH (minor changes only) +5. Otherwise → cache is STALE, need full recrawl +""" + +import httpx +from dataclasses import dataclass +from typing import Optional, Tuple +from enum import Enum + +from .utils import compute_head_fingerprint + + +class CacheValidationResult(Enum): + """Result of cache validation check.""" + FRESH = "fresh" # Content unchanged, use cache + STALE = "stale" # Content changed, need recrawl + UNKNOWN = "unknown" # Couldn't determine, need recrawl + ERROR = "error" # Request failed, use cache as fallback + + +@dataclass +class ValidationResult: + """Detailed result of a cache validation attempt.""" + status: CacheValidationResult + new_etag: Optional[str] = None + new_last_modified: Optional[str] = None + new_head_fingerprint: Optional[str] = None + reason: str = "" + + +class CacheValidator: + """ + Validates cache freshness using lightweight HTTP requests. + + This validator uses httpx to make fast HTTP requests without needing + a full browser. It supports two validation methods: + + 1. HTTP Conditional Requests (Layer 3): + - Uses If-None-Match with stored ETag + - Uses If-Modified-Since with stored Last-Modified + - Server returns 304 if content unchanged + + 2. Head Fingerprinting (Layer 4): + - Fetches only the section (~5KB) + - Compares fingerprint of key meta tags + - Catches changes even without server support for conditional requests + """ + + def __init__(self, timeout: float = 10.0, user_agent: Optional[str] = None): + """ + Initialize the cache validator. + + Args: + timeout: Request timeout in seconds + user_agent: Custom User-Agent string (optional) + """ + self.timeout = timeout + self.user_agent = user_agent or "Mozilla/5.0 (compatible; Crawl4AI/1.0)" + self._client: Optional[httpx.AsyncClient] = None + + async def _get_client(self) -> httpx.AsyncClient: + """Get or create the httpx client.""" + if self._client is None: + self._client = httpx.AsyncClient( + http2=True, + timeout=self.timeout, + follow_redirects=True, + headers={"User-Agent": self.user_agent} + ) + return self._client + + async def validate( + self, + url: str, + stored_etag: Optional[str] = None, + stored_last_modified: Optional[str] = None, + stored_head_fingerprint: Optional[str] = None, + ) -> ValidationResult: + """ + Validate if cached content is still fresh. + + Args: + url: The URL to validate + stored_etag: Previously stored ETag header value + stored_last_modified: Previously stored Last-Modified header value + stored_head_fingerprint: Previously computed head fingerprint + + Returns: + ValidationResult with status and any updated metadata + """ + client = await self._get_client() + + # Build conditional request headers + headers = {} + if stored_etag: + headers["If-None-Match"] = stored_etag + if stored_last_modified: + headers["If-Modified-Since"] = stored_last_modified + + try: + # Step 1: Try HEAD request with conditional headers + if headers: + response = await client.head(url, headers=headers) + + if response.status_code == 304: + return ValidationResult( + status=CacheValidationResult.FRESH, + reason="Server returned 304 Not Modified" + ) + + # Got 200, extract new headers for potential update + new_etag = response.headers.get("etag") + new_last_modified = response.headers.get("last-modified") + + # If we have fingerprint, compare it + if stored_head_fingerprint: + head_html, _, _ = await self._fetch_head(url) + if head_html: + new_fingerprint = compute_head_fingerprint(head_html) + if new_fingerprint and new_fingerprint == stored_head_fingerprint: + return ValidationResult( + status=CacheValidationResult.FRESH, + new_etag=new_etag, + new_last_modified=new_last_modified, + new_head_fingerprint=new_fingerprint, + reason="Head fingerprint matches" + ) + elif new_fingerprint: + return ValidationResult( + status=CacheValidationResult.STALE, + new_etag=new_etag, + new_last_modified=new_last_modified, + new_head_fingerprint=new_fingerprint, + reason="Head fingerprint changed" + ) + + # Headers changed and no fingerprint match + return ValidationResult( + status=CacheValidationResult.STALE, + new_etag=new_etag, + new_last_modified=new_last_modified, + reason="Server returned 200, content may have changed" + ) + + # Step 2: No conditional headers available, try fingerprint only + if stored_head_fingerprint: + head_html, new_etag, new_last_modified = await self._fetch_head(url) + + if head_html: + new_fingerprint = compute_head_fingerprint(head_html) + + if new_fingerprint and new_fingerprint == stored_head_fingerprint: + return ValidationResult( + status=CacheValidationResult.FRESH, + new_etag=new_etag, + new_last_modified=new_last_modified, + new_head_fingerprint=new_fingerprint, + reason="Head fingerprint matches" + ) + elif new_fingerprint: + return ValidationResult( + status=CacheValidationResult.STALE, + new_etag=new_etag, + new_last_modified=new_last_modified, + new_head_fingerprint=new_fingerprint, + reason="Head fingerprint changed" + ) + + # Step 3: No validation data available + return ValidationResult( + status=CacheValidationResult.UNKNOWN, + reason="No validation data available (no etag, last-modified, or fingerprint)" + ) + + except httpx.TimeoutException: + return ValidationResult( + status=CacheValidationResult.ERROR, + reason="Validation request timed out" + ) + except httpx.RequestError as e: + return ValidationResult( + status=CacheValidationResult.ERROR, + reason=f"Validation request failed: {type(e).__name__}" + ) + except Exception as e: + # On unexpected error, prefer using cache over failing + return ValidationResult( + status=CacheValidationResult.ERROR, + reason=f"Validation error: {str(e)}" + ) + + async def _fetch_head(self, url: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: + """ + Fetch only the section of a page. + + Uses streaming to stop reading after is found, + minimizing bandwidth usage. + + Args: + url: The URL to fetch + + Returns: + Tuple of (head_html, etag, last_modified) + """ + client = await self._get_client() + + try: + async with client.stream( + "GET", + url, + headers={"Accept-Encoding": "identity"} # Disable compression for easier parsing + ) as response: + etag = response.headers.get("etag") + last_modified = response.headers.get("last-modified") + + if response.status_code != 200: + return None, etag, last_modified + + # Read until or max 64KB + chunks = [] + total_bytes = 0 + max_bytes = 65536 + + async for chunk in response.aiter_bytes(4096): + chunks.append(chunk) + total_bytes += len(chunk) + + content = b''.join(chunks) + # Check for (case insensitive) + if b'' in content.lower() or b'' in content: + break + if total_bytes >= max_bytes: + break + + html = content.decode('utf-8', errors='replace') + + # Extract just the head section + head_end = html.lower().find('') + if head_end != -1: + html = html[:head_end + 7] + + return html, etag, last_modified + + except Exception: + return None, None, None + + async def close(self): + """Close the HTTP client and release resources.""" + if self._client: + await self._client.aclose() + self._client = None + + async def __aenter__(self): + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() diff --git a/crawl4ai/chunking_strategy.py b/crawl4ai/chunking_strategy.py new file mode 100644 index 0000000..a0bfe1b --- /dev/null +++ b/crawl4ai/chunking_strategy.py @@ -0,0 +1,255 @@ +from abc import ABC, abstractmethod +import re +from collections import Counter +import string +from .model_loader import load_nltk_punkt + +# Define the abstract base class for chunking strategies +class ChunkingStrategy(ABC): + """ + Abstract base class for chunking strategies. + """ + + @abstractmethod + def chunk(self, text: str) -> list: + """ + Abstract method to chunk the given text. + + Args: + text (str): The text to chunk. + + Returns: + list: A list of chunks. + """ + pass + + +# Create an identity chunking strategy f(x) = [x] +class IdentityChunking(ChunkingStrategy): + """ + Chunking strategy that returns the input text as a single chunk. + """ + + def chunk(self, text: str) -> list: + return [text] + + +# Regex-based chunking +class RegexChunking(ChunkingStrategy): + """ + Chunking strategy that splits text based on regular expression patterns. + """ + + def __init__(self, patterns=None, **kwargs): + """ + Initialize the RegexChunking object. + + Args: + patterns (list): A list of regular expression patterns to split text. + """ + if patterns is None: + patterns = [r"\n\n"] # Default split pattern + self.patterns = patterns + + def chunk(self, text: str) -> list: + paragraphs = [text] + for pattern in self.patterns: + new_paragraphs = [] + for paragraph in paragraphs: + new_paragraphs.extend(re.split(pattern, paragraph)) + paragraphs = new_paragraphs + return paragraphs + + +# NLP-based sentence chunking +class NlpSentenceChunking(ChunkingStrategy): + """ + Chunking strategy that splits text into sentences using NLTK's sentence tokenizer. + """ + + def __init__(self, **kwargs): + """ + Initialize the NlpSentenceChunking object. + """ + load_nltk_punkt() + + def chunk(self, text: str) -> list: + # Improved regex for sentence splitting + # sentence_endings = re.compile( + # r'(? list: + # Use the TextTilingTokenizer to segment the text + segmented_topics = self.tokenizer.tokenize(text) + return segmented_topics + + def extract_keywords(self, text: str) -> list: + # Tokenize and remove stopwords and punctuation + import nltk as nl + + tokens = nl.toknize.word_tokenize(text) + tokens = [ + token.lower() + for token in tokens + if token not in nl.corpus.stopwords.words("english") + and token not in string.punctuation + ] + + # Calculate frequency distribution + freq_dist = Counter(tokens) + keywords = [word for word, freq in freq_dist.most_common(self.num_keywords)] + return keywords + + def chunk_with_topics(self, text: str) -> list: + # Segment the text into topics + segments = self.chunk(text) + # Extract keywords for each topic segment + segments_with_topics = [ + (segment, self.extract_keywords(segment)) for segment in segments + ] + return segments_with_topics + + +# Fixed-length word chunks +class FixedLengthWordChunking(ChunkingStrategy): + """ + Chunking strategy that splits text into fixed-length word chunks. + + How it works: + 1. Split the text into words + 2. Create chunks of fixed length + 3. Return the list of chunks + """ + + def __init__(self, chunk_size=100, **kwargs): + """ + Initialize the fixed-length word chunking strategy with the given chunk size. + + Args: + chunk_size (int): The size of each chunk in words. + """ + self.chunk_size = chunk_size + + def chunk(self, text: str) -> list: + words = text.split() + return [ + " ".join(words[i : i + self.chunk_size]) + for i in range(0, len(words), self.chunk_size) + ] + + +# Sliding window chunking +class SlidingWindowChunking(ChunkingStrategy): + """ + Chunking strategy that splits text into overlapping word chunks. + + How it works: + 1. Split the text into words + 2. Create chunks of fixed length + 3. Return the list of chunks + """ + + def __init__(self, window_size=100, step=50, **kwargs): + """ + Initialize the sliding window chunking strategy with the given window size and + step size. + + Args: + window_size (int): The size of the sliding window in words. + step (int): The step size for sliding the window in words. + """ + self.window_size = window_size + self.step = step + + def chunk(self, text: str) -> list: + words = text.split() + chunks = [] + + if len(words) <= self.window_size: + return [text] + + for i in range(0, len(words) - self.window_size + 1, self.step): + chunk = " ".join(words[i : i + self.window_size]) + chunks.append(chunk) + + # Handle the last chunk if it doesn't align perfectly + if i + self.window_size < len(words): + chunks.append(" ".join(words[-self.window_size :])) + + return chunks + + +class OverlappingWindowChunking(ChunkingStrategy): + """ + Chunking strategy that splits text into overlapping word chunks. + + How it works: + 1. Split the text into words using whitespace + 2. Create chunks of fixed length equal to the window size + 3. Slide the window by the overlap size + 4. Return the list of chunks + """ + + def __init__(self, window_size=1000, overlap=100, **kwargs): + """ + Initialize the overlapping window chunking strategy with the given window size and + overlap size. + + Args: + window_size (int): The size of the window in words. + overlap (int): The size of the overlap between consecutive chunks in words. + """ + self.window_size = window_size + self.overlap = overlap + + def chunk(self, text: str) -> list: + words = text.split() + chunks = [] + + if len(words) <= self.window_size: + return [text] + + start = 0 + while start < len(words): + end = start + self.window_size + chunk = " ".join(words[start:end]) + chunks.append(chunk) + + if end >= len(words): + break + + start = end - self.overlap + + return chunks diff --git a/crawl4ai/cli.py b/crawl4ai/cli.py new file mode 100644 index 0000000..02b6715 --- /dev/null +++ b/crawl4ai/cli.py @@ -0,0 +1,1654 @@ +import click +import os +import sys +import time + +import humanize +from typing import Dict, Any, Optional, List +import json +import yaml +import anyio +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich.prompt import Prompt, Confirm + +from crawl4ai import ( + CacheMode, + AsyncWebCrawler, + CrawlResult, + BrowserConfig, + CrawlerRunConfig, + LLMExtractionStrategy, + LXMLWebScrapingStrategy, + JsonCssExtractionStrategy, + JsonXPathExtractionStrategy, + BM25ContentFilter, + PruningContentFilter, + BrowserProfiler, + DefaultMarkdownGenerator, + LLMConfig, + BFSDeepCrawlStrategy, + DFSDeepCrawlStrategy, + BestFirstCrawlingStrategy, +) +from crawl4ai.browser_profiler import ShrinkLevel, _format_size +from crawl4ai.config import USER_SETTINGS +from crawl4ai.cloud import cloud_cmd +from litellm import completion +from pathlib import Path + + +# Initialize rich console +console = Console() + +def get_global_config() -> dict: + config_dir = Path.home() / ".crawl4ai" + config_file = config_dir / "global.yml" + + if not config_file.exists(): + config_dir.mkdir(parents=True, exist_ok=True) + return {} + + with open(config_file) as f: + return yaml.safe_load(f) or {} + +def save_global_config(config: dict): + config_file = Path.home() / ".crawl4ai" / "global.yml" + with open(config_file, "w", encoding="utf-8") as f: + yaml.dump(config, f) + +def setup_llm_config() -> tuple[str, str]: + config = get_global_config() + provider = config.get("DEFAULT_LLM_PROVIDER") + token = config.get("DEFAULT_LLM_PROVIDER_TOKEN") + + if not provider: + click.echo("\nNo default LLM provider configured.") + click.echo("Provider format: 'company/model' (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')") + click.echo("See available providers at: https://docs.litellm.ai/docs/providers") + provider = click.prompt("Enter provider") + + if not provider.startswith("ollama/"): + if not token: + token = click.prompt("Enter API token for " + provider, hide_input=True) + else: + token = "no-token" + + if not config.get("DEFAULT_LLM_PROVIDER") or not config.get("DEFAULT_LLM_PROVIDER_TOKEN"): + config["DEFAULT_LLM_PROVIDER"] = provider + config["DEFAULT_LLM_PROVIDER_TOKEN"] = token + save_global_config(config) + click.echo("\nConfiguration saved to ~/.crawl4ai/global.yml") + + return provider, token + +async def stream_llm_response(url: str, markdown: str, query: str, provider: str, token: str): + response = completion( + model=provider, + api_key=token, + messages=[ + { + "content": f"You are Crawl4ai assistant, answering user question based on the provided context which is crawled from {url}.", + "role": "system" + }, + { + "content": f"<|start of context|>\n{markdown}\n<|end of context|>\n\n{query}", + "role": "user" + }, + ], + stream=True, + ) + + for chunk in response: + if content := chunk["choices"][0]["delta"].get("content"): + print(content, end="", flush=True) + print() # New line at end + + + +def parse_key_values(ctx, param, value) -> Dict[str, Any]: + if not value: + return {} + result = {} + pairs = value.split(',') + for pair in pairs: + try: + k, v = pair.split('=', 1) + # Handle common value types + if v.lower() == 'true': v = True + elif v.lower() == 'false': v = False + elif v.isdigit(): v = int(v) + elif v.replace('.','',1).isdigit(): v = float(v) + elif v.startswith('[') and v.endswith(']'): + v = [x.strip() for x in v[1:-1].split(',') if x.strip()] + elif v.startswith('{') and v.endswith('}'): + try: + v = json.loads(v) + except json.JSONDecodeError: + raise click.BadParameter(f'Invalid JSON object: {v}') + result[k.strip()] = v + except ValueError: + raise click.BadParameter(f'Invalid key=value pair: {pair}') + return result + +def load_config_file(path: Optional[str]) -> dict: + if not path: + return {} + + try: + with open(path) as f: + if path.endswith((".yaml", ".yml")): + return yaml.safe_load(f) + return json.load(f) + except Exception as e: + raise click.BadParameter(f'Error loading config file {path}: {str(e)}') + +def load_schema_file(path: Optional[str]) -> dict: + if not path: + return None + return load_config_file(path) + +async def run_crawler(url: str, browser_cfg: BrowserConfig, crawler_cfg: CrawlerRunConfig, verbose: bool): + if verbose: + click.echo("Starting crawler with configurations:") + click.echo(f"Browser config: {browser_cfg.dump()}") + click.echo(f"Crawler config: {crawler_cfg.dump()}") + + async with AsyncWebCrawler(config=browser_cfg) as crawler: + try: + result = await crawler.arun(url=url, config=crawler_cfg) + return result + except Exception as e: + raise click.ClickException(f"Crawling failed: {str(e)}") + +def show_examples(): + examples = """ +🚀 Crawl4AI CLI Examples + +1️⃣ Basic Usage: + # Simple crawl with default settings + crwl https://example.com + + # Get markdown output + crwl https://example.com -o markdown + + # Verbose JSON output with cache bypass + crwl https://example.com -o json -v --bypass-cache + +2️⃣ Using Config Files: + # Using browser and crawler configs + crwl https://example.com -B browser.yml -C crawler.yml + + # CSS-based extraction + crwl https://example.com -e extract_css.yml -s css_schema.json -o json + + # LLM-based extraction with config file + crwl https://example.com -e extract_llm.yml -s llm_schema.json -o json + + # Quick LLM-based JSON extraction (prompts for LLM provider first time) + crwl https://example.com -j # Auto-extracts structured data + crwl https://example.com -j "Extract product details including name, price, and features" # With specific instructions + +3️⃣ Direct Parameters: + # Browser settings + crwl https://example.com -b "headless=true,viewport_width=1280,user_agent_mode=random" + + # Crawler settings + crwl https://example.com -c "css_selector=#main,delay_before_return_html=2,scan_full_page=true" + +4️⃣ Profile Management for Identity-Based Crawling: + # Launch interactive profile manager + crwl profiles + + # Create, list, and delete browser profiles for identity-based crawling + # Use a profile for crawling (keeps you logged in) + crwl https://example.com -p my-profile-name + + # Example: Crawl a site that requires login + # 1. First create a profile and log in: + crwl profiles + # 2. Then use that profile to crawl the authenticated site: + crwl https://site-requiring-login.com/dashboard -p my-profile-name + +5️⃣ CDP Mode for Browser Automation: + # Launch browser with CDP debugging on default port 9222 + crwl cdp + + # Use a specific profile and custom port + crwl cdp -p my-profile -P 9223 + + # Launch headless browser with CDP enabled + crwl cdp --headless + + # Launch in incognito mode (ignores profile) + crwl cdp --incognito + + # Use the CDP URL with other tools (Puppeteer, Playwright, etc.) + # The URL will be displayed in the terminal when the browser starts + + +6️⃣ Sample Config Files: + +browser.yml: + headless: true + viewport_width: 1280 + user_agent_mode: "random" + verbose: true + ignore_https_errors: true + +extract_css.yml: + type: "json-css" + params: + verbose: true + +css_schema.json: + { + "name": "ArticleExtractor", + "baseSelector": ".article", + "fields": [ + { + "name": "title", + "selector": "h1.title", + "type": "text" + }, + { + "name": "link", + "selector": "a.read-more", + "type": "attribute", + "attribute": "href" + } + ] + } + +extract_llm.yml: + type: "llm" + provider: "openai/gpt-4" + instruction: "Extract all articles with their titles and links" + api_token: "your-token" + params: + temperature: 0.3 + max_tokens: 1000 + +llm_schema.json: + { + "title": "Article", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "The title of the article" + }, + "link": { + "type": "string", + "description": "URL to the full article" + } + } + } + +7️⃣ Advanced Usage: + # Combine configs with direct parameters + crwl https://example.com -B browser.yml -b "headless=false,viewport_width=1920" + + # Full extraction pipeline with config files + crwl https://example.com \\ + -B browser.yml \\ + -C crawler.yml \\ + -e extract_llm.yml \\ + -s llm_schema.json \\ + -o json \\ + -v + + # Quick LLM-based extraction with specific instructions + crwl https://amazon.com/dp/B01DFKC2SO \\ + -j "Extract product title, current price, original price, rating, and all product specifications" \\ + -b "headless=true,viewport_width=1280" \\ + -v + + # Content filtering with BM25 + crwl https://example.com \\ + -f filter_bm25.yml \\ + -o markdown-fit + + # Authenticated crawling with profile + crwl https://login-required-site.com \\ + -p my-authenticated-profile \\ + -c "css_selector=.dashboard-content" \\ + -o markdown + +For more documentation visit: https://github.com/unclecode/crawl4ai + +8️⃣ Q&A with LLM: + # Ask a question about the content + crwl https://example.com -q "What is the main topic discussed?" + + # First view content, then ask questions + crwl https://example.com -o markdown # See the crawled content first + crwl https://example.com -q "Summarize the key points" + crwl https://example.com -q "What are the conclusions?" + + # Advanced crawling with Q&A + crwl https://example.com \\ + -B browser.yml \\ + -c "css_selector=article,scan_full_page=true" \\ + -q "What are the pros and cons mentioned?" + + Note: First time using -q will prompt for LLM provider and API token. + These will be saved in ~/.crawl4ai/global.yml for future use. + + Supported provider format: 'company/model' + Examples: + - ollama/llama3.3 + - openai/gpt-4 + - anthropic/claude-3-sonnet + - cohere/command + - google/gemini-pro + + See full list of providers: https://docs.litellm.ai/docs/providers + + # Set default LLM provider and token in advance + crwl config set DEFAULT_LLM_PROVIDER "anthropic/claude-3-sonnet" + crwl config set DEFAULT_LLM_PROVIDER_TOKEN "your-api-token-here" + + # Set default browser behavior + crwl config set BROWSER_HEADLESS false # Always show browser window + crwl config set USER_AGENT_MODE random # Use random user agent + +9️⃣ Profile Management: + # Launch interactive profile manager + crwl profiles + + # Create a profile and use it for crawling + crwl profiles # Create and set up your profile interactively + crwl https://example.com -p my-profile-name # Use profile for crawling + + # Example workflow for authenticated site + # 1. First create a profile and log in to the site: + crwl profiles # Select "Create new profile" option + # 2. Then use that profile to crawl authenticated content: + crwl https://site-requiring-login.com/dashboard -p my-profile-name + +🔄 Builtin Browser Management: + # Start a builtin browser (runs in the background) + crwl browser start + + # Check builtin browser status + crwl browser status + + # Open a visible window to see the browser + crwl browser view --url https://example.com + + # Stop the builtin browser + crwl browser stop + + # Restart with different options + crwl browser restart --browser-type chromium --port 9223 --no-headless + + # Use the builtin browser in your code + # (Just set browser_mode="builtin" in your BrowserConfig) + browser_config = BrowserConfig( + browser_mode="builtin", + headless=True + ) + + # Usage via CLI: + crwl https://example.com -b "browser_mode=builtin" +""" + click.echo(examples) + +def get_directory_size(path: str) -> int: + """Calculate the total size of a directory in bytes""" + total_size = 0 + for dirpath, _, filenames in os.walk(path): + for f in filenames: + fp = os.path.join(dirpath, f) + if not os.path.islink(fp): + total_size += os.path.getsize(fp) + return total_size + +def display_profiles_table(profiles: List[Dict[str, Any]]): + """Display a rich table of browser profiles""" + if not profiles: + console.print(Panel("[yellow]No profiles found. Create one with the 'create' command.[/yellow]", + title="Browser Profiles", border_style="blue")) + return + + table = Table(title="Browser Profiles", show_header=True, header_style="bold cyan", border_style="blue") + table.add_column("#", style="dim", width=4) + table.add_column("Name", style="cyan", no_wrap=True) + table.add_column("Path", style="green") + table.add_column("Created", style="yellow") + table.add_column("Browser", style="magenta") + table.add_column("Size", style="blue", justify="right") + + for i, profile in enumerate(profiles): + # Calculate folder size + size = get_directory_size(profile["path"]) + human_size = humanize.naturalsize(size) + + # Format creation date + created = profile["created"].strftime("%Y-%m-%d %H:%M") + + # Add row to table + table.add_row( + str(i+1), + profile["name"], + profile["path"], + created, + profile["type"].capitalize(), + human_size + ) + + console.print(table) + +async def create_profile_interactive(profiler: BrowserProfiler): + """Interactive profile creation wizard""" + console.print(Panel("[bold cyan]Create Browser Profile[/bold cyan]\n" + "This will open a browser window for you to set up your identity.\n" + "Log in to sites, adjust settings, then press 'q' to save.", + border_style="cyan")) + + profile_name = Prompt.ask("[cyan]Enter profile name[/cyan]", default=f"profile_{int(time.time())}") + + console.print("[cyan]Creating profile...[/cyan]") + console.print("[yellow]A browser window will open. After logging in to sites, press 'q' in this terminal to save.[/yellow]") + + # Create the profile + try: + profile_path = await profiler.create_profile(profile_name) + + if profile_path: + console.print(f"[green]Profile successfully created at:[/green] {profile_path}") + else: + console.print("[red]Failed to create profile.[/red]") + except Exception as e: + console.print(f"[red]Error creating profile: {str(e)}[/red]") + +def delete_profile_interactive(profiler: BrowserProfiler): + """Interactive profile deletion""" + profiles = profiler.list_profiles() + + if not profiles: + console.print("[yellow]No profiles found to delete.[/yellow]") + return + + # Display profiles + display_profiles_table(profiles) + + # Get profile selection + idx = Prompt.ask( + "[red]Enter number of profile to delete[/red]", + console=console, + choices=[str(i+1) for i in range(len(profiles))], + show_choices=False + ) + + try: + idx = int(idx) - 1 + profile = profiles[idx] + + # Confirm deletion + if Confirm.ask(f"[red]Are you sure you want to delete profile '{profile['name']}'?[/red]"): + success = profiler.delete_profile(profile["path"]) + + if success: + console.print(f"[green]Profile '{profile['name']}' deleted successfully.[/green]") + else: + console.print(f"[red]Failed to delete profile '{profile['name']}'.[/red]") + except (ValueError, IndexError): + console.print("[red]Invalid selection.[/red]") + +async def crawl_with_profile_cli(profile_path, url): + """Use a profile to crawl a website via CLI""" + console.print(f"[cyan]Crawling [bold]{url}[/bold] using profile at [bold]{profile_path}[/bold][/cyan]") + + # Create browser config with the profile + browser_cfg = BrowserConfig( + headless=False, # Set to False to see the browser in action + use_managed_browser=True, + user_data_dir=profile_path + ) + + # Default crawler config + crawler_cfg = CrawlerRunConfig() + + # Ask for output format + output_format = Prompt.ask( + "[cyan]Output format[/cyan]", + choices=["all", "json", "markdown", "md", "title"], + default="markdown" + ) + + try: + # Run the crawler + result = await run_crawler(url, browser_cfg, crawler_cfg, True) + + # Get JSON output config + config = get_global_config() + ensure_ascii = config.get("JSON_ENSURE_ASCII", USER_SETTINGS["JSON_ENSURE_ASCII"]["default"]) + + # Handle output + if output_format == "all": + console.print(json.dumps(result.model_dump(), indent=2, ensure_ascii=ensure_ascii)) + elif output_format == "json": + console.print(json.dumps(json.loads(result.extracted_content), indent=2, ensure_ascii=ensure_ascii)) + elif output_format in ["markdown", "md"]: + console.print(result.markdown.raw_markdown) + elif output_format == "title": + console.print(result.metadata.get("title", "No title found")) + + console.print(f"[green]Successfully crawled[/green] {url}") + return result + except Exception as e: + console.print(f"[red]Error crawling:[/red] {str(e)}") + return None + +async def use_profile_to_crawl(): + """Interactive profile selection for crawling""" + profiler = BrowserProfiler() + profiles = profiler.list_profiles() + + if not profiles: + console.print("[yellow]No profiles found. Create one first.[/yellow]") + return + + # Display profiles + display_profiles_table(profiles) + + # Get profile selection + idx = Prompt.ask( + "[cyan]Enter number of profile to use[/cyan]", + console=console, + choices=[str(i+1) for i in range(len(profiles))], + show_choices=False + ) + + try: + idx = int(idx) - 1 + profile = profiles[idx] + + # Get URL + url = Prompt.ask("[cyan]Enter URL to crawl[/cyan]") + if url: + # Crawl with the selected profile + await crawl_with_profile_cli(profile["path"], url) + else: + console.print("[red]No URL provided[/red]") + except (ValueError, IndexError): + console.print("[red]Invalid selection[/red]") + +async def manage_profiles(): + """Interactive profile management menu""" + profiler = BrowserProfiler() + + options = { + "1": "List profiles", + "2": "Create new profile", + "3": "Delete profile", + "4": "Use a profile to crawl a website", + "5": "Exit", + } + + while True: + console.print(Panel("[bold cyan]Browser Profile Manager[/bold cyan]", border_style="cyan")) + + for key, value in options.items(): + color = "green" if key == "1" else "yellow" if key == "2" else "red" if key == "3" else "blue" if key == "4" else "cyan" + console.print(f"[{color}]{key}[/{color}]. {value}") + + choice = Prompt.ask("Enter choice", choices=list(options.keys()), default="1") + + if choice == "1": + # List profiles + profiles = profiler.list_profiles() + display_profiles_table(profiles) + + elif choice == "2": + # Create profile + await create_profile_interactive(profiler) + + elif choice == "3": + # Delete profile + delete_profile_interactive(profiler) + + elif choice == "4": + # Use profile to crawl + await use_profile_to_crawl() + + elif choice == "5": + # Exit + console.print("[cyan]Exiting profile manager.[/cyan]") + break + + # Add a separator between operations + console.print("\n") + + + +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) +def cli(): + """Crawl4AI CLI - Web content extraction and browser profile management tool""" + pass + +# Add cloud command group +cli.add_command(cloud_cmd) + + +@cli.group("browser") +def browser_cmd(): + """Manage browser instances for Crawl4AI + + Commands to manage browser instances for Crawl4AI, including: + - status - Check status of the builtin browser + - start - Start a new builtin browser + - stop - Stop the running builtin browser + - restart - Restart the builtin browser + """ + pass + +@browser_cmd.command("status") +def browser_status_cmd(): + """Show status of the builtin browser""" + profiler = BrowserProfiler() + + try: + status = anyio.run(profiler.get_builtin_browser_status) + + if status["running"]: + info = status["info"] + console.print(Panel( + f"[green]Builtin browser is running[/green]\n\n" + f"CDP URL: [cyan]{info['cdp_url']}[/cyan]\n" + f"Process ID: [yellow]{info['pid']}[/yellow]\n" + f"Browser type: [blue]{info['browser_type']}[/blue]\n" + f"User data directory: [magenta]{info['user_data_dir']}[/magenta]\n" + f"Started: [cyan]{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(info['start_time']))}[/cyan]", + title="Builtin Browser Status", + border_style="green" + )) + else: + console.print(Panel( + "[yellow]Builtin browser is not running[/yellow]\n\n" + "Use 'crwl browser start' to start a builtin browser", + title="Builtin Browser Status", + border_style="yellow" + )) + + except Exception as e: + console.print(f"[red]Error checking browser status: {str(e)}[/red]") + sys.exit(1) + +@browser_cmd.command("start") +@click.option("--browser-type", "-b", type=click.Choice(["chromium", "firefox"]), default="chromium", + help="Browser type (default: chromium)") +@click.option("--port", "-p", type=int, default=9222, help="Debugging port (default: 9222)") +@click.option("--headless/--no-headless", default=True, help="Run browser in headless mode") +def browser_start_cmd(browser_type: str, port: int, headless: bool): + """Start a builtin browser instance + + This will start a persistent browser instance that can be used by Crawl4AI + by setting browser_mode="builtin" in BrowserConfig. + """ + profiler = BrowserProfiler() + + # First check if browser is already running + status = anyio.run(profiler.get_builtin_browser_status) + if status["running"]: + console.print(Panel( + "[yellow]Builtin browser is already running[/yellow]\n\n" + f"CDP URL: [cyan]{status['cdp_url']}[/cyan]\n\n" + "Use 'crwl browser restart' to restart the browser", + title="Builtin Browser Start", + border_style="yellow" + )) + return + + try: + console.print(Panel( + f"[cyan]Starting builtin browser[/cyan]\n\n" + f"Browser type: [green]{browser_type}[/green]\n" + f"Debugging port: [yellow]{port}[/yellow]\n" + f"Headless: [cyan]{'Yes' if headless else 'No'}[/cyan]", + title="Builtin Browser Start", + border_style="cyan" + )) + + cdp_url = anyio.run( + profiler.launch_builtin_browser, + browser_type, + port, + headless + ) + + if cdp_url: + console.print(Panel( + f"[green]Builtin browser started successfully[/green]\n\n" + f"CDP URL: [cyan]{cdp_url}[/cyan]\n\n" + "This browser will be used automatically when setting browser_mode='builtin'", + title="Builtin Browser Start", + border_style="green" + )) + else: + console.print(Panel( + "[red]Failed to start builtin browser[/red]", + title="Builtin Browser Start", + border_style="red" + )) + sys.exit(1) + + except Exception as e: + console.print(f"[red]Error starting builtin browser: {str(e)}[/red]") + sys.exit(1) + +@browser_cmd.command("stop") +def browser_stop_cmd(): + """Stop the running builtin browser""" + profiler = BrowserProfiler() + + try: + # First check if browser is running + status = anyio.run(profiler.get_builtin_browser_status) + if not status["running"]: + console.print(Panel( + "[yellow]No builtin browser is currently running[/yellow]", + title="Builtin Browser Stop", + border_style="yellow" + )) + return + + console.print(Panel( + "[cyan]Stopping builtin browser...[/cyan]", + title="Builtin Browser Stop", + border_style="cyan" + )) + + success = anyio.run(profiler.kill_builtin_browser) + + if success: + console.print(Panel( + "[green]Builtin browser stopped successfully[/green]", + title="Builtin Browser Stop", + border_style="green" + )) + else: + console.print(Panel( + "[red]Failed to stop builtin browser[/red]", + title="Builtin Browser Stop", + border_style="red" + )) + sys.exit(1) + + except Exception as e: + console.print(f"[red]Error stopping builtin browser: {str(e)}[/red]") + sys.exit(1) + +@browser_cmd.command("view") +@click.option("--url", "-u", help="URL to navigate to (defaults to about:blank)") +def browser_view_cmd(url: Optional[str]): + """ + Open a visible window of the builtin browser + + This command connects to the running builtin browser and opens a visible window, + allowing you to see what the browser is currently viewing or navigate to a URL. + """ + profiler = BrowserProfiler() + + try: + # First check if browser is running + status = anyio.run(profiler.get_builtin_browser_status) + if not status["running"]: + console.print(Panel( + "[yellow]No builtin browser is currently running[/yellow]\n\n" + "Use 'crwl browser start' to start a builtin browser first", + title="Builtin Browser View", + border_style="yellow" + )) + return + + info = status["info"] + cdp_url = info["cdp_url"] + + console.print(Panel( + f"[cyan]Opening visible window connected to builtin browser[/cyan]\n\n" + f"CDP URL: [green]{cdp_url}[/green]\n" + f"URL to load: [yellow]{url or 'about:blank'}[/yellow]", + title="Builtin Browser View", + border_style="cyan" + )) + + # Use the CDP URL to launch a new visible window + import subprocess + import os + + # Determine the browser command based on platform + if sys.platform == "darwin": # macOS + browser_cmd = ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] + elif sys.platform == "win32": # Windows + browser_cmd = ["C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"] + else: # Linux + browser_cmd = ["google-chrome"] + + # Add arguments + browser_args = [ + f"--remote-debugging-port={info['debugging_port']}", + "--remote-debugging-address=localhost", + "--no-first-run", + "--no-default-browser-check" + ] + + # Add URL if provided + if url: + browser_args.append(url) + + # Launch browser + try: + subprocess.Popen(browser_cmd + browser_args) + console.print("[green]Browser window opened. Close it when finished viewing.[/green]") + except Exception as e: + console.print(f"[red]Error launching browser: {str(e)}[/red]") + console.print(f"[yellow]Try connecting manually to {cdp_url} in Chrome or using the '--remote-debugging-port' flag.[/yellow]") + + except Exception as e: + console.print(f"[red]Error viewing builtin browser: {str(e)}[/red]") + sys.exit(1) + +@browser_cmd.command("restart") +@click.option("--browser-type", "-b", type=click.Choice(["chromium", "firefox"]), default=None, + help="Browser type (defaults to same as current)") +@click.option("--port", "-p", type=int, default=None, help="Debugging port (defaults to same as current)") +@click.option("--headless/--no-headless", default=None, help="Run browser in headless mode") +def browser_restart_cmd(browser_type: Optional[str], port: Optional[int], headless: Optional[bool]): + """Restart the builtin browser + + Stops the current builtin browser if running and starts a new one. + By default, uses the same configuration as the current browser. + """ + profiler = BrowserProfiler() + + try: + # First check if browser is running and get its config + status = anyio.run(profiler.get_builtin_browser_status) + current_config = {} + + if status["running"]: + info = status["info"] + current_config = { + "browser_type": info["browser_type"], + "port": info["debugging_port"], + "headless": True # Default assumption + } + + # Stop the browser + console.print(Panel( + "[cyan]Stopping current builtin browser...[/cyan]", + title="Builtin Browser Restart", + border_style="cyan" + )) + + success = anyio.run(profiler.kill_builtin_browser) + if not success: + console.print(Panel( + "[red]Failed to stop current browser[/red]", + title="Builtin Browser Restart", + border_style="red" + )) + sys.exit(1) + + # Use provided options or defaults from current config + browser_type = browser_type or current_config.get("browser_type", "chromium") + port = port or current_config.get("port", 9222) + headless = headless if headless is not None else current_config.get("headless", True) + + # Start a new browser + console.print(Panel( + f"[cyan]Starting new builtin browser[/cyan]\n\n" + f"Browser type: [green]{browser_type}[/green]\n" + f"Debugging port: [yellow]{port}[/yellow]\n" + f"Headless: [cyan]{'Yes' if headless else 'No'}[/cyan]", + title="Builtin Browser Restart", + border_style="cyan" + )) + + cdp_url = anyio.run( + profiler.launch_builtin_browser, + browser_type, + port, + headless + ) + + if cdp_url: + console.print(Panel( + f"[green]Builtin browser restarted successfully[/green]\n\n" + f"CDP URL: [cyan]{cdp_url}[/cyan]", + title="Builtin Browser Restart", + border_style="green" + )) + else: + console.print(Panel( + "[red]Failed to restart builtin browser[/red]", + title="Builtin Browser Restart", + border_style="red" + )) + sys.exit(1) + + except Exception as e: + console.print(f"[red]Error restarting builtin browser: {str(e)}[/red]") + sys.exit(1) + +@cli.command("cdp") +@click.option("--user-data-dir", "-d", help="Directory to use for browser data (will be created if it doesn't exist)") +@click.option("--port", "-P", type=int, default=9222, help="Debugging port (default: 9222)") +@click.option("--browser-type", "-b", type=click.Choice(["chromium", "firefox"]), default="chromium", + help="Browser type (default: chromium)") +@click.option("--headless", is_flag=True, help="Run browser in headless mode") +@click.option("--incognito", is_flag=True, help="Run in incognito/private mode (ignores user-data-dir)") +def cdp_cmd(user_data_dir: Optional[str], port: int, browser_type: str, headless: bool, incognito: bool): + """Launch a standalone browser with CDP debugging enabled + + This command launches a browser with Chrome DevTools Protocol (CDP) debugging enabled, + prints the CDP URL, and keeps the browser running until you press 'q'. + + The CDP URL can be used for various automation and debugging tasks. + + Examples: + # Launch Chromium with CDP on default port 9222 + crwl cdp + + # Use a specific directory for browser data and custom port + crwl cdp --user-data-dir ~/browser-data --port 9223 + + # Launch in headless mode + crwl cdp --headless + + # Launch in incognito mode (ignores user-data-dir) + crwl cdp --incognito + """ + profiler = BrowserProfiler() + + try: + # Handle data directory + data_dir = None + if not incognito and user_data_dir: + # Expand user path (~/something) + expanded_path = os.path.expanduser(user_data_dir) + + # Create directory if it doesn't exist + if not os.path.exists(expanded_path): + console.print(f"[yellow]Directory '{expanded_path}' doesn't exist. Creating it.[/yellow]") + os.makedirs(expanded_path, exist_ok=True) + + data_dir = expanded_path + + # Print launch info + console.print(Panel( + f"[cyan]Launching browser with CDP debugging[/cyan]\n\n" + f"Browser type: [green]{browser_type}[/green]\n" + f"Debugging port: [yellow]{port}[/yellow]\n" + f"User data directory: [cyan]{data_dir or 'Temporary directory'}[/cyan]\n" + f"Headless: [cyan]{'Yes' if headless else 'No'}[/cyan]\n" + f"Incognito: [cyan]{'Yes' if incognito else 'No'}[/cyan]\n\n" + f"[yellow]Press 'q' to quit when done[/yellow]", + title="CDP Browser", + border_style="cyan" + )) + + # Run the browser + cdp_url = anyio.run( + profiler.launch_standalone_browser, + browser_type, + data_dir, + port, + headless + ) + + if not cdp_url: + console.print("[red]Failed to launch browser or get CDP URL[/red]") + sys.exit(1) + + except Exception as e: + console.print(f"[red]Error launching CDP browser: {str(e)}[/red]") + sys.exit(1) + + +@cli.command("crawl") +@click.argument("url", required=True) +@click.option("--browser-config", "-B", type=click.Path(exists=True), help="Browser config file (YAML/JSON)") +@click.option("--crawler-config", "-C", type=click.Path(exists=True), help="Crawler config file (YAML/JSON)") +@click.option("--filter-config", "-f", type=click.Path(exists=True), help="Content filter config file") +@click.option("--extraction-config", "-e", type=click.Path(exists=True), help="Extraction strategy config file") +@click.option("--json-extract", "-j", is_flag=False, flag_value="", default=None, help="Extract structured data using LLM with optional description") +@click.option("--schema", "-s", type=click.Path(exists=True), help="JSON schema for extraction") +@click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2") +@click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2") +@click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all") +@click.option("--output-file", "-O", type=click.Path(), help="Output file path (default: stdout)") +@click.option("--bypass-cache", "-bc", is_flag=True, default=True, help="Bypass cache when crawling") +@click.option("--question", "-q", help="Ask a question about the crawled content") +@click.option("--verbose", "-v", is_flag=True) +@click.option("--profile", "-p", help="Use a specific browser profile (by name)") +@click.option("--deep-crawl", type=click.Choice(["bfs", "dfs", "best-first"]), help="Enable deep crawling with specified strategy (bfs, dfs, or best-first)") +@click.option("--max-pages", type=int, default=10, help="Maximum number of pages to crawl in deep crawl mode") +@click.option("--json-ensure-ascii/--no-json-ensure-ascii", default=None, help="Escape non-ASCII characters in JSON output (default: from global config)") +def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config: str, + extraction_config: str, json_extract: str, schema: str, browser: Dict, crawler: Dict, + output: str, output_file: str, bypass_cache: bool, question: str, verbose: bool, profile: str, deep_crawl: str, max_pages: int, json_ensure_ascii: Optional[bool]): + """Crawl a website and extract content + + Simple Usage: + crwl crawl https://example.com + """ + + # Handle profile option + if profile: + profiler = BrowserProfiler() + profile_path = profiler.get_profile_path(profile) + + if not profile_path: + profiles = profiler.list_profiles() + + if profiles: + console.print(f"[red]Profile '{profile}' not found. Available profiles:[/red]") + display_profiles_table(profiles) + else: + console.print("[red]No profiles found. Create one with 'crwl profiles'[/red]") + + return + + # Include the profile in browser config + if not browser: + browser = {} + browser["user_data_dir"] = profile_path + browser["use_managed_browser"] = True + + if verbose: + console.print(f"[green]Using browser profile:[/green] {profile}") + + try: + # Load base configurations + browser_cfg = BrowserConfig.load(load_config_file(browser_config)) + crawler_cfg = CrawlerRunConfig.load(load_config_file(crawler_config)) + + # Override with CLI params + if browser: + browser_cfg = browser_cfg.clone(**browser) + if crawler: + crawler_cfg = crawler_cfg.clone(**crawler) + + # Handle content filter config + if filter_config or output in ["markdown-fit", "md-fit"]: + if filter_config: + filter_conf = load_config_file(filter_config) + elif not filter_config and output in ["markdown-fit", "md-fit"]: + filter_conf = { + "type": "pruning", + "query": "", + "threshold": 0.48 + } + if filter_conf["type"] == "bm25": + crawler_cfg.markdown_generator = DefaultMarkdownGenerator( + content_filter = BM25ContentFilter( + user_query=filter_conf.get("query"), + bm25_threshold=filter_conf.get("threshold", 1.0), + use_stemming=filter_conf.get("use_stemming", True), + ) + ) + elif filter_conf["type"] == "pruning": + crawler_cfg.markdown_generator = DefaultMarkdownGenerator( + content_filter = PruningContentFilter( + user_query=filter_conf.get("query"), + threshold=filter_conf.get("threshold", 0.48) + ) + ) + + # Handle json-extract option (takes precedence over extraction-config) + if json_extract is not None: + # Get LLM provider and token + provider, token = setup_llm_config() + + # Default sophisticated instruction for structured data extraction + default_instruction = """Analyze the web page content and extract structured data as JSON. +If the page contains a list of items with repeated patterns, extract all items in an array. +If the page is an article or contains unique content, extract a comprehensive JSON object with all relevant information. +Look at the content, intention of content, what it offers and find the data item(s) in the page. +Always return valid, properly formatted JSON.""" + + + default_instruction_with_user_query = """Analyze the web page content and extract structured data as JSON, following the below instruction and explanation of schema and always return valid, properly formatted JSON. \n\nInstruction:\n\n""" + json_extract + + # Determine instruction based on whether json_extract is empty or has content + instruction = default_instruction_with_user_query if json_extract else default_instruction + + # Create LLM extraction strategy + crawler_cfg.extraction_strategy = LLMExtractionStrategy( + llm_config=LLMConfig(provider=provider, api_token=token), + instruction=instruction, + schema=load_schema_file(schema), # Will be None if no schema is provided + extraction_type="schema", #if schema else "block", + apply_chunking=False, + force_json_response=True, + verbose=verbose, + ) + + # Set output to JSON if not explicitly specified + if output == "all": + output = "json" + + # Handle extraction strategy from config file (only if json-extract wasn't used) + elif extraction_config: + extract_conf = load_config_file(extraction_config) + schema_data = load_schema_file(schema) + + # Check if type does not exist show proper message + if not extract_conf.get("type"): + raise click.ClickException("Extraction type not specified") + if extract_conf["type"] not in ["llm", "json-css", "json-xpath"]: + raise click.ClickException(f"Invalid extraction type: {extract_conf['type']}") + + if extract_conf["type"] == "llm": + # if no provider show error emssage + if not extract_conf.get("provider") or not extract_conf.get("api_token"): + raise click.ClickException("LLM provider and API token are required for LLM extraction") + + crawler_cfg.extraction_strategy = LLMExtractionStrategy( + llm_config=LLMConfig(provider=extract_conf["provider"], api_token=extract_conf["api_token"]), + instruction=extract_conf["instruction"], + schema=schema_data, + **extract_conf.get("params", {}) + ) + elif extract_conf["type"] == "json-css": + crawler_cfg.extraction_strategy = JsonCssExtractionStrategy( + schema=schema_data + ) + elif extract_conf["type"] == "json-xpath": + crawler_cfg.extraction_strategy = JsonXPathExtractionStrategy( + schema=schema_data + ) + + + # No cache + if bypass_cache: + crawler_cfg.cache_mode = CacheMode.BYPASS + + crawler_cfg.scraping_strategy = LXMLWebScrapingStrategy() + + # Handle deep crawling configuration + if deep_crawl: + if deep_crawl == "bfs": + crawler_cfg.deep_crawl_strategy = BFSDeepCrawlStrategy( + max_depth=3, + max_pages=max_pages + ) + elif deep_crawl == "dfs": + crawler_cfg.deep_crawl_strategy = DFSDeepCrawlStrategy( + max_depth=3, + max_pages=max_pages + ) + elif deep_crawl == "best-first": + crawler_cfg.deep_crawl_strategy = BestFirstCrawlingStrategy( + max_depth=3, + max_pages=max_pages + ) + + if verbose: + console.print(f"[green]Deep crawling enabled:[/green] {deep_crawl} strategy, max {max_pages} pages") + + config = get_global_config() + + browser_cfg.verbose = config.get("VERBOSE", False) + crawler_cfg.verbose = config.get("VERBOSE", False) + + # Get JSON output config (priority: CLI flag > global config) + if json_ensure_ascii is not None: + ensure_ascii = json_ensure_ascii + else: + ensure_ascii = config.get("JSON_ENSURE_ASCII", USER_SETTINGS["JSON_ENSURE_ASCII"]["default"]) + + # Run crawler + result : CrawlResult = anyio.run( + run_crawler, + url, + browser_cfg, + crawler_cfg, + verbose + ) + + # Handle deep crawl results (list) vs single result + if isinstance(result, list): + if len(result) == 0: + click.echo("No results found during deep crawling") + return + # Use the first result for question answering and output + main_result = result[0] + all_results = result + else: + # Single result from regular crawling + main_result = result + all_results = [result] + + # Handle question + if question: + provider, token = setup_llm_config() + markdown = main_result.markdown.raw_markdown + anyio.run(stream_llm_response, url, markdown, question, provider, token) + return + + # Handle output + if not output_file: + if output == "all": + if isinstance(result, list): + output_data = [r.model_dump() for r in all_results] + click.echo(json.dumps(output_data, indent=2, ensure_ascii=ensure_ascii)) + else: + click.echo(json.dumps(main_result.model_dump(), indent=2, ensure_ascii=ensure_ascii)) + elif output == "json": + print(main_result.extracted_content) + extracted_items = json.loads(main_result.extracted_content) + click.echo(json.dumps(extracted_items, indent=2, ensure_ascii=ensure_ascii)) + + elif output in ["markdown", "md"]: + if isinstance(result, list): + # Combine markdown from all crawled pages for deep crawl + for r in all_results: + click.echo(f"\n\n{'='*60}\n# {r.url}\n{'='*60}\n\n") + click.echo(r.markdown.raw_markdown) + else: + click.echo(main_result.markdown.raw_markdown) + elif output in ["markdown-fit", "md-fit"]: + if isinstance(result, list): + # Combine fit markdown from all crawled pages for deep crawl + for r in all_results: + click.echo(f"\n\n{'='*60}\n# {r.url}\n{'='*60}\n\n") + click.echo(r.markdown.fit_markdown) + else: + click.echo(main_result.markdown.fit_markdown) + else: + if output == "all": + with open(output_file, "w", encoding="utf-8") as f: + if isinstance(result, list): + output_data = [r.model_dump() for r in all_results] + f.write(json.dumps(output_data, indent=2, ensure_ascii=ensure_ascii)) + else: + f.write(json.dumps(main_result.model_dump(), indent=2, ensure_ascii=ensure_ascii)) + elif output == "json": + with open(output_file, "w", encoding="utf-8") as f: + f.write(main_result.extracted_content) + elif output in ["markdown", "md"]: + with open(output_file, "w", encoding="utf-8") as f: + if isinstance(result, list): + # Combine markdown from all crawled pages for deep crawl + for r in all_results: + f.write(f"\n\n{'='*60}\n# {r.url}\n{'='*60}\n\n") + f.write(r.markdown.raw_markdown) + else: + f.write(main_result.markdown.raw_markdown) + elif output in ["markdown-fit", "md-fit"]: + with open(output_file, "w", encoding="utf-8") as f: + if isinstance(result, list): + # Combine fit markdown from all crawled pages for deep crawl + for r in all_results: + f.write(f"\n\n{'='*60}\n# {r.url}\n{'='*60}\n\n") + f.write(r.markdown.fit_markdown) + else: + f.write(main_result.markdown.fit_markdown) + + except Exception as e: + raise click.ClickException(str(e)) + +@cli.command("examples") +def examples_cmd(): + """Show usage examples""" + show_examples() + +@cli.group("config") +def config_cmd(): + """Manage global configuration settings + + Commands to view and update global configuration settings: + - list: Display all current configuration settings + - get: Get the value of a specific setting + - set: Set the value of a specific setting + """ + pass + +@config_cmd.command("list") +def config_list_cmd(): + """List all configuration settings""" + config = get_global_config() + + table = Table(title="Crawl4AI Configuration", show_header=True, header_style="bold cyan", border_style="blue") + table.add_column("Setting", style="cyan") + table.add_column("Value", style="green") + table.add_column("Default", style="yellow") + table.add_column("Description", style="white") + + for key, setting in USER_SETTINGS.items(): + value = config.get(key, setting["default"]) + + # Handle secret values + display_value = value + if setting.get("secret", False) and value: + display_value = "********" + + # Handle boolean values + if setting["type"] == "boolean": + display_value = str(value).lower() + default_value = str(setting["default"]).lower() + else: + default_value = str(setting["default"]) + + table.add_row( + key, + str(display_value), + default_value, + setting["description"] + ) + + console.print(table) + +@config_cmd.command("get") +@click.argument("key", required=True) +def config_get_cmd(key: str): + """Get a specific configuration setting""" + config = get_global_config() + + # Normalize key to uppercase + key = key.upper() + + if key not in USER_SETTINGS: + console.print(f"[red]Error: Unknown setting '{key}'[/red]") + return + + value = config.get(key, USER_SETTINGS[key]["default"]) + + # Handle secret values + display_value = value + if USER_SETTINGS[key].get("secret", False) and value: + display_value = "********" + + console.print(f"[cyan]{key}[/cyan] = [green]{display_value}[/green]") + console.print(f"[dim]Description: {USER_SETTINGS[key]['description']}[/dim]") + +@config_cmd.command("set") +@click.argument("key", required=True) +@click.argument("value", required=True) +def config_set_cmd(key: str, value: str): + """Set a configuration setting""" + config = get_global_config() + + # Normalize key to uppercase + key = key.upper() + + if key not in USER_SETTINGS: + console.print(f"[red]Error: Unknown setting '{key}'[/red]") + console.print(f"[yellow]Available settings: {', '.join(USER_SETTINGS.keys())}[/yellow]") + return + + setting = USER_SETTINGS[key] + + # Type conversion and validation + if setting["type"] == "boolean": + if value.lower() in ["true", "yes", "1", "y"]: + typed_value = True + elif value.lower() in ["false", "no", "0", "n"]: + typed_value = False + else: + console.print(f"[red]Error: Invalid boolean value. Use 'true' or 'false'.[/red]") + return + elif setting["type"] == "string": + typed_value = value + + # Check if the value should be one of the allowed options + if "options" in setting and value not in setting["options"]: + console.print(f"[red]Error: Value must be one of: {', '.join(setting['options'])}[/red]") + return + + # Update config + config[key] = typed_value + save_global_config(config) + + # Handle secret values for display + display_value = typed_value + if setting.get("secret", False) and typed_value: + display_value = "********" + + console.print(f"[green]Successfully set[/green] [cyan]{key}[/cyan] = [green]{display_value}[/green]") + +@cli.group("profiles", invoke_without_command=True) +@click.pass_context +def profiles_cmd(ctx): + """Manage browser profiles for authenticated crawling + + Launch an interactive browser profile manager where you can: + - List all existing profiles + - Create new profiles for authenticated browsing + - Delete unused profiles + + Subcommands: + crwl profiles create - Create a new profile + crwl profiles list - List all profiles + crwl profiles delete - Delete a profile + + Or run without subcommand for interactive menu: + crwl profiles + """ + # If no subcommand provided, run interactive manager + if ctx.invoked_subcommand is None: + anyio.run(manage_profiles) + + +@profiles_cmd.command("create") +@click.argument("name") +def profiles_create_cmd(name: str): + """Create a new browser profile + + Opens a browser window for you to log in and set up your identity. + Press 'q' in the terminal when finished to save the profile. + + Example: + crwl profiles create github-auth + """ + profiler = BrowserProfiler() + console.print(Panel(f"[bold cyan]Creating Profile: {name}[/bold cyan]\n" + "A browser window will open for you to set up your identity.\n" + "Log in to sites, adjust settings, then press 'q' to save.", + border_style="cyan")) + + async def _create(): + try: + profile_path = await profiler.create_profile(name) + if profile_path: + console.print(f"[green]Profile successfully created at:[/green] {profile_path}") + else: + console.print("[red]Failed to create profile.[/red]") + sys.exit(1) + except Exception as e: + console.print(f"[red]Error creating profile: {str(e)}[/red]") + sys.exit(1) + + anyio.run(_create) + + +@profiles_cmd.command("list") +def profiles_list_cmd(): + """List all browser profiles + + Example: + crwl profiles list + """ + profiler = BrowserProfiler() + profiles = profiler.list_profiles() + display_profiles_table(profiles) + + +@profiles_cmd.command("delete") +@click.argument("name") +@click.option("--force", "-f", is_flag=True, help="Skip confirmation") +def profiles_delete_cmd(name: str, force: bool): + """Delete a browser profile + + Example: + crwl profiles delete old-profile + crwl profiles delete old-profile --force + """ + profiler = BrowserProfiler() + + # Find profile by name + profiles = profiler.list_profiles() + profile = next((p for p in profiles if p["name"] == name), None) + + if not profile: + console.print(f"[red]Profile not found:[/red] {name}") + sys.exit(1) + + if not force: + if not Confirm.ask(f"[yellow]Delete profile '{name}'?[/yellow]"): + console.print("[cyan]Cancelled.[/cyan]") + return + + try: + profiler.delete_profile(name) + console.print(f"[green]Profile '{name}' deleted successfully.[/green]") + except Exception as e: + console.print(f"[red]Error deleting profile: {str(e)}[/red]") + sys.exit(1) + + +@cli.command("shrink") +@click.argument("profile_name") +@click.option( + "--level", "-l", + type=click.Choice(["light", "medium", "aggressive", "minimal"]), + default="aggressive", + help="Shrink level (default: aggressive)" +) +@click.option("--dry-run", "-n", is_flag=True, help="Preview without removing files") +def shrink_cmd(profile_name: str, level: str, dry_run: bool): + """Shrink a browser profile to reduce storage. + + Removes cache, history, and other non-essential data while preserving + authentication (cookies, localStorage, IndexedDB). + + Shrink levels: + light - Remove caches only + medium - Remove caches + history + aggressive - Keep only auth data (recommended) + minimal - Keep only cookies + localStorage + + Examples: + crwl shrink my_profile + crwl shrink my_profile --level minimal + crwl shrink my_profile --dry-run + """ + profiler = BrowserProfiler() + + try: + result = profiler.shrink(profile_name, ShrinkLevel(level), dry_run) + except ValueError as e: + console.print(f"[red]Error:[/red] {e}") + sys.exit(1) + + # Display results + action = "Would remove" if dry_run else "Removed" + console.print(f"\n[cyan]Shrink Results ({level.upper()}):[/cyan]") + console.print(f" {action}: {len(result['removed'])} items") + console.print(f" Kept: {len(result['kept'])} items") + console.print(f" Space freed: {_format_size(result['bytes_freed'])}") + + if result.get("size_before"): + console.print(f" Size before: {_format_size(result['size_before'])}") + if result.get("size_after"): + console.print(f" Size after: {_format_size(result['size_after'])}") + + if result["errors"]: + console.print(f"\n[red]Errors ({len(result['errors'])}):[/red]") + for err in result["errors"]: + console.print(f" - {err}") + + if dry_run: + console.print("\n[yellow]Dry run - no files were actually removed.[/yellow]") + +@cli.command(name="") +@click.argument("url", required=False) +@click.option("--example", is_flag=True, help="Show usage examples") +@click.option("--browser-config", "-B", type=click.Path(exists=True), help="Browser config file (YAML/JSON)") +@click.option("--crawler-config", "-C", type=click.Path(exists=True), help="Crawler config file (YAML/JSON)") +@click.option("--filter-config", "-f", type=click.Path(exists=True), help="Content filter config file") +@click.option("--extraction-config", "-e", type=click.Path(exists=True), help="Extraction strategy config file") +@click.option("--json-extract", "-j", is_flag=False, flag_value="", default=None, help="Extract structured data using LLM with optional description") +@click.option("--schema", "-s", type=click.Path(exists=True), help="JSON schema for extraction") +@click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2") +@click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2") +@click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all") +@click.option("--bypass-cache", is_flag=True, default=True, help="Bypass cache when crawling") +@click.option("--question", "-q", help="Ask a question about the crawled content") +@click.option("--verbose", "-v", is_flag=True) +@click.option("--profile", "-p", help="Use a specific browser profile (by name)") +@click.option("--deep-crawl", type=click.Choice(["bfs", "dfs", "best-first"]), help="Enable deep crawling with specified strategy") +@click.option("--max-pages", type=int, default=10, help="Maximum number of pages to crawl in deep crawl mode") +@click.option("--json-ensure-ascii/--no-json-ensure-ascii", default=None, help="Escape non-ASCII characters in JSON output (default: from global config)") +def default(url: str, example: bool, browser_config: str, crawler_config: str, filter_config: str, + extraction_config: str, json_extract: str, schema: str, browser: Dict, crawler: Dict, + output: str, bypass_cache: bool, question: str, verbose: bool, profile: str, deep_crawl: str, max_pages: int, json_ensure_ascii: Optional[bool]): + """Crawl4AI CLI - Web content extraction tool + + Simple Usage: + crwl https://example.com + + Run with --example to see detailed usage examples. + + Other commands: + crwl profiles - Manage browser profiles for identity-based crawling + crwl crawl - Crawl a website with advanced options + crwl cdp - Launch browser with CDP debugging enabled + crwl browser - Manage builtin browser (start, stop, status, restart) + crwl config - Manage global configuration settings + crwl examples - Show more usage examples + + Configuration Examples: + crwl config list - List all configuration settings + crwl config get DEFAULT_LLM_PROVIDER - Show current LLM provider + crwl config set VERBOSE true - Enable verbose mode globally + crwl config set BROWSER_HEADLESS false - Default to visible browser + """ + + if example: + show_examples() + return + + if not url: + # Show help without error message + ctx = click.get_current_context() + click.echo(ctx.get_help()) + return + + # Forward to crawl command + ctx = click.get_current_context() + ctx.invoke( + crawl_cmd, + url=url, + browser_config=browser_config, + crawler_config=crawler_config, + filter_config=filter_config, + extraction_config=extraction_config, + json_extract=json_extract, + schema=schema, + browser=browser, + crawler=crawler, + output=output, + bypass_cache=bypass_cache, + question=question, + verbose=verbose, + profile=profile, + deep_crawl=deep_crawl, + max_pages=max_pages, + json_ensure_ascii=json_ensure_ascii + ) + +def main(): + import sys + if len(sys.argv) < 2 or sys.argv[1] not in cli.commands: + sys.argv.insert(1, "crawl") + cli() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/crawl4ai/cloud/__init__.py b/crawl4ai/cloud/__init__.py new file mode 100644 index 0000000..74f7548 --- /dev/null +++ b/crawl4ai/cloud/__init__.py @@ -0,0 +1,16 @@ +""" +Crawl4AI Cloud Module - Integration with Crawl4AI Cloud service. + +This module provides: +- CLI commands for cloud profile management +- API client for cloud operations (future) +- Cloud configuration utilities +""" + +from .cli import cloud_cmd, get_cloud_config, require_auth + +__all__ = [ + "cloud_cmd", + "get_cloud_config", + "require_auth", +] diff --git a/crawl4ai/cloud/cli.py b/crawl4ai/cloud/cli.py new file mode 100644 index 0000000..355b48f --- /dev/null +++ b/crawl4ai/cloud/cli.py @@ -0,0 +1,473 @@ +""" +Crawl4AI Cloud CLI - Commands for interacting with Crawl4AI Cloud service. + +Commands: + crwl cloud auth - Authenticate with API key + crwl cloud profiles upload - Upload a profile to cloud + crwl cloud profiles list - List cloud profiles + crwl cloud profiles delete - Delete a cloud profile +""" + +import click +import httpx +import os +import shutil +import sys +import tarfile +import tempfile +from pathlib import Path + +import yaml +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from crawl4ai import BrowserProfiler +from crawl4ai.browser_profiler import ShrinkLevel, _format_size + +console = Console() + +# Default cloud API URL +DEFAULT_CLOUD_API_URL = "https://api.crawl4ai.com" + + +def get_global_config() -> dict: + """Load global config from ~/.crawl4ai/global.yml""" + config_file = Path.home() / ".crawl4ai" / "global.yml" + if not config_file.exists(): + return {} + with open(config_file) as f: + return yaml.safe_load(f) or {} + + +def save_global_config(config: dict): + """Save global config to ~/.crawl4ai/global.yml""" + config_dir = Path.home() / ".crawl4ai" + config_dir.mkdir(parents=True, exist_ok=True) + config_file = config_dir / "global.yml" + with open(config_file, "w") as f: + yaml.dump(config, f) + + +def get_cloud_config() -> tuple[str, str]: + """Get cloud API key and URL from config.""" + config = get_global_config() + api_key = config.get("CLOUD_API_KEY") + api_url = config.get("CLOUD_API_URL", DEFAULT_CLOUD_API_URL) + return api_key, api_url + + +def require_auth() -> tuple[str, str]: + """Require authentication, exit if not configured.""" + api_key, api_url = get_cloud_config() + if not api_key: + console.print("[red]Not authenticated with Crawl4AI Cloud.[/red]") + console.print("\nRun [cyan]crwl cloud auth[/cyan] to authenticate.") + sys.exit(1) + return api_key, api_url + + +# ==================== Cloud Command Group ==================== + +@click.group("cloud") +def cloud_cmd(): + """Crawl4AI Cloud commands - manage cloud profiles and authentication. + + Use browser profiles for authenticated crawling in the cloud. + + Getting started: + 1. Get an API key at https://api.crawl4ai.com/dashboard + 2. Run: crwl cloud auth + 3. Create a local profile: crwl profiles + 4. Upload to cloud: crwl cloud profiles upload my_profile + """ + pass + + +# ==================== Auth Commands ==================== + +@cloud_cmd.command("auth") +@click.option("--api-key", "-k", help="API key (will prompt if not provided)") +@click.option("--api-url", "-u", help=f"API URL (default: {DEFAULT_CLOUD_API_URL})") +@click.option("--logout", is_flag=True, help="Remove saved credentials") +@click.option("--status", is_flag=True, help="Show current auth status") +def auth_cmd(api_key: str, api_url: str, logout: bool, status: bool): + """Authenticate with Crawl4AI Cloud. + + Your API key is saved locally in ~/.crawl4ai/global.yml + + To get an API key: + 1. Go to https://api.crawl4ai.com/dashboard + 2. Sign in or create an account + 3. Navigate to API Keys section + 4. Create a new key and copy it + + Examples: + crwl cloud auth # Interactive authentication + crwl cloud auth --api-key sk_... # Provide key directly + crwl cloud auth --status # Check current status + crwl cloud auth --logout # Remove saved credentials + """ + config = get_global_config() + + if status: + current_key = config.get("CLOUD_API_KEY") + current_url = config.get("CLOUD_API_URL", DEFAULT_CLOUD_API_URL) + + if current_key: + # Mask the key for display + masked = current_key[:8] + "..." + current_key[-4:] if len(current_key) > 12 else "***" + console.print(Panel( + f"[green]Authenticated[/green]\n\n" + f"API Key: [cyan]{masked}[/cyan]\n" + f"API URL: [blue]{current_url}[/blue]", + title="Cloud Auth Status", + border_style="green" + )) + else: + console.print(Panel( + "[yellow]Not authenticated[/yellow]\n\n" + "Run [cyan]crwl cloud auth[/cyan] to authenticate.\n\n" + "Get your API key at:\n" + "[blue]https://api.crawl4ai.com/dashboard[/blue]", + title="Cloud Auth Status", + border_style="yellow" + )) + return + + if logout: + if "CLOUD_API_KEY" in config: + del config["CLOUD_API_KEY"] + save_global_config(config) + console.print("[green]Logged out successfully.[/green]") + else: + console.print("[yellow]Not currently authenticated.[/yellow]") + return + + # Interactive auth + if not api_key: + console.print(Panel( + "[cyan]Crawl4AI Cloud Authentication[/cyan]\n\n" + "To get your API key:\n" + " 1. Go to [blue]https://api.crawl4ai.com/dashboard[/blue]\n" + " 2. Sign in or create an account\n" + " 3. Navigate to API Keys section\n" + " 4. Create a new key and paste it below", + title="Setup", + border_style="cyan" + )) + api_key = click.prompt("\nEnter your API key", hide_input=True) + + if not api_key: + console.print("[red]API key is required.[/red]") + sys.exit(1) + + # Validate the key by making a test request + test_url = api_url or config.get("CLOUD_API_URL", DEFAULT_CLOUD_API_URL) + + console.print("\n[dim]Validating API key...[/dim]") + + try: + response = httpx.get( + f"{test_url}/v1/profiles", + headers={"X-API-Key": api_key}, + timeout=10.0 + ) + + if response.status_code == 401: + console.print("[red]Invalid API key.[/red]") + sys.exit(1) + elif response.status_code != 200: + console.print(f"[red]Error validating key: {response.status_code}[/red]") + sys.exit(1) + + except httpx.RequestError as e: + console.print(f"[red]Connection error: {e}[/red]") + sys.exit(1) + + # Save to config + config["CLOUD_API_KEY"] = api_key + if api_url: + config["CLOUD_API_URL"] = api_url + save_global_config(config) + + console.print("[green]Authentication successful![/green]") + console.print(f"Credentials saved to [cyan]~/.crawl4ai/global.yml[/cyan]") + + +# ==================== Profiles Command Group ==================== + +@cloud_cmd.group("profiles") +def profiles_cmd(): + """Manage cloud browser profiles. + + Upload local browser profiles to Crawl4AI Cloud for authenticated crawling. + + Workflow: + 1. Create a local profile: crwl profiles + 2. Shrink it (optional): crwl shrink my_profile + 3. Upload to cloud: crwl cloud profiles upload my_profile + 4. Use in API: {"browser_config": {"profile_id": "..."}} + """ + pass + + +@profiles_cmd.command("upload") +@click.argument("profile_name") +@click.option("--name", "-n", help="Cloud profile name (defaults to local name)") +@click.option("--level", "-l", + type=click.Choice(["light", "medium", "aggressive", "minimal"]), + default="aggressive", + help="Shrink level before upload (default: aggressive)") +@click.option("--no-shrink", is_flag=True, help="Skip shrinking (upload as-is)") +def upload_cmd(profile_name: str, name: str, level: str, no_shrink: bool): + """Upload a browser profile to Crawl4AI Cloud. + + The profile will be shrunk to remove caches before uploading. + Use --no-shrink to upload the profile as-is. + + Examples: + crwl cloud profiles upload my_profile + crwl cloud profiles upload my_profile --name github-auth + crwl cloud profiles upload my_profile --level minimal + crwl cloud profiles upload my_profile --no-shrink + """ + api_key, api_url = require_auth() + + # Find the profile + profiler = BrowserProfiler() + profile_path = profiler.get_profile_path(profile_name) + + if not profile_path: + console.print(f"[red]Profile not found: {profile_name}[/red]") + console.print("\nAvailable profiles:") + for p in profiler.list_profiles(): + console.print(f" - {p['name']}") + sys.exit(1) + + cloud_name = name or profile_name + + console.print(f"\n[cyan]Uploading profile:[/cyan] {profile_name}") + console.print(f"[cyan]Cloud name:[/cyan] {cloud_name}") + + # Step 1: Shrink (unless --no-shrink) + if not no_shrink: + console.print(f"\n[dim][1/4] Shrinking profile ({level})...[/dim]") + try: + result = profiler.shrink(profile_name, ShrinkLevel(level), dry_run=False) + console.print(f" Freed: {_format_size(result['bytes_freed'])}") + if result.get("size_after"): + console.print(f" Size: {_format_size(result['size_after'])}") + except Exception as e: + console.print(f"[yellow]Warning: Could not shrink profile: {e}[/yellow]") + else: + console.print("\n[dim][1/4] Skipping shrink...[/dim]") + + # Step 2: Package as tar.gz + console.print("[dim][2/4] Packaging profile...[/dim]") + + temp_dir = Path(tempfile.mkdtemp(prefix="crawl4ai_upload_")) + tar_path = temp_dir / f"{cloud_name}.tar.gz" + + try: + with tarfile.open(tar_path, "w:gz") as tar: + # Add profile contents (not the directory itself) + for item in Path(profile_path).iterdir(): + tar.add(item, arcname=item.name) + + size_bytes = tar_path.stat().st_size + console.print(f" Created: {tar_path.name} ({_format_size(size_bytes)})") + + # Step 3: Upload + console.print("[dim][3/4] Uploading to cloud...[/dim]") + + with open(tar_path, "rb") as f: + response = httpx.post( + f"{api_url}/v1/profiles", + headers={"X-API-Key": api_key}, + files={"file": (f"{cloud_name}.tar.gz", f, "application/gzip")}, + data={"name": cloud_name}, + timeout=120.0 + ) + + if response.status_code == 409: + console.print(f"[red]Profile '{cloud_name}' already exists in cloud.[/red]") + console.print("Use --name to specify a different name, or delete the existing profile first.") + sys.exit(1) + elif response.status_code == 400: + error = response.json().get("detail", "Unknown error") + console.print(f"[red]Upload rejected: {error}[/red]") + sys.exit(1) + elif response.status_code != 200: + console.print(f"[red]Upload failed: {response.status_code}[/red]") + console.print(response.text) + sys.exit(1) + + result = response.json() + profile_id = result["id"] + + console.print("[dim][4/4] Done![/dim]") + + # Success output + console.print(Panel( + f"[green]Profile uploaded successfully![/green]\n\n" + f"Profile ID: [cyan]{profile_id}[/cyan]\n" + f"Name: [blue]{cloud_name}[/blue]\n" + f"Size: {_format_size(size_bytes)}\n\n" + f"[dim]Use in API:[/dim]\n" + f' {{"browser_config": {{"profile_id": "{profile_id}"}}}}', + title="Upload Complete", + border_style="green" + )) + + if result.get("scan_warnings"): + console.print("\n[yellow]Scan warnings:[/yellow]") + for warning in result["scan_warnings"]: + console.print(f" - {warning}") + + finally: + # Cleanup temp directory + shutil.rmtree(temp_dir, ignore_errors=True) + + +@profiles_cmd.command("list") +def list_cmd(): + """List all cloud profiles. + + Shows all profiles uploaded to your Crawl4AI Cloud account. + """ + api_key, api_url = require_auth() + + console.print("\n[dim]Fetching profiles...[/dim]") + + try: + response = httpx.get( + f"{api_url}/v1/profiles", + headers={"X-API-Key": api_key}, + timeout=30.0 + ) + + if response.status_code != 200: + console.print(f"[red]Error: {response.status_code}[/red]") + console.print(response.text) + sys.exit(1) + + data = response.json() + profiles = data.get("profiles", []) + + if not profiles: + console.print(Panel( + "[yellow]No cloud profiles found.[/yellow]\n\n" + "Upload a profile with:\n" + " [cyan]crwl cloud profiles upload [/cyan]", + title="Cloud Profiles", + border_style="yellow" + )) + return + + # Create table + table = Table(title="Cloud Profiles") + table.add_column("Name", style="cyan") + table.add_column("Profile ID", style="dim") + table.add_column("Size", justify="right") + table.add_column("Created", style="green") + table.add_column("Last Used", style="blue") + + for p in profiles: + size = _format_size(p.get("size_bytes", 0)) if p.get("size_bytes") else "-" + created = p.get("created_at", "-")[:10] if p.get("created_at") else "-" + last_used = p.get("last_used_at", "-")[:10] if p.get("last_used_at") else "Never" + + table.add_row( + p["name"], + p["id"][:8] + "...", + size, + created, + last_used + ) + + console.print(table) + console.print(f"\nTotal: {len(profiles)} profile(s)") + + except httpx.RequestError as e: + console.print(f"[red]Connection error: {e}[/red]") + sys.exit(1) + + +@profiles_cmd.command("delete") +@click.argument("profile_name_or_id") +@click.option("--yes", "-y", is_flag=True, help="Skip confirmation") +def delete_cmd(profile_name_or_id: str, yes: bool): + """Delete a cloud profile. + + You can specify either the profile name or ID. + + Examples: + crwl cloud profiles delete my_profile + crwl cloud profiles delete abc123... + crwl cloud profiles delete my_profile --yes + """ + api_key, api_url = require_auth() + + # First, try to find the profile + console.print("\n[dim]Finding profile...[/dim]") + + try: + # List profiles to find by name + response = httpx.get( + f"{api_url}/v1/profiles", + headers={"X-API-Key": api_key}, + timeout=30.0 + ) + + if response.status_code != 200: + console.print(f"[red]Error: {response.status_code}[/red]") + sys.exit(1) + + profiles = response.json().get("profiles", []) + + # Find matching profile + profile = None + for p in profiles: + if p["name"] == profile_name_or_id or p["id"] == profile_name_or_id or p["id"].startswith(profile_name_or_id): + profile = p + break + + if not profile: + console.print(f"[red]Profile not found: {profile_name_or_id}[/red]") + console.print("\nAvailable profiles:") + for p in profiles: + console.print(f" - {p['name']} ({p['id'][:8]}...)") + sys.exit(1) + + # Confirm deletion + console.print(f"\nProfile: [cyan]{profile['name']}[/cyan]") + console.print(f"ID: [dim]{profile['id']}[/dim]") + + if not yes: + if not click.confirm("\nAre you sure you want to delete this profile?"): + console.print("[yellow]Cancelled.[/yellow]") + return + + # Delete + console.print("\n[dim]Deleting...[/dim]") + + response = httpx.delete( + f"{api_url}/v1/profiles/{profile['id']}", + headers={"X-API-Key": api_key}, + timeout=30.0 + ) + + if response.status_code == 404: + console.print("[red]Profile not found (may have been already deleted).[/red]") + sys.exit(1) + elif response.status_code != 200: + console.print(f"[red]Error: {response.status_code}[/red]") + console.print(response.text) + sys.exit(1) + + console.print(f"[green]Profile '{profile['name']}' deleted successfully.[/green]") + + except httpx.RequestError as e: + console.print(f"[red]Connection error: {e}[/red]") + sys.exit(1) diff --git a/crawl4ai/components/crawler_monitor.py b/crawl4ai/components/crawler_monitor.py new file mode 100644 index 0000000..2a5e7e9 --- /dev/null +++ b/crawl4ai/components/crawler_monitor.py @@ -0,0 +1,869 @@ +import time +import uuid +import threading +import psutil +from datetime import datetime, timedelta +from typing import Dict, Optional, List +import threading +from rich.console import Console +from rich.layout import Layout +from rich.panel import Panel +from rich.table import Table +from rich.text import Text +from rich.live import Live +from rich import box +from ..models import CrawlStatus + +class TerminalUI: + """Terminal user interface for CrawlerMonitor using rich library.""" + + def __init__(self, refresh_rate: float = 1.0, max_width: int = 120): + """ + Initialize the terminal UI. + + Args: + refresh_rate: How often to refresh the UI (in seconds) + max_width: Maximum width of the UI in characters + """ + self.console = Console(width=max_width) + self.layout = Layout() + self.refresh_rate = refresh_rate + self.stop_event = threading.Event() + self.ui_thread = None + self.monitor = None # Will be set by CrawlerMonitor + self.max_width = max_width + + # Setup layout - vertical layout (top to bottom) + self.layout.split( + Layout(name="header", size=3), + Layout(name="pipeline_status", size=10), + Layout(name="task_details", ratio=1), + Layout(name="footer", size=3) # Increased footer size to fit all content + ) + + def start(self, monitor): + """Start the UI thread.""" + self.monitor = monitor + self.stop_event.clear() + self.ui_thread = threading.Thread(target=self._ui_loop) + self.ui_thread.daemon = True + self.ui_thread.start() + + def stop(self): + """Stop the UI thread.""" + if self.ui_thread and self.ui_thread.is_alive(): + self.stop_event.set() + # Only try to join if we're not in the UI thread + # This prevents "cannot join current thread" errors + if threading.current_thread() != self.ui_thread: + self.ui_thread.join(timeout=5.0) + + def _ui_loop(self): + """Main UI rendering loop.""" + import os + import sys + + if os.name == 'nt': + self._ui_loop_windows() + else: + self._ui_loop_unix() + + def _ui_loop_unix(self): + """UI loop for Unix/macOS using termios.""" + import sys + import select + import termios + import tty + + # Setup terminal for non-blocking input + old_settings = termios.tcgetattr(sys.stdin) + try: + tty.setcbreak(sys.stdin.fileno()) + + # Use Live display to render the UI + with Live(self.layout, refresh_per_second=1/self.refresh_rate, screen=True) as live: + self.live = live # Store the live display for updates + + # Main UI loop + while not self.stop_event.is_set(): + self._update_display() + + # Check for key press (non-blocking) + if select.select([sys.stdin], [], [], 0)[0]: + key = sys.stdin.read(1) + # Check for 'q' to quit + if key == 'q': + # Signal stop but don't call monitor.stop() from UI thread + # as it would cause the thread to try to join itself + self.stop_event.set() + self.monitor.is_running = False + break + + time.sleep(self.refresh_rate) + + # Just check if the monitor was stopped + if not self.monitor.is_running: + break + finally: + # Restore terminal settings + termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings) + + def _ui_loop_windows(self): + """UI loop for Windows using msvcrt.""" + import msvcrt + + with Live(self.layout, refresh_per_second=1/self.refresh_rate, screen=True) as live: + self.live = live + + while not self.stop_event.is_set(): + self._update_display() + + if msvcrt.kbhit(): + key = msvcrt.getch().decode("utf-8", errors="ignore") + if key == 'q': + self.stop_event.set() + self.monitor.is_running = False + break + + time.sleep(self.refresh_rate) + + if not self.monitor.is_running: + break + + def _update_display(self): + """Update the terminal display with current statistics.""" + if not self.monitor: + return + + # Update crawler status panel + self.layout["header"].update(self._create_status_panel()) + + # Update pipeline status panel and task details panel + self.layout["pipeline_status"].update(self._create_pipeline_panel()) + self.layout["task_details"].update(self._create_task_details_panel()) + + # Update footer + self.layout["footer"].update(self._create_footer()) + + def _create_status_panel(self) -> Panel: + """Create the crawler status panel.""" + summary = self.monitor.get_summary() + + # Format memory status with icon + memory_status = self.monitor.get_memory_status() + memory_icon = "🟢" # Default NORMAL + if memory_status == "PRESSURE": + memory_icon = "🟠" + elif memory_status == "CRITICAL": + memory_icon = "🔴" + + # Get current memory usage + current_memory = psutil.Process().memory_info().rss / (1024 * 1024) # MB + memory_percent = (current_memory / psutil.virtual_memory().total) * 100 + + # Format runtime + runtime = self.monitor._format_time(time.time() - self.monitor.start_time if self.monitor.start_time else 0) + + # Create the status text + status_text = Text() + status_text.append(f"Web Crawler Dashboard | Runtime: {runtime} | Memory: {memory_percent:.1f}% {memory_icon}\n") + status_text.append(f"Status: {memory_status} | URLs: {summary['urls_completed']}/{summary['urls_total']} | ") + status_text.append(f"Peak Mem: {summary['peak_memory_percent']:.1f}% at {self.monitor._format_time(summary['peak_memory_time'])}") + + return Panel(status_text, title="Crawler Status", border_style="blue") + + def _create_pipeline_panel(self) -> Panel: + """Create the pipeline status panel.""" + summary = self.monitor.get_summary() + queue_stats = self.monitor.get_queue_stats() + + # Create a table for status counts + table = Table(show_header=True, box=None) + table.add_column("Status", style="cyan") + table.add_column("Count", justify="right") + table.add_column("Percentage", justify="right") + table.add_column("Stat", style="cyan") + table.add_column("Value", justify="right") + + # Calculate overall progress + progress = f"{summary['urls_completed']}/{summary['urls_total']}" + progress_percent = f"{summary['completion_percentage']:.1f}%" + + # Add rows for each status + table.add_row( + "Overall Progress", + progress, + progress_percent, + "Est. Completion", + summary.get('estimated_completion_time', "N/A") + ) + + # Add rows for each status + status_counts = summary['status_counts'] + total = summary['urls_total'] or 1 # Avoid division by zero + + # Status rows + table.add_row( + "Completed", + str(status_counts.get(CrawlStatus.COMPLETED.name, 0)), + f"{status_counts.get(CrawlStatus.COMPLETED.name, 0) / total * 100:.1f}%", + "Avg. Time/URL", + f"{summary.get('avg_task_duration', 0):.2f}s" + ) + + table.add_row( + "Failed", + str(status_counts.get(CrawlStatus.FAILED.name, 0)), + f"{status_counts.get(CrawlStatus.FAILED.name, 0) / total * 100:.1f}%", + "Concurrent Tasks", + str(status_counts.get(CrawlStatus.IN_PROGRESS.name, 0)) + ) + + table.add_row( + "In Progress", + str(status_counts.get(CrawlStatus.IN_PROGRESS.name, 0)), + f"{status_counts.get(CrawlStatus.IN_PROGRESS.name, 0) / total * 100:.1f}%", + "Queue Size", + str(queue_stats['total_queued']) + ) + + table.add_row( + "Queued", + str(status_counts.get(CrawlStatus.QUEUED.name, 0)), + f"{status_counts.get(CrawlStatus.QUEUED.name, 0) / total * 100:.1f}%", + "Max Wait Time", + f"{queue_stats['highest_wait_time']:.1f}s" + ) + + # Requeued is a special case as it's not a status + requeued_count = summary.get('requeued_count', 0) + table.add_row( + "Requeued", + str(requeued_count), + f"{summary.get('requeue_rate', 0):.1f}%", + "Avg Wait Time", + f"{queue_stats['avg_wait_time']:.1f}s" + ) + + # Add empty row for spacing + table.add_row( + "", + "", + "", + "Requeue Rate", + f"{summary.get('requeue_rate', 0):.1f}%" + ) + + return Panel(table, title="Pipeline Status", border_style="green") + + def _create_task_details_panel(self) -> Panel: + """Create the task details panel.""" + # Create a table for task details + table = Table(show_header=True, expand=True) + table.add_column("Task ID", style="cyan", no_wrap=True, width=10) + table.add_column("URL", style="blue", ratio=3) + table.add_column("Status", style="green", width=15) + table.add_column("Memory", justify="right", width=8) + table.add_column("Peak", justify="right", width=8) + table.add_column("Duration", justify="right", width=10) + + # Get all task stats + task_stats = self.monitor.get_all_task_stats() + + # Add summary row + active_tasks = sum(1 for stats in task_stats.values() + if stats['status'] == CrawlStatus.IN_PROGRESS.name) + + total_memory = sum(stats['memory_usage'] for stats in task_stats.values()) + total_peak = sum(stats['peak_memory'] for stats in task_stats.values()) + + # Summary row with separators + table.add_row( + "SUMMARY", + f"Total: {len(task_stats)}", + f"Active: {active_tasks}", + f"{total_memory:.1f}", + f"{total_peak:.1f}", + "N/A" + ) + + # Add a separator + table.add_row("—" * 10, "—" * 20, "—" * 10, "—" * 8, "—" * 8, "—" * 10) + + # Status icons + status_icons = { + CrawlStatus.QUEUED.name: "⏳", + CrawlStatus.IN_PROGRESS.name: "🔄", + CrawlStatus.COMPLETED.name: "✅", + CrawlStatus.FAILED.name: "❌" + } + + # Calculate how many rows we can display based on available space + # We can display more rows now that we have a dedicated panel + display_count = min(len(task_stats), 20) # Display up to 20 tasks + + # Add rows for each task + for task_id, stats in sorted( + list(task_stats.items())[:display_count], + # Sort: 1. IN_PROGRESS first, 2. QUEUED, 3. COMPLETED/FAILED by recency + key=lambda x: ( + 0 if x[1]['status'] == CrawlStatus.IN_PROGRESS.name else + 1 if x[1]['status'] == CrawlStatus.QUEUED.name else + 2, + -1 * (x[1].get('end_time', 0) or 0) # Most recent first + ) + ): + # Truncate task_id and URL for display + short_id = task_id[:8] + url = stats['url'] + if len(url) > 50: # Allow longer URLs in the dedicated panel + url = url[:47] + "..." + + # Format status with icon + status = f"{status_icons.get(stats['status'], '?')} {stats['status']}" + + # Add row + table.add_row( + short_id, + url, + status, + f"{stats['memory_usage']:.1f}", + f"{stats['peak_memory']:.1f}", + stats['duration'] if 'duration' in stats else "0:00" + ) + + return Panel(table, title="Task Details", border_style="yellow") + + def _create_footer(self) -> Panel: + """Create the footer panel.""" + from rich.columns import Columns + from rich.align import Align + + memory_status = self.monitor.get_memory_status() + memory_icon = "🟢" # Default NORMAL + if memory_status == "PRESSURE": + memory_icon = "🟠" + elif memory_status == "CRITICAL": + memory_icon = "🔴" + + # Left section - memory status + left_text = Text() + left_text.append("Memory Status: ", style="bold") + status_style = "green" if memory_status == "NORMAL" else "yellow" if memory_status == "PRESSURE" else "red bold" + left_text.append(f"{memory_icon} {memory_status}", style=status_style) + + # Center section - copyright + center_text = Text("© Crawl4AI 2025 | Made by UnclecCode", style="cyan italic") + + # Right section - quit instruction + right_text = Text() + right_text.append("Press ", style="bold") + right_text.append("q", style="white on blue") + right_text.append(" to quit", style="bold") + + # Create columns with the three sections + footer_content = Columns( + [ + Align.left(left_text), + Align.center(center_text), + Align.right(right_text) + ], + expand=True + ) + + # Create a more visible footer panel + return Panel( + footer_content, + border_style="white", + padding=(0, 1) # Add padding for better visibility + ) + + +class CrawlerMonitor: + """ + Comprehensive monitoring and visualization system for tracking web crawler operations in real-time. + Provides a terminal-based dashboard that displays task statuses, memory usage, queue statistics, + and performance metrics. + """ + + def __init__( + self, + urls_total: int = 0, + refresh_rate: float = 1.0, + enable_ui: bool = True, + max_width: int = 120 + ): + """ + Initialize the CrawlerMonitor. + + Args: + urls_total: Total number of URLs to be crawled + refresh_rate: How often to refresh the UI (in seconds) + enable_ui: Whether to display the terminal UI + max_width: Maximum width of the UI in characters + """ + # Core monitoring attributes + self.stats = {} # Task ID -> stats dict + self.memory_status = "NORMAL" + self.start_time = None + self.end_time = None + self.is_running = False + self.queue_stats = { + "total_queued": 0, + "highest_wait_time": 0.0, + "avg_wait_time": 0.0 + } + self.urls_total = urls_total + self.urls_completed = 0 + self.peak_memory_percent = 0.0 + self.peak_memory_time = 0.0 + + # Status counts + self.status_counts = { + CrawlStatus.QUEUED.name: 0, + CrawlStatus.IN_PROGRESS.name: 0, + CrawlStatus.COMPLETED.name: 0, + CrawlStatus.FAILED.name: 0 + } + + # Requeue tracking + self.requeued_count = 0 + + # Thread-safety + self._lock = threading.RLock() + + # Terminal UI + self.enable_ui = enable_ui + self.terminal_ui = TerminalUI( + refresh_rate=refresh_rate, + max_width=max_width + ) if enable_ui else None + + def start(self): + """ + Start the monitoring session. + + - Initializes the start_time + - Sets is_running to True + - Starts the terminal UI if enabled + """ + with self._lock: + self.start_time = time.time() + self.is_running = True + + # Start the terminal UI + if self.enable_ui and self.terminal_ui: + self.terminal_ui.start(self) + + def stop(self): + """ + Stop the monitoring session. + + - Records end_time + - Sets is_running to False + - Stops the terminal UI + - Generates final summary statistics + """ + with self._lock: + self.end_time = time.time() + self.is_running = False + + # Stop the terminal UI + if self.enable_ui and self.terminal_ui: + self.terminal_ui.stop() + + def add_task(self, task_id: str, url: str): + """ + Register a new task with the monitor. + + Args: + task_id: Unique identifier for the task + url: URL being crawled + + The task is initialized with: + - status: QUEUED + - url: The URL to crawl + - enqueue_time: Current time + - memory_usage: 0 + - peak_memory: 0 + - wait_time: 0 + - retry_count: 0 + """ + with self._lock: + self.stats[task_id] = { + "task_id": task_id, + "url": url, + "status": CrawlStatus.QUEUED.name, + "enqueue_time": time.time(), + "start_time": None, + "end_time": None, + "memory_usage": 0.0, + "peak_memory": 0.0, + "error_message": "", + "wait_time": 0.0, + "retry_count": 0, + "duration": "0:00", + "counted_requeue": False + } + + # Update status counts + self.status_counts[CrawlStatus.QUEUED.name] += 1 + + def update_task( + self, + task_id: str, + status: Optional[CrawlStatus] = None, + start_time: Optional[float] = None, + end_time: Optional[float] = None, + memory_usage: Optional[float] = None, + peak_memory: Optional[float] = None, + error_message: Optional[str] = None, + retry_count: Optional[int] = None, + wait_time: Optional[float] = None + ): + """ + Update statistics for a specific task. + + Args: + task_id: Unique identifier for the task + status: New status (QUEUED, IN_PROGRESS, COMPLETED, FAILED) + start_time: When task execution started + end_time: When task execution ended + memory_usage: Current memory usage in MB + peak_memory: Maximum memory usage in MB + error_message: Error description if failed + retry_count: Number of retry attempts + wait_time: Time spent in queue + + Updates task statistics and updates status counts. + If status changes, decrements old status count and + increments new status count. + """ + with self._lock: + # Check if task exists + if task_id not in self.stats: + return + + task_stats = self.stats[task_id] + + # Update status counts if status is changing + old_status = task_stats["status"] + if status and status.name != old_status: + self.status_counts[old_status] -= 1 + self.status_counts[status.name] += 1 + + # Track completion + if status == CrawlStatus.COMPLETED: + self.urls_completed += 1 + + # Track requeues + if old_status in [CrawlStatus.COMPLETED.name, CrawlStatus.FAILED.name] and not task_stats.get("counted_requeue", False): + self.requeued_count += 1 + task_stats["counted_requeue"] = True + + # Update task statistics + if status: + task_stats["status"] = status.name + if start_time is not None: + task_stats["start_time"] = start_time + if end_time is not None: + task_stats["end_time"] = end_time + if memory_usage is not None: + task_stats["memory_usage"] = memory_usage + + # Update peak memory if necessary + current_percent = (memory_usage / psutil.virtual_memory().total) * 100 + if current_percent > self.peak_memory_percent: + self.peak_memory_percent = current_percent + self.peak_memory_time = time.time() + + if peak_memory is not None: + task_stats["peak_memory"] = peak_memory + if error_message is not None: + task_stats["error_message"] = error_message + if retry_count is not None: + task_stats["retry_count"] = retry_count + if wait_time is not None: + task_stats["wait_time"] = wait_time + + # Calculate duration + if task_stats["start_time"]: + end = task_stats["end_time"] or time.time() + duration = end - task_stats["start_time"] + task_stats["duration"] = self._format_time(duration) + + def update_memory_status(self, status: str): + """ + Update the current memory status. + + Args: + status: Memory status (NORMAL, PRESSURE, CRITICAL, or custom) + + Also updates the UI to reflect the new status. + """ + with self._lock: + self.memory_status = status + + def update_queue_statistics( + self, + total_queued: int, + highest_wait_time: float, + avg_wait_time: float + ): + """ + Update statistics related to the task queue. + + Args: + total_queued: Number of tasks currently in queue + highest_wait_time: Longest wait time of any queued task + avg_wait_time: Average wait time across all queued tasks + """ + with self._lock: + self.queue_stats = { + "total_queued": total_queued, + "highest_wait_time": highest_wait_time, + "avg_wait_time": avg_wait_time + } + + def get_task_stats(self, task_id: str) -> Dict: + """ + Get statistics for a specific task. + + Args: + task_id: Unique identifier for the task + + Returns: + Dictionary containing all task statistics + """ + with self._lock: + return self.stats.get(task_id, {}).copy() + + def get_all_task_stats(self) -> Dict[str, Dict]: + """ + Get statistics for all tasks. + + Returns: + Dictionary mapping task_ids to their statistics + """ + with self._lock: + return self.stats.copy() + + def get_memory_status(self) -> str: + """ + Get the current memory status. + + Returns: + Current memory status string + """ + with self._lock: + return self.memory_status + + def get_queue_stats(self) -> Dict: + """ + Get current queue statistics. + + Returns: + Dictionary with queue statistics including: + - total_queued: Number of tasks in queue + - highest_wait_time: Longest wait time + - avg_wait_time: Average wait time + """ + with self._lock: + return self.queue_stats.copy() + + def get_summary(self) -> Dict: + """ + Get a summary of all crawler statistics. + + Returns: + Dictionary containing: + - runtime: Total runtime in seconds + - urls_total: Total URLs to process + - urls_completed: Number of completed URLs + - completion_percentage: Percentage complete + - status_counts: Count of tasks in each status + - memory_status: Current memory status + - peak_memory_percent: Highest memory usage + - peak_memory_time: When peak memory occurred + - avg_task_duration: Average task processing time + - estimated_completion_time: Projected finish time + - requeue_rate: Percentage of tasks requeued + """ + with self._lock: + # Calculate runtime + current_time = time.time() + runtime = current_time - (self.start_time or current_time) + + # Calculate completion percentage + completion_percentage = 0 + if self.urls_total > 0: + completion_percentage = (self.urls_completed / self.urls_total) * 100 + + # Calculate average task duration for completed tasks + completed_tasks = [ + task for task in self.stats.values() + if task["status"] == CrawlStatus.COMPLETED.name and task.get("start_time") and task.get("end_time") + ] + + avg_task_duration = 0 + if completed_tasks: + total_duration = sum(task["end_time"] - task["start_time"] for task in completed_tasks) + avg_task_duration = total_duration / len(completed_tasks) + + # Calculate requeue rate + requeue_rate = 0 + if len(self.stats) > 0: + requeue_rate = (self.requeued_count / len(self.stats)) * 100 + + # Calculate estimated completion time + estimated_completion_time = "N/A" + if avg_task_duration > 0 and self.urls_total > 0 and self.urls_completed > 0: + remaining_tasks = self.urls_total - self.urls_completed + estimated_seconds = remaining_tasks * avg_task_duration + estimated_completion_time = self._format_time(estimated_seconds) + + return { + "runtime": runtime, + "urls_total": self.urls_total, + "urls_completed": self.urls_completed, + "completion_percentage": completion_percentage, + "status_counts": self.status_counts.copy(), + "memory_status": self.memory_status, + "peak_memory_percent": self.peak_memory_percent, + "peak_memory_time": self.peak_memory_time, + "avg_task_duration": avg_task_duration, + "estimated_completion_time": estimated_completion_time, + "requeue_rate": requeue_rate, + "requeued_count": self.requeued_count + } + + def render(self): + """ + Render the terminal UI. + + This is the main UI rendering loop that: + 1. Updates all statistics + 2. Formats the display + 3. Renders the ASCII interface + 4. Handles keyboard input + + Note: The actual rendering is handled by the TerminalUI class + which uses the rich library's Live display. + """ + if self.enable_ui and self.terminal_ui: + # Force an update of the UI + if hasattr(self.terminal_ui, '_update_display'): + self.terminal_ui._update_display() + + def _format_time(self, seconds: float) -> str: + """ + Format time in hours:minutes:seconds. + + Args: + seconds: Time in seconds + + Returns: + Formatted time string (e.g., "1:23:45") + """ + delta = timedelta(seconds=int(seconds)) + hours, remainder = divmod(delta.seconds, 3600) + minutes, seconds = divmod(remainder, 60) + + if hours > 0: + return f"{hours}:{minutes:02}:{seconds:02}" + else: + return f"{minutes}:{seconds:02}" + + def _calculate_estimated_completion(self) -> str: + """ + Calculate estimated completion time based on current progress. + + Returns: + Formatted time string + """ + summary = self.get_summary() + return summary.get("estimated_completion_time", "N/A") + + +# Example code for testing +if __name__ == "__main__": + # Initialize the monitor + monitor = CrawlerMonitor(urls_total=100) + + # Start monitoring + monitor.start() + + try: + # Simulate some tasks + for i in range(20): + task_id = str(uuid.uuid4()) + url = f"https://example.com/page{i}" + monitor.add_task(task_id, url) + + # Simulate 20% of tasks are already running + if i < 4: + monitor.update_task( + task_id=task_id, + status=CrawlStatus.IN_PROGRESS, + start_time=time.time() - 30, # Started 30 seconds ago + memory_usage=10.5 + ) + + # Simulate 10% of tasks are completed + if i >= 4 and i < 6: + start_time = time.time() - 60 + end_time = time.time() - 15 + monitor.update_task( + task_id=task_id, + status=CrawlStatus.IN_PROGRESS, + start_time=start_time, + memory_usage=8.2 + ) + monitor.update_task( + task_id=task_id, + status=CrawlStatus.COMPLETED, + end_time=end_time, + memory_usage=0, + peak_memory=15.7 + ) + + # Simulate 5% of tasks fail + if i >= 6 and i < 7: + start_time = time.time() - 45 + end_time = time.time() - 20 + monitor.update_task( + task_id=task_id, + status=CrawlStatus.IN_PROGRESS, + start_time=start_time, + memory_usage=12.3 + ) + monitor.update_task( + task_id=task_id, + status=CrawlStatus.FAILED, + end_time=end_time, + memory_usage=0, + peak_memory=18.2, + error_message="Connection timeout" + ) + + # Simulate memory pressure + monitor.update_memory_status("PRESSURE") + + # Simulate queue statistics + monitor.update_queue_statistics( + total_queued=16, # 20 - 4 (in progress) + highest_wait_time=120.5, + avg_wait_time=60.2 + ) + + # Keep the monitor running for a demonstration + print("Crawler Monitor is running. Press 'q' to exit.") + while monitor.is_running: + time.sleep(0.1) + + except KeyboardInterrupt: + print("\nExiting crawler monitor...") + finally: + # Stop the monitor + monitor.stop() + print("Crawler monitor exited successfully.") \ No newline at end of file diff --git a/crawl4ai/config.py b/crawl4ai/config.py new file mode 100644 index 0000000..5d39413 --- /dev/null +++ b/crawl4ai/config.py @@ -0,0 +1,155 @@ +import os +from dotenv import load_dotenv + +load_dotenv() # Load environment variables from .env file + +# Default provider, ONLY used when the extraction strategy is LLMExtractionStrategy +DEFAULT_PROVIDER = "openai/gpt-4o" +DEFAULT_PROVIDER_API_KEY = "OPENAI_API_KEY" +MODEL_REPO_BRANCH = "new-release-0.0.2" +# Provider-model dictionary, ONLY used when the extraction strategy is LLMExtractionStrategy +PROVIDER_MODELS = { + "ollama/llama3": "no-token-needed", # Any model from Ollama no need for API token + "groq/llama3-70b-8192": os.getenv("GROQ_API_KEY"), + "groq/llama3-8b-8192": os.getenv("GROQ_API_KEY"), + "openai/gpt-4o-mini": os.getenv("OPENAI_API_KEY"), + "openai/gpt-4o": os.getenv("OPENAI_API_KEY"), + "openai/o1-mini": os.getenv("OPENAI_API_KEY"), + "openai/o1-preview": os.getenv("OPENAI_API_KEY"), + "openai/o3-mini": os.getenv("OPENAI_API_KEY"), + "openai/o3-mini-high": os.getenv("OPENAI_API_KEY"), + "anthropic/claude-3-haiku-20240307": os.getenv("ANTHROPIC_API_KEY"), + "anthropic/claude-3-opus-20240229": os.getenv("ANTHROPIC_API_KEY"), + "anthropic/claude-3-sonnet-20240229": os.getenv("ANTHROPIC_API_KEY"), + "anthropic/claude-3-5-sonnet-20240620": os.getenv("ANTHROPIC_API_KEY"), + "gemini/gemini-pro": os.getenv("GEMINI_API_KEY"), + 'gemini/gemini-1.5-pro': os.getenv("GEMINI_API_KEY"), + 'gemini/gemini-2.0-flash': os.getenv("GEMINI_API_KEY"), + 'gemini/gemini-2.0-flash-exp': os.getenv("GEMINI_API_KEY"), + 'gemini/gemini-2.0-flash-lite-preview-02-05': os.getenv("GEMINI_API_KEY"), + "deepseek/deepseek-chat": os.getenv("DEEPSEEK_API_KEY"), +} +PROVIDER_MODELS_PREFIXES = { + "ollama": "no-token-needed", # Any model from Ollama no need for API token + "groq": os.getenv("GROQ_API_KEY"), + "openai": os.getenv("OPENAI_API_KEY"), + "anthropic": os.getenv("ANTHROPIC_API_KEY"), + "gemini": os.getenv("GEMINI_API_KEY"), + "deepseek": os.getenv("DEEPSEEK_API_KEY"), + "bedrock": None, # Bedrock uses AWS credential chain (SigV4) or explicit api_token for bearer auth +} + +# Chunk token threshold +CHUNK_TOKEN_THRESHOLD = 2**11 # 2048 tokens +OVERLAP_RATE = 0.1 +WORD_TOKEN_RATE = 1.3 + +# Threshold for the minimum number of word in a HTML tag to be considered +MIN_WORD_THRESHOLD = 1 +IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD = 1 + +IMPORTANT_ATTRS = ["src", "href", "alt", "title", "width", "height", "class", "id", "rowspan", "colspan"] +ONLY_TEXT_ELIGIBLE_TAGS = [ + "b", + "i", + "u", + "span", + "del", + "ins", + "sub", + "sup", + "strong", + "em", + "code", + "kbd", + "var", + "s", + "q", + "abbr", + "cite", + "dfn", + "time", + "small", + "mark", +] +SOCIAL_MEDIA_DOMAINS = [ + "facebook.com", + "twitter.com", + "x.com", + "linkedin.com", + "instagram.com", + "pinterest.com", + "tiktok.com", + "snapchat.com", + "reddit.com", +] + +# Threshold for the Image extraction - Range is 1 to 6 +# Images are scored based on point based system, to filter based on usefulness. Points are assigned +# to each image based on the following aspects. +# If either height or width exceeds 150px +# If image size is greater than 10Kb +# If alt property is set +# If image format is in jpg, png or webp +# If image is in the first half of the total images extracted from the page +IMAGE_SCORE_THRESHOLD = 2 + +MAX_METRICS_HISTORY = 1000 + +NEED_MIGRATION = True +URL_LOG_SHORTEN_LENGTH = 30 +SHOW_DEPRECATION_WARNINGS = True +SCREENSHOT_HEIGHT_TRESHOLD = 10000 +PAGE_TIMEOUT = 60000 +DOWNLOAD_PAGE_TIMEOUT = 60000 + +# Delimiter for concatenating multiple HTML examples in schema generation +HTML_EXAMPLE_DELIMITER = "=== HTML EXAMPLE {index} ===" + +# Global user settings with descriptions and default values +USER_SETTINGS = { + "DEFAULT_LLM_PROVIDER": { + "default": "openai/gpt-4o", + "description": "Default LLM provider in 'company/model' format (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')", + "type": "string" + }, + "DEFAULT_LLM_PROVIDER_TOKEN": { + "default": "", + "description": "API token for the default LLM provider", + "type": "string", + "secret": True + }, + "VERBOSE": { + "default": False, + "description": "Enable verbose output for all commands", + "type": "boolean" + }, + "BROWSER_HEADLESS": { + "default": True, + "description": "Run browser in headless mode by default", + "type": "boolean" + }, + "BROWSER_TYPE": { + "default": "chromium", + "description": "Default browser type (chromium or firefox)", + "type": "string", + "options": ["chromium", "firefox"] + }, + "CACHE_MODE": { + "default": "bypass", + "description": "Default cache mode (bypass, use, or refresh)", + "type": "string", + "options": ["bypass", "use", "refresh"] + }, + "USER_AGENT_MODE": { + "default": "default", + "description": "Default user agent mode (default, random, or mobile)", + "type": "string", + "options": ["default", "random", "mobile"] + }, + "JSON_ENSURE_ASCII": { + "default": True, + "description": "Whether to escape non-ASCII characters in JSON output (False preserves Unicode like 'š', True escapes as '\\u0161')", + "type": "boolean" + } +} diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py new file mode 100644 index 0000000..ab99a79 --- /dev/null +++ b/crawl4ai/content_filter_strategy.py @@ -0,0 +1,1110 @@ +import inspect +import re +import time +from bs4 import BeautifulSoup, Tag +from typing import List, Tuple, Dict, Optional +from rank_bm25 import BM25Okapi +from collections import deque +from bs4 import NavigableString, Comment + +from .utils import ( + clean_tokens, + perform_completion_with_backoff, + escape_json_string, + sanitize_html, + get_home_folder, + extract_xml_data, + merge_chunks, +) +from .types import LLMConfig +from .config import DEFAULT_PROVIDER, OVERLAP_RATE, WORD_TOKEN_RATE +from abc import ABC, abstractmethod +import math +from snowballstemmer import stemmer +from .models import TokenUsage +from .prompts import PROMPT_FILTER_CONTENT +import json +import hashlib +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor +from .async_logger import AsyncLogger, LogLevel, LogColor + + +class RelevantContentFilter(ABC): + """Abstract base class for content filtering strategies""" + + def __init__( + self, + user_query: str = None, + verbose: bool = False, + logger: Optional[AsyncLogger] = None, + ): + """ + Initializes the RelevantContentFilter class with optional user query. + + Args: + user_query (str): User query for filtering (optional). + verbose (bool): Enable verbose logging (default: False). + """ + self.user_query = user_query + self.included_tags = { + # Primary structure + "article", + "main", + "section", + "div", + # List structures + "ul", + "ol", + "li", + "dl", + "dt", + "dd", + # Text content + "p", + "span", + "blockquote", + "pre", + "code", + # Headers + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + # Tables + "table", + "thead", + "tbody", + "tr", + "td", + "th", + # Other semantic elements + "figure", + "figcaption", + "details", + "summary", + # Text formatting + "em", + "strong", + "b", + "i", + "mark", + "small", + # Rich content + "time", + "address", + "cite", + "q", + } + self.excluded_tags = { + "nav", + "footer", + "header", + "aside", + "script", + "style", + "form", + "iframe", + "noscript", + } + self.header_tags = {"h1", "h2", "h3", "h4", "h5", "h6"} + self.negative_patterns = re.compile( + r"nav|footer|header|sidebar|ads|comment|promo|advert|social|share", re.I + ) + self.min_word_count = 2 + self.verbose = False + self.logger = logger + + @abstractmethod + def filter_content(self, html: str) -> List[str]: + """Abstract method to be implemented by specific filtering strategies""" + pass + + def extract_page_query(self, soup: BeautifulSoup, body: Tag) -> str: + """Common method to extract page metadata with fallbacks""" + if self.user_query: + return self.user_query + + query_parts = [] + + # Title + try: + title = soup.title.string + if title: + query_parts.append(title) + except Exception: + pass + + if soup.find("h1"): + query_parts.append(soup.find("h1").get_text()) + + # Meta tags + temp = "" + for meta_name in ["keywords", "description"]: + meta = soup.find("meta", attrs={"name": meta_name}) + if meta and meta.get("content"): + query_parts.append(meta["content"]) + temp += meta["content"] + + # If still empty, grab first significant paragraph + if not temp: + # Find the first tag P thatits text contains more than 50 characters + for p in body.find_all("p"): + if len(p.get_text()) > 150: + query_parts.append(p.get_text()[:150]) + break + + return " ".join(filter(None, query_parts)) + + def extract_text_chunks( + self, body: Tag, min_word_threshold: int = None + ) -> List[Tuple[str, str]]: + """ + Extracts text chunks from a BeautifulSoup body element while preserving order. + Returns list of tuples (text, tag_name) for classification. + + Args: + body: BeautifulSoup Tag object representing the body element + + Returns: + List of (text, tag_name) tuples + """ + # Tags to ignore - inline elements that shouldn't break text flow + INLINE_TAGS = { + "a", + "abbr", + "acronym", + "b", + "bdo", + "big", + "br", + "button", + "cite", + "code", + "dfn", + "em", + "i", + "img", + "input", + "kbd", + "label", + "map", + "object", + "q", + "samp", + "script", + "select", + "small", + "span", + "strong", + "sub", + "sup", + "textarea", + "time", + "tt", + "var", + } + + # Tags that typically contain meaningful headers + HEADER_TAGS = {"h1", "h2", "h3", "h4", "h5", "h6", "header"} + + chunks = [] + current_text = [] + chunk_index = 0 + + def should_break_chunk(tag: Tag) -> bool: + """Determine if a tag should cause a break in the current text chunk""" + return tag.name not in INLINE_TAGS and not ( + tag.name == "p" and len(current_text) == 0 + ) + + # Use deque for efficient push/pop operations + stack = deque([(body, False)]) + + while stack: + element, visited = stack.pop() + + if visited: + # End of block element - flush accumulated text + if current_text and should_break_chunk(element): + text = " ".join("".join(current_text).split()) + if text: + tag_type = ( + "header" if element.name in HEADER_TAGS else "content" + ) + chunks.append((chunk_index, text, tag_type, element)) + chunk_index += 1 + current_text = [] + continue + + if isinstance(element, NavigableString): + if str(element).strip(): + current_text.append(str(element).strip()) + continue + + # Pre-allocate children to avoid multiple list operations + children = list(element.children) + if not children: + continue + + # Mark block for revisit after processing children + stack.append((element, True)) + + # Add children in reverse order for correct processing + for child in reversed(children): + if isinstance(child, (Tag, NavigableString)): + stack.append((child, False)) + + # Handle any remaining text + if current_text: + text = " ".join("".join(current_text).split()) + if text: + chunks.append((chunk_index, text, "content", body)) + + if min_word_threshold: + chunks = [ + chunk for chunk in chunks if len(chunk[1].split()) >= min_word_threshold + ] + + return chunks + + def _deprecated_extract_text_chunks( + self, soup: BeautifulSoup + ) -> List[Tuple[int, str, Tag]]: + """Common method for extracting text chunks""" + _text_cache = {} + + def fast_text(element: Tag) -> str: + elem_id = id(element) + if elem_id in _text_cache: + return _text_cache[elem_id] + texts = [] + for content in element.contents: + if isinstance(content, str): + text = content.strip() + if text: + texts.append(text) + result = " ".join(texts) + _text_cache[elem_id] = result + return result + + candidates = [] + index = 0 + + def dfs(element): + nonlocal index + if isinstance(element, Tag): + if element.name in self.included_tags: + if not self.is_excluded(element): + text = fast_text(element) + word_count = len(text.split()) + + # Headers pass through with adjusted minimum + if element.name in self.header_tags: + if word_count >= 3: # Minimal sanity check for headers + candidates.append((index, text, element)) + index += 1 + # Regular content uses standard minimum + elif word_count >= self.min_word_count: + candidates.append((index, text, element)) + index += 1 + + for child in element.children: + dfs(child) + + dfs(soup.body if soup.body else soup) + return candidates + + def is_excluded(self, tag: Tag) -> bool: + """Common method for exclusion logic""" + if tag.name in self.excluded_tags: + return True + class_id = " ".join( + filter(None, [" ".join(tag.get("class", [])), tag.get("id", "")]) + ) + return bool(self.negative_patterns.search(class_id)) + + def clean_element(self, tag: Tag) -> str: + """Common method for cleaning HTML elements with minimal overhead""" + if not tag or not isinstance(tag, Tag): + return "" + + unwanted_tags = {"script", "style", "aside", "form", "iframe", "noscript"} + unwanted_attrs = { + "style", + "onclick", + "onmouseover", + "align", + "bgcolor", + "class", + "id", + } + + # Use string builder pattern for better performance + builder = [] + + def render_tag(elem): + if not isinstance(elem, Tag): + if isinstance(elem, str): + builder.append(elem.strip()) + return + + if elem.name in unwanted_tags: + return + + # Start tag + builder.append(f"<{elem.name}") + + # Add cleaned attributes + attrs = {k: v for k, v in elem.attrs.items() if k not in unwanted_attrs} + for key, value in attrs.items(): + builder.append(f' {key}="{value}"') + + builder.append(">") + + # Process children + for child in elem.children: + render_tag(child) + + # Close tag + builder.append(f"") + + try: + render_tag(tag) + return "".join(builder) + except Exception: + return str(tag) # Fallback to original if anything fails + + +class BM25ContentFilter(RelevantContentFilter): + """ + Content filtering using BM25 algorithm with priority tag handling. + + How it works: + 1. Extracts page metadata with fallbacks. + 2. Extracts text chunks from the body element. + 3. Tokenizes the corpus and query. + 4. Applies BM25 algorithm to calculate scores for each chunk. + 5. Filters out chunks below the threshold. + 6. Sorts chunks by score in descending order. + 7. Returns the top N chunks. + + Attributes: + user_query (str): User query for filtering (optional). + bm25_threshold (float): BM25 threshold for filtering (default: 1.0). + language (str): Language for stemming (default: 'english'). + + Methods: + filter_content(self, html: str, min_word_threshold: int = None) + """ + + def __init__( + self, + user_query: str = None, + bm25_threshold: float = 1.0, + language: str = "english", + use_stemming: bool = True, + ): + """ + Initializes the BM25ContentFilter class, if not provided, falls back to page metadata. + + Note: + If no query is given and no page metadata is available, then it tries to pick up the first significant paragraph. + + Args: + user_query (str): User query for filtering (optional). + bm25_threshold (float): BM25 threshold for filtering (default: 1.0). + language (str): Language for stemming (default: 'english'). + use_stemming (bool): Whether to apply stemming (default: True). + """ + super().__init__(user_query=user_query) + self.bm25_threshold = bm25_threshold + self.use_stemming = use_stemming + self.priority_tags = { + "h1": 5.0, + "h2": 4.0, + "h3": 3.0, + "title": 4.0, + "strong": 2.0, + "b": 1.5, + "em": 1.5, + "blockquote": 2.0, + "code": 2.0, + "pre": 1.5, + "th": 1.5, # Table headers + } + self.stemmer = stemmer(language) if use_stemming else None + + def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]: + """ + Implements content filtering using BM25 algorithm with priority tag handling. + + Note: + This method implements the filtering logic for the BM25ContentFilter class. + It takes HTML content as input and returns a list of filtered text chunks. + + Args: + html (str): HTML content to be filtered. + min_word_threshold (int): Minimum word threshold for filtering (optional). + + Returns: + List[str]: List of filtered text chunks. + """ + if not html or not isinstance(html, str): + return [] + + soup = BeautifulSoup(html, "lxml") + + # Check if body is present + if not soup.body: + # Wrap in body tag if missing + soup = BeautifulSoup(f"{html}", "lxml") + body = soup.find("body") + + query = self.extract_page_query(soup, body) + + if not query: + return [] + # return [self.clean_element(soup)] + + candidates = self.extract_text_chunks(body, min_word_threshold) + + if not candidates: + return [] + + # Tokenize corpus + # tokenized_corpus = [chunk.lower().split() for _, chunk, _, _ in candidates] + # tokenized_query = query.lower().split() + + # tokenized_corpus = [[ps.stem(word) for word in chunk.lower().split()] + # for _, chunk, _, _ in candidates] + # tokenized_query = [ps.stem(word) for word in query.lower().split()] + + if self.use_stemming: + tokenized_corpus = [ + [self.stemmer.stemWord(word) for word in chunk.lower().split()] + for _, chunk, _, _ in candidates + ] + tokenized_query = [ + self.stemmer.stemWord(word) for word in query.lower().split() + ] + else: + tokenized_corpus = [ + chunk.lower().split() for _, chunk, _, _ in candidates + ] + tokenized_query = query.lower().split() + + # tokenized_corpus = [[self.stemmer.stemWord(word) for word in tokenize_text(chunk.lower())] + # for _, chunk, _, _ in candidates] + # tokenized_query = [self.stemmer.stemWord(word) for word in tokenize_text(query.lower())] + + # Clean from stop words and noise + tokenized_corpus = [clean_tokens(tokens) for tokens in tokenized_corpus] + tokenized_query = clean_tokens(tokenized_query) + + bm25 = BM25Okapi(tokenized_corpus) + scores = bm25.get_scores(tokenized_query) + + # Adjust scores with tag weights + adjusted_candidates = [] + for score, (index, chunk, tag_type, tag) in zip(scores, candidates): + tag_weight = self.priority_tags.get(tag.name, 1.0) + adjusted_score = score * tag_weight + adjusted_candidates.append((adjusted_score, index, chunk, tag)) + + # Filter candidates by threshold + selected_candidates = [ + (index, chunk, tag) + for adjusted_score, index, chunk, tag in adjusted_candidates + if adjusted_score >= self.bm25_threshold + ] + + if not selected_candidates: + return [] + + # Sort selected candidates by original document order + selected_candidates.sort(key=lambda x: x[0]) + + # Deduplicate by chunk text, keeping first occurrence (lowest index) + seen_texts = set() + unique_candidates = [] + for index, chunk, tag in selected_candidates: + if chunk not in seen_texts: + seen_texts.add(chunk) + unique_candidates.append((index, chunk, tag)) + + return [self.clean_element(tag) for _, _, tag in unique_candidates] + + +class PruningContentFilter(RelevantContentFilter): + """ + Content filtering using pruning algorithm with dynamic threshold. + + How it works: + 1. Extracts page metadata with fallbacks. + 2. Extracts text chunks from the body element. + 3. Applies pruning algorithm to calculate scores for each chunk. + 4. Filters out chunks below the threshold. + 5. Sorts chunks by score in descending order. + 6. Returns the top N chunks. + + Attributes: + user_query (str): User query for filtering (optional), if not provided, falls back to page metadata. + min_word_threshold (int): Minimum word threshold for filtering (optional). + threshold_type (str): Threshold type for dynamic threshold (default: 'fixed'). + threshold (float): Fixed threshold value (default: 0.48). + + Methods: + filter_content(self, html: str, min_word_threshold: int = None): + """ + + def __init__( + self, + user_query: str = None, + min_word_threshold: int = None, + threshold_type: str = "fixed", + threshold: float = 0.48, + preserve_classes: list = None, + preserve_tags: list = None, + ): + """ + Initializes the PruningContentFilter class, if not provided, falls back to page metadata. + + Note: + If no query is given and no page metadata is available, then it tries to pick up the first significant paragraph. + + Args: + user_query (str): User query for filtering (optional). + min_word_threshold (int): Minimum word threshold for filtering (optional). + threshold_type (str): Threshold type for dynamic threshold (default: 'fixed'). + threshold (float): Fixed threshold value (default: 0.48). + preserve_classes (list): CSS class names to always keep regardless of score (optional). + preserve_tags (list): HTML tag names to always keep regardless of score (optional). + """ + super().__init__(None) + self.min_word_threshold = min_word_threshold + self.threshold_type = threshold_type + self.threshold = threshold + self.preserve_classes = set(preserve_classes) if preserve_classes else set() + self.preserve_tags = set(preserve_tags) if preserve_tags else set() + + # Add tag importance for dynamic threshold + self.tag_importance = { + "article": 1.5, + "main": 1.4, + "section": 1.3, + "p": 1.2, + "h1": 1.4, + "h2": 1.3, + "h3": 1.2, + "div": 0.7, + "span": 0.6, + } + + # Metric configuration + self.metric_config = { + "text_density": True, + "link_density": True, + "tag_weight": True, + "class_id_weight": True, + "text_length": True, + } + + self.metric_weights = { + "text_density": 0.4, + "link_density": 0.2, + "tag_weight": 0.2, + "class_id_weight": 0.1, + "text_length": 0.1, + } + + self.tag_weights = { + "div": 0.5, + "p": 1.0, + "article": 1.5, + "section": 1.0, + "span": 0.3, + "li": 0.5, + "ul": 0.5, + "ol": 0.5, + "h1": 1.2, + "h2": 1.1, + "h3": 1.0, + "h4": 0.9, + "h5": 0.8, + "h6": 0.7, + } + + def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]: + """ + Implements content filtering using pruning algorithm with dynamic threshold. + + Note: + This method implements the filtering logic for the PruningContentFilter class. + It takes HTML content as input and returns a list of filtered text chunks. + + Args: + html (str): HTML content to be filtered. + min_word_threshold (int): Minimum word threshold for filtering (optional). + + Returns: + List[str]: List of filtered text chunks. + """ + if not html or not isinstance(html, str): + return [] + + soup = BeautifulSoup(html, "lxml") + if not soup.body: + soup = BeautifulSoup(f"{html}", "lxml") + + # Remove comments and unwanted tags + self._remove_comments(soup) + self._remove_unwanted_tags(soup) + + # Prune tree starting from body + body = soup.find("body") + self._prune_tree(body) + + # Extract remaining content as list of HTML strings + content_blocks = [] + for element in body.children: + if isinstance(element, str) or not hasattr(element, "name"): + continue + if len(element.get_text(strip=True)) > 0: + content_blocks.append(str(element)) + + return content_blocks + + def _remove_comments(self, soup): + """Removes HTML comments""" + for element in soup(string=lambda string: isinstance(string, Comment)): + element.extract() + + def _remove_unwanted_tags(self, soup): + """Removes unwanted tags""" + for tag in self.excluded_tags: + for element in soup.find_all(tag): + element.decompose() + + def _is_preserved(self, node): + """Check if a node matches the preserve whitelist.""" + if self.preserve_tags and node.name in self.preserve_tags: + return True + if self.preserve_classes and "class" in getattr(node, "attrs", {}): + node_classes = set(node["class"]) if isinstance(node["class"], list) else {node["class"]} + if node_classes & self.preserve_classes: + return True + return False + + def _prune_tree(self, node): + """ + Prunes the tree starting from the given node. + + Args: + node (Tag): The node from which the pruning starts. + """ + if not node or not hasattr(node, "name") or node.name is None: + return + + # Skip pruning for preserved nodes — always keep them + if self._is_preserved(node): + return + + text_len = len(node.get_text(strip=True)) + tag_len = len(node.encode_contents().decode("utf-8")) + link_text_len = sum( + len(s.strip()) + for s in (a.string for a in node.find_all("a", recursive=False)) + if s + ) + + metrics = { + "node": node, + "tag_name": node.name, + "text_len": text_len, + "tag_len": tag_len, + "link_text_len": link_text_len, + } + + score = self._compute_composite_score(metrics, text_len, tag_len, link_text_len) + + if self.threshold_type == "fixed": + should_remove = score < self.threshold + else: # dynamic + tag_importance = self.tag_importance.get(node.name, 0.7) + text_ratio = text_len / tag_len if tag_len > 0 else 0 + link_ratio = link_text_len / text_len if text_len > 0 else 1 + + threshold = self.threshold # base threshold + if tag_importance > 1: + threshold *= 0.8 + if text_ratio > 0.4: + threshold *= 0.9 + if link_ratio > 0.6: + threshold *= 1.2 + + should_remove = score < threshold + + if should_remove: + node.decompose() + else: + children = [child for child in node.children if hasattr(child, "name")] + for child in children: + self._prune_tree(child) + + def _compute_composite_score(self, metrics, text_len, tag_len, link_text_len): + """Computes the composite score""" + if self.min_word_threshold: + # Get raw text from metrics node - avoid extra processing + text = metrics["node"].get_text(strip=True) + word_count = text.count(" ") + 1 + if word_count < self.min_word_threshold: + return -1.0 # Guaranteed removal + score = 0.0 + total_weight = 0.0 + + if self.metric_config["text_density"]: + density = text_len / tag_len if tag_len > 0 else 0 + score += self.metric_weights["text_density"] * density + total_weight += self.metric_weights["text_density"] + + if self.metric_config["link_density"]: + density = 1 - (link_text_len / text_len if text_len > 0 else 0) + score += self.metric_weights["link_density"] * density + total_weight += self.metric_weights["link_density"] + + if self.metric_config["tag_weight"]: + tag_score = self.tag_weights.get(metrics["tag_name"], 0.5) + score += self.metric_weights["tag_weight"] * tag_score + total_weight += self.metric_weights["tag_weight"] + + if self.metric_config["class_id_weight"]: + class_score = self._compute_class_id_weight(metrics["node"]) + score += self.metric_weights["class_id_weight"] * max(0, class_score) + total_weight += self.metric_weights["class_id_weight"] + + if self.metric_config["text_length"]: + score += self.metric_weights["text_length"] * math.log(text_len + 1) + total_weight += self.metric_weights["text_length"] + + return score / total_weight if total_weight > 0 else 0 + + def _compute_class_id_weight(self, node): + """Computes the class ID weight""" + class_id_score = 0 + if "class" in node.attrs: + classes = " ".join(node["class"]) + if self.negative_patterns.match(classes): + class_id_score -= 0.5 + if "id" in node.attrs: + element_id = node["id"] + if self.negative_patterns.match(element_id): + class_id_score -= 0.5 + return class_id_score + + +class LLMContentFilter(RelevantContentFilter): + """Content filtering using LLMs to generate relevant markdown. + + How it works: + 1. Extracts page metadata with fallbacks. + 2. Extracts text chunks from the body element. + 3. Applies LLMs to generate markdown for each chunk. + 4. Filters out chunks below the threshold. + 5. Sorts chunks by score in descending order. + 6. Returns the top N chunks. + + Attributes: + llm_config (LLMConfig): LLM configuration object. + instruction (str): Instruction for LLM markdown generation + chunk_token_threshold (int): Chunk token threshold for splitting (default: 1e9). + overlap_rate (float): Overlap rate for chunking (default: 0.5). + word_token_rate (float): Word token rate for chunking (default: 0.2). + verbose (bool): Enable verbose logging (default: False). + logger (AsyncLogger): Custom logger for LLM operations (optional). + """ + _UNWANTED_PROPS = { + 'provider' : 'Instead, use llm_config=LLMConfig(provider="...")', + 'api_token' : 'Instead, use llm_config=LlMConfig(api_token="...")', + 'base_url' : 'Instead, use llm_config=LLMConfig(base_url="...")', + 'api_base' : 'Instead, use llm_config=LLMConfig(base_url="...")', + } + + def __init__( + self, + llm_config: "LLMConfig" = None, + instruction: str = None, + chunk_token_threshold: int = int(1e9), + overlap_rate: float = OVERLAP_RATE, + word_token_rate: float = WORD_TOKEN_RATE, + # char_token_rate: float = WORD_TOKEN_RATE * 5, + # chunk_mode: str = "char", + verbose: bool = False, + logger: Optional[AsyncLogger] = None, + ignore_cache: bool = True, + # Deprecated properties + provider: str = DEFAULT_PROVIDER, + api_token: Optional[str] = None, + base_url: Optional[str] = None, + api_base: Optional[str] = None, + extra_args: Dict = None, + ): + super().__init__(None) + self.provider = provider + self.api_token = api_token + self.base_url = base_url or api_base + self.llm_config = llm_config + self.instruction = instruction + self.chunk_token_threshold = chunk_token_threshold + self.overlap_rate = overlap_rate + self.word_token_rate = word_token_rate or WORD_TOKEN_RATE + # self.chunk_mode: str = chunk_mode + # self.char_token_rate = char_token_rate or word_token_rate / 5 + # self.token_rate = word_token_rate if chunk_mode == "word" else self.char_token_rate + self.token_rate = word_token_rate or WORD_TOKEN_RATE + self.extra_args = extra_args or {} + self.ignore_cache = ignore_cache + self.verbose = verbose + + # Setup logger with custom styling for LLM operations + if logger: + self.logger = logger + elif verbose: + self.logger = AsyncLogger( + verbose=verbose, + icons={ + **AsyncLogger.DEFAULT_ICONS, + "LLM": "★", # Star for LLM operations + "CHUNK": "◈", # Diamond for chunks + "CACHE": "⚡", # Lightning for cache operations + }, + colors={ + **AsyncLogger.DEFAULT_COLORS, + LogLevel.INFO: LogColor.DIM_MAGENTA # Dimmed purple for LLM ops + }, + ) + else: + self.logger = None + + self.usages = [] + self.total_usage = TokenUsage() + + def __setattr__(self, name, value): + """Handle attribute setting.""" + # TODO: Planning to set properties dynamically based on the __init__ signature + sig = inspect.signature(self.__init__) + all_params = sig.parameters # Dictionary of parameter names and their details + + if name in self._UNWANTED_PROPS and value is not all_params[name].default: + raise AttributeError(f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}") + + super().__setattr__(name, value) + + def _get_cache_key(self, html: str, instruction: str) -> str: + """Generate a unique cache key based on HTML and instruction""" + content = f"{html}{instruction}" + return hashlib.md5(content.encode()).hexdigest() + + def _merge_chunks(self, text: str) -> List[str]: + """Split text into chunks with overlap using char or word mode.""" + ov = int(self.chunk_token_threshold * self.overlap_rate) + sections = merge_chunks( + docs=[text], + target_size=self.chunk_token_threshold, + overlap=ov, + word_token_ratio=self.word_token_rate, + ) + return sections + + def filter_content(self, html: str, ignore_cache: bool = True) -> List[str]: + if not html or not isinstance(html, str): + return [] + + if self.logger: + self.logger.info( + "Starting LLM markdown content filtering process", + tag="LLM", + params={"provider": self.llm_config.provider}, + colors={"provider": LogColor.CYAN}, + ) + + # Cache handling + cache_dir = Path(get_home_folder()) / "llm_cache" / "content_filter" + cache_dir.mkdir(parents=True, exist_ok=True) + cache_key = self._get_cache_key(html, self.instruction or "") + cache_file = cache_dir / f"{cache_key}.json" + + # if ignore_cache == None: + ignore_cache = self.ignore_cache + + if not ignore_cache and cache_file.exists(): + if self.logger: + self.logger.info("Found cached markdown result", tag="CACHE") + try: + with cache_file.open("r") as f: + cached_data = json.load(f) + usage = TokenUsage(**cached_data["usage"]) + self.usages.append(usage) + self.total_usage.completion_tokens += usage.completion_tokens + self.total_usage.prompt_tokens += usage.prompt_tokens + self.total_usage.total_tokens += usage.total_tokens + return cached_data["blocks"] + except Exception as e: + if self.logger: + self.logger.error( + f"LLM markdown: Cache read error: {str(e)}", tag="CACHE" + ) + + # Split into chunks + html_chunks = self._merge_chunks(html) + if self.logger: + self.logger.info( + "LLM markdown: Split content into {chunk_count} chunks", + tag="CHUNK", + params={"chunk_count": len(html_chunks)}, + colors={"chunk_count": LogColor.YELLOW}, + ) + + start_time = time.time() + + # Process chunks in parallel + with ThreadPoolExecutor(max_workers=4) as executor: + futures = [] + for i, chunk in enumerate(html_chunks): + if self.logger: + self.logger.debug( + "LLM markdown: Processing chunk {chunk_num}/{total_chunks}", + tag="CHUNK", + params={"chunk_num": i + 1, "total_chunks": len(html_chunks)}, + ) + + prompt_variables = { + "HTML": escape_json_string(sanitize_html(chunk)), + "REQUEST": self.instruction + or "Convert this HTML into clean, relevant markdown, removing any noise or irrelevant content.", + } + + prompt = PROMPT_FILTER_CONTENT + for var, value in prompt_variables.items(): + prompt = prompt.replace("{" + var + "}", value) + + def _proceed_with_chunk( + provider: str, + prompt: str, + api_token: str, + base_url: Optional[str] = None, + extra_args: Dict = {}, + ) -> List[str]: + if self.logger: + self.logger.info( + "LLM Markdown: Processing chunk {chunk_num}", + tag="CHUNK", + params={"chunk_num": i + 1}, + ) + return perform_completion_with_backoff( + provider, + prompt, + api_token, + base_url=base_url, + base_delay=self.llm_config.backoff_base_delay, + max_attempts=self.llm_config.backoff_max_attempts, + exponential_factor=self.llm_config.backoff_exponential_factor, + extra_args=extra_args, + ) + + future = executor.submit( + _proceed_with_chunk, + self.llm_config.provider, + prompt, + self.llm_config.api_token, + self.llm_config.base_url, + self.extra_args, + ) + futures.append((i, future)) + + # Collect results in order + ordered_results = [] + for i, future in sorted(futures): + try: + response = future.result() + + # Track usage + usage = TokenUsage( + completion_tokens=response.usage.completion_tokens, + prompt_tokens=response.usage.prompt_tokens, + total_tokens=response.usage.total_tokens, + completion_tokens_details=( + response.usage.completion_tokens_details.__dict__ + if response.usage.completion_tokens_details + else {} + ), + prompt_tokens_details=( + response.usage.prompt_tokens_details.__dict__ + if response.usage.prompt_tokens_details + else {} + ), + ) + self.usages.append(usage) + self.total_usage.completion_tokens += usage.completion_tokens + self.total_usage.prompt_tokens += usage.prompt_tokens + self.total_usage.total_tokens += usage.total_tokens + + blocks = extract_xml_data( + ["content"], response.choices[0].message.content + )["content"] + if blocks: + ordered_results.append(blocks) + if self.logger: + self.logger.success( + "LLM markdown: Successfully processed chunk {chunk_num}", + tag="CHUNK", + params={"chunk_num": i + 1}, + ) + except Exception as e: + if self.logger: + self.logger.error( + "LLM markdown: Error processing chunk {chunk_num}: {error}", + tag="CHUNK", + params={"chunk_num": i + 1, "error": str(e)}, + ) + + end_time = time.time() + if self.logger: + self.logger.success( + "LLM markdown: Completed processing in {time:.2f}s", + tag="LLM", + params={"time": end_time - start_time}, + colors={"time": LogColor.YELLOW}, + ) + + result = ordered_results if ordered_results else [] + + # Cache the final result + cache_data = {"blocks": result, "usage": self.total_usage.__dict__} + with cache_file.open("w") as f: + json.dump(cache_data, f) + if self.logger: + self.logger.info("Cached results for future use", tag="CACHE") + + return result + + def show_usage(self) -> None: + """Print usage statistics""" + print("\n=== Token Usage Summary ===") + print(f"{'Type':<15} {'Count':>12}") + print("-" * 30) + print(f"{'Completion':<15} {self.total_usage.completion_tokens:>12,}") + print(f"{'Prompt':<15} {self.total_usage.prompt_tokens:>12,}") + print(f"{'Total':<15} {self.total_usage.total_tokens:>12,}") + + if self.usages: + print("\n=== Usage History ===") + print(f"{'Request #':<10} {'Completion':>12} {'Prompt':>12} {'Total':>12}") + print("-" * 48) + for i, usage in enumerate(self.usages, 1): + print( + f"{i:<10} {usage.completion_tokens:>12,} " + f"{usage.prompt_tokens:>12,} {usage.total_tokens:>12,}" + ) diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py new file mode 100644 index 0000000..67e8725 --- /dev/null +++ b/crawl4ai/content_scraping_strategy.py @@ -0,0 +1,1014 @@ +import re +from itertools import chain +from abc import ABC, abstractmethod +from typing import Dict, Any, Optional +from bs4 import BeautifulSoup +import asyncio +import requests +from .config import ( + MIN_WORD_THRESHOLD, + IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, + IMAGE_SCORE_THRESHOLD, + ONLY_TEXT_ELIGIBLE_TAGS, + IMPORTANT_ATTRS, + SOCIAL_MEDIA_DOMAINS, +) +from bs4 import NavigableString, Comment +from bs4 import PageElement, Tag +from urllib.parse import urljoin +from requests.exceptions import InvalidSchema +from .utils import ( + extract_metadata, + normalize_url, + is_external_url, + get_base_domain, + extract_metadata_using_lxml, + extract_page_context, + calculate_link_intrinsic_score, +) +from lxml import etree +from lxml import html as lhtml +from typing import List +from .models import ScrapingResult, MediaItem, Link, Media, Links +import copy + +# Pre-compile regular expressions for Open Graph and Twitter metadata +OG_REGEX = re.compile(r"^og:") +TWITTER_REGEX = re.compile(r"^twitter:") +DIMENSION_REGEX = re.compile(r"(\d+)(\D*)") + + +# Function to parse srcset +def parse_srcset(s: str) -> List[Dict]: + if not s: + return [] + variants = [] + for part in s.split(","): + part = part.strip() + if not part: + continue + parts = part.split() + if len(parts) >= 1: + url = parts[0] + width = ( + parts[1].rstrip("w").split('.')[0] + if len(parts) > 1 and parts[1].endswith("w") + else None + ) + variants.append({"url": url, "width": width}) + return variants + + +# Function to parse image height/width value and units +def parse_dimension(dimension): + if dimension: + # match = re.match(r"(\d+)(\D*)", dimension) + match = DIMENSION_REGEX.match(dimension) + if match: + number = int(match.group(1)) + unit = match.group(2) or "px" # Default unit is 'px' if not specified + return number, unit + return None, None + + +# Fetch image file metadata to extract size and extension +def fetch_image_file_size(img, base_url): + # If src is relative path construct full URL, if not it may be CDN URL + img_url = urljoin(base_url, img.get("src")) + try: + response = requests.head(img_url) + if response.status_code == 200: + return response.headers.get("Content-Length", None) + else: + print(f"Failed to retrieve file size for {img_url}") + return None + except InvalidSchema: + return None + finally: + return + + +class ContentScrapingStrategy(ABC): + @abstractmethod + def scrap(self, url: str, html: str, **kwargs) -> ScrapingResult: + pass + + @abstractmethod + async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult: + pass + + +class LXMLWebScrapingStrategy(ContentScrapingStrategy): + """ + LXML-based implementation for fast web content scraping. + + This is the primary scraping strategy in Crawl4AI, providing high-performance + HTML parsing and content extraction using the lxml library. + + Note: WebScrapingStrategy is now an alias for this class to maintain + backward compatibility. + """ + def __init__(self, logger=None): + self.logger = logger + self.DIMENSION_REGEX = re.compile(r"(\d+)(\D*)") + self.BASE64_PATTERN = re.compile(r'data:image/[^;]+;base64,([^"]+)') + + def _log(self, level, message, tag="SCRAPE", **kwargs): + """Helper method to safely use logger.""" + if self.logger: + log_method = getattr(self.logger, level) + log_method(message=message, tag=tag, **kwargs) + + def scrap(self, url: str, html: str, **kwargs) -> ScrapingResult: + """ + Main entry point for content scraping. + + Args: + url (str): The URL of the page to scrape. + html (str): The HTML content of the page. + **kwargs: Additional keyword arguments. + + Returns: + ScrapingResult: A structured result containing the scraped content. + """ + actual_url = kwargs.get("redirected_url", url) + raw_result = self._scrap(actual_url, html, **kwargs) + if raw_result is None: + return ScrapingResult( + cleaned_html="", + success=False, + media=Media(), + links=Links(), + metadata={}, + ) + + # Convert media items + media = Media( + images=[ + MediaItem(**img) + for img in raw_result.get("media", {}).get("images", []) + if img + ], + videos=[ + MediaItem(**vid) + for vid in raw_result.get("media", {}).get("videos", []) + if vid + ], + audios=[ + MediaItem(**aud) + for aud in raw_result.get("media", {}).get("audios", []) + if aud + ], + tables=raw_result.get("media", {}).get("tables", []) + ) + + # Convert links + links = Links( + internal=[ + Link(**link) + for link in raw_result.get("links", {}).get("internal", []) + if link + ], + external=[ + Link(**link) + for link in raw_result.get("links", {}).get("external", []) + if link + ], + ) + + return ScrapingResult( + cleaned_html=raw_result.get("cleaned_html", ""), + success=raw_result.get("success", False), + media=media, + links=links, + metadata=raw_result.get("metadata", {}), + ) + + async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult: + """ + Main entry point for asynchronous content scraping. + + Args: + url (str): The URL of the page to scrape. + html (str): The HTML content of the page. + **kwargs: Additional keyword arguments. + + Returns: + ScrapingResult: A structured result containing the scraped content. + """ + return await asyncio.to_thread(self.scrap, url, html, **kwargs) + + def process_element(self, url, element: lhtml.HtmlElement, **kwargs) -> Dict[str, Any]: + """ + Process an HTML element. + + How it works: + 1. Check if the element is an image, video, or audio. + 2. Extract the element's attributes and content. + 3. Process the element based on its type. + 4. Return the processed element information. + + Args: + url (str): The URL of the page containing the element. + element (lhtml.HtmlElement): The HTML element to process. + **kwargs: Additional keyword arguments. + + Returns: + dict: A dictionary containing the processed element information. + """ + media = {"images": [], "videos": [], "audios": [], "tables": []} + internal_links_dict = {} + external_links_dict = {} + self._process_element( + url, element, media, internal_links_dict, external_links_dict, **kwargs + ) + return { + "media": media, + "internal_links_dict": internal_links_dict, + "external_links_dict": external_links_dict, + } + + def _process_element( + self, + url: str, + element: lhtml.HtmlElement, + media: Dict[str, List], + internal_links_dict: Dict[str, Any], + external_links_dict: Dict[str, Any], + page_context: dict = None, + **kwargs, + ) -> bool: + base_domain = kwargs.get("base_domain", get_base_domain(url)) + exclude_domains = set(kwargs.get("exclude_domains", [])) + + # Process links + try: + base_element = element.xpath("//head/base[@href]") + if base_element: + base_href = base_element[0].get("href", "").strip() + if base_href: + url = base_href + except Exception as e: + self._log("error", f"Error extracting base URL: {str(e)}", "SCRAPE") + pass + + for link in element.xpath(".//a[@href]"): + href = link.get("href", "").strip() + if not href: + continue + + try: + normalized_href = normalize_url( + href, url, + preserve_https=kwargs.get('preserve_https_for_internal_links', False), + original_scheme=kwargs.get('original_scheme') + ) + link_data = { + "href": normalized_href, + "text": link.text_content().strip(), + "title": link.get("title", "").strip(), + "base_domain": base_domain, + } + + # Add intrinsic scoring if enabled + if kwargs.get("score_links", False) and page_context is not None: + try: + intrinsic_score = calculate_link_intrinsic_score( + link_text=link_data["text"], + url=normalized_href, + title_attr=link_data["title"], + class_attr=link.get("class", ""), + rel_attr=link.get("rel", ""), + page_context=page_context + ) + link_data["intrinsic_score"] = intrinsic_score + except Exception: + # Fail gracefully - assign default score + link_data["intrinsic_score"] = 0 + else: + # No scoring enabled - assign infinity (all links equal priority) + link_data["intrinsic_score"] = 0 + + is_external = is_external_url(normalized_href, base_domain) + if is_external: + link_base_domain = get_base_domain(normalized_href) + link_data["base_domain"] = link_base_domain + if ( + kwargs.get("exclude_external_links", False) + or link_base_domain in exclude_domains + ): + link.getparent().remove(link) + continue + + if normalized_href not in external_links_dict: + external_links_dict[normalized_href] = link_data + else: + if normalized_href not in internal_links_dict: + internal_links_dict[normalized_href] = link_data + + except Exception as e: + self._log("error", f"Error processing link: {str(e)}", "SCRAPE") + continue + + # Process images + images = element.xpath(".//img") + total_images = len(images) + + for idx, img in enumerate(images): + src = img.get("src") or "" + img_domain = get_base_domain(src) + + # Decide if we need to exclude this image + # 1) If its domain is in exclude_domains, remove. + # 2) Or if exclude_external_images=True and it's an external domain, remove. + if (img_domain in exclude_domains) or ( + kwargs.get("exclude_external_images", False) + and is_external_url(src, base_domain) + ): + parent = img.getparent() + if parent is not None: + parent.remove(img) + continue + + # Otherwise, process the image as usual. + try: + processed_images = self.process_image( + img, url, idx, total_images, **kwargs + ) + if processed_images: + media["images"].extend(processed_images) + except Exception as e: + self._log("error", f"Error processing image: {str(e)}", "SCRAPE") + + # Process videos and audios + for media_type in ["video", "audio"]: + for elem in element.xpath(f".//{media_type}"): + media_info = { + "src": elem.get("src"), + "alt": elem.get("alt"), + "type": media_type, + "description": self.find_closest_parent_with_useful_text( + elem, **kwargs + ), + } + media[f"{media_type}s"].append(media_info) + + # Process source tags within media elements + for source in elem.xpath(".//source"): + if src := source.get("src"): + media[f"{media_type}s"].append({**media_info, "src": src}) + + # Clean up unwanted elements + if kwargs.get("remove_forms", False): + for form in element.xpath(".//form"): + form.getparent().remove(form) + + if excluded_tags := kwargs.get("excluded_tags", []): + for tag in excluded_tags: + for elem in element.xpath(f".//{tag}"): + elem.getparent().remove(elem) + + if excluded_selector := kwargs.get("excluded_selector", ""): + try: + for elem in element.cssselect(excluded_selector): + elem.getparent().remove(elem) + except Exception: + pass # Invalid selector + + return True + + def find_closest_parent_with_useful_text( + self, element: lhtml.HtmlElement, **kwargs + ) -> Optional[str]: + image_description_min_word_threshold = kwargs.get( + "image_description_min_word_threshold", IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD + ) + current = element + while current is not None: + if ( + current.text + and len(current.text_content().split()) + >= image_description_min_word_threshold + ): + return current.text_content().strip() + current = current.getparent() + return None + + def flatten_nested_elements(self, element: lhtml.HtmlElement) -> lhtml.HtmlElement: + """Flatten nested elements of the same type in LXML tree""" + if len(element) == 1 and element.tag == element[0].tag: + return self.flatten_nested_elements(element[0]) + + for child in element: + child_idx = element.index(child) + flattened_child = self.flatten_nested_elements(child) + if flattened_child is not child: # Only replace if actually flattened + element[child_idx] = flattened_child + + return element + + def process_image( + self, img: lhtml.HtmlElement, url: str, index: int, total_images: int, **kwargs + ) -> Optional[List[Dict]]: + # Quick validation checks + style = img.get("style", "") + alt = img.get("alt", "") + src = img.get("src", "") + data_src = img.get("data-src", "") + srcset = img.get("srcset", "") + data_srcset = img.get("data-srcset", "") + + if "display:none" in style: + return None + + parent = img.getparent() + if parent.tag in ["button", "input"]: + return None + + parent_classes = parent.get("class", "").split() + if any( + "button" in cls or "icon" in cls or "logo" in cls for cls in parent_classes + ): + return None + + # If src is in class or alt, likely an icon + if (src and any(c in src for c in ["button", "icon", "logo"])) or ( + alt and any(c in alt for c in ["button", "icon", "logo"]) + ): + return None + + # Score calculation + score = 0 + if (width := img.get("width")) and width.isdigit(): + score += 1 if int(width) > 150 else 0 + if (height := img.get("height")) and height.isdigit(): + score += 1 if int(height) > 150 else 0 + if alt: + score += 1 + score += index / total_images < 0.5 + + # Check formats in all possible sources + image_formats = {"jpg", "jpeg", "png", "webp", "avif", "gif"} + detected_format = None + for url in [src, data_src, srcset, data_srcset]: + if url: + format_matches = [fmt for fmt in image_formats if fmt in url.lower()] + if format_matches: + detected_format = format_matches[0] + score += 1 + break + + if srcset or data_srcset: + score += 1 + + if picture := img.xpath("./ancestor::picture[1]"): + score += 1 + + if score <= kwargs.get("image_score_threshold", IMAGE_SCORE_THRESHOLD): + return None + + # Process image variants + unique_urls = set() + image_variants = [] + base_info = { + "alt": alt, + "desc": self.find_closest_parent_with_useful_text(img, **kwargs), + "score": score, + "type": "image", + "group_id": index, + "format": detected_format, + } + + def add_variant(src: str, width: Optional[str] = None): + if src and not src.startswith("data:") and src not in unique_urls: + unique_urls.add(src) + variant = {**base_info, "src": src} + if width: + variant["width"] = width + image_variants.append(variant) + + # Add variants from different sources + add_variant(src) + add_variant(data_src) + + for srcset_attr in [srcset, data_srcset]: + if srcset_attr: + for source in parse_srcset(srcset_attr): + add_variant(source["url"], source["width"]) + + # Handle picture element + if picture: + for source in picture[0].xpath(".//source[@srcset]"): + if source_srcset := source.get("srcset"): + for src_data in parse_srcset(source_srcset): + add_variant(src_data["url"], src_data["width"]) + + # Check framework-specific attributes + for attr, value in img.attrib.items(): + if ( + attr.startswith("data-") + and ("src" in attr or "srcset" in attr) + and "http" in value + ): + add_variant(value) + + return image_variants if image_variants else None + + def remove_empty_elements_fast(self, root, word_count_threshold=5): + """ + Remove elements that fall below the desired word threshold in a single pass from the bottom up. + Skips non-element nodes like HtmlComment and bypasses certain tags that are allowed to have no content. + """ + bypass_tags = { + "a", + "img", + "br", + "hr", + "input", + "meta", + "link", + "source", + "track", + "wbr", + "tr", + "td", + "th", + } + + for el in reversed(list(root.iterdescendants())): + if not isinstance(el, lhtml.HtmlElement): + continue + + if el.tag in bypass_tags: + continue + + # Skip elements inside 
 or  tags where whitespace is significant
+            # This preserves whitespace-only spans (e.g.,  ) in code blocks
+            is_in_code_block = False
+            ancestor = el.getparent()
+            while ancestor is not None:
+                if ancestor.tag in ("pre", "code"):
+                    is_in_code_block = True
+                    break
+                ancestor = ancestor.getparent()
+
+            if is_in_code_block:
+                continue
+
+            text_content = (el.text_content() or "").strip()
+            if (
+                len(text_content.split()) < word_count_threshold
+                and not el.getchildren()
+            ):
+                parent = el.getparent()
+                if parent is not None:
+                    # Preserve .tail text before removing the element
+                    tail = el.tail
+                    if tail:
+                        prev = el.getprevious()
+                        if prev is not None:
+                            prev.tail = (prev.tail or "") + tail
+                        else:
+                            parent.text = (parent.text or "") + tail
+                    parent.remove(el)
+
+        return root
+
+    def remove_unwanted_attributes_fast(
+        self, root: lhtml.HtmlElement, important_attrs=None, keep_data_attributes=False
+    ) -> lhtml.HtmlElement:
+        """
+        Removes all attributes from each element (including root) except those in `important_attrs`.
+        If `keep_data_attributes=True`, also retain any attribute starting with 'data-'.
+
+        Returns the same root element, mutated in-place, for fluent usage.
+        """
+        if important_attrs is None:
+            important_attrs = set(IMPORTANT_ATTRS)
+
+        # If you want to handle the root as well, use 'include_self=True'
+        # so you don't miss attributes on the top-level element.
+        # Manually include the root, then all its descendants
+        for el in chain((root,), root.iterdescendants()):
+            # We only remove attributes on HtmlElement nodes, skip comments or text nodes
+            if not isinstance(el, lhtml.HtmlElement):
+                continue
+
+            old_attribs = dict(el.attrib)
+            new_attribs = {}
+
+            for attr_name, attr_val in old_attribs.items():
+                # If it's an important attribute, keep it
+                if attr_name in important_attrs:
+                    new_attribs[attr_name] = attr_val
+                # Or if keep_data_attributes is True and it's a 'data-*' attribute
+                elif keep_data_attributes and attr_name.startswith("data-"):
+                    new_attribs[attr_name] = attr_val
+
+            # Clear old attributes and set the filtered set
+            el.attrib.clear()
+            el.attrib.update(new_attribs)
+
+        return root
+
+
+    def _scrap(
+        self,
+        url: str,
+        html: str,
+        word_count_threshold: int = MIN_WORD_THRESHOLD,
+        css_selector: str = None,
+        target_elements: List[str] = None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        if not html:
+            return None
+
+        success = True
+        try:
+            doc = lhtml.document_fromstring(html)
+            # Match BeautifulSoup's behavior of using body or full doc
+            # body = doc.xpath('//body')[0] if doc.xpath('//body') else doc
+            body = doc
+
+            base_domain = get_base_domain(url)
+            
+            # Extract page context for link scoring (if enabled) - do this BEFORE any removals
+            page_context = None
+            if kwargs.get("score_links", False):
+                try:
+                    # Extract title
+                    title_elements = doc.xpath('//title')
+                    page_title = title_elements[0].text_content() if title_elements else ""
+                    
+                    # Extract headlines
+                    headlines = []
+                    for tag in ['h1', 'h2', 'h3']:
+                        elements = doc.xpath(f'//{tag}')
+                        for el in elements:
+                            text = el.text_content().strip()
+                            if text:
+                                headlines.append(text)
+                    headlines_text = ' '.join(headlines)
+                    
+                    # Extract meta description
+                    meta_desc_elements = doc.xpath('//meta[@name="description"]/@content')
+                    meta_description = meta_desc_elements[0] if meta_desc_elements else ""
+                    
+                    # Create page context
+                    page_context = extract_page_context(page_title, headlines_text, meta_description, url)
+                except Exception:
+                    page_context = {}  # Fail gracefully
+            
+            # Early removal of all images if exclude_all_images is set
+            # This is more efficient in lxml as we remove elements before any processing
+            if kwargs.get("exclude_all_images", False):
+                for img in body.xpath('//img'):
+                    if img.getparent() is not None:
+                        img.getparent().remove(img)
+
+            # Add comment removal
+            if kwargs.get("remove_comments", False):
+                comments = body.xpath("//comment()")
+                for comment in comments:
+                    comment.getparent().remove(comment)
+
+            # Handle tag-based removal first
+            excluded_tags = set(kwargs.get("excluded_tags", []) or [])
+            if excluded_tags:
+                for tag in excluded_tags:
+                    for element in body.xpath(f".//{tag}"):
+                        if element.getparent() is not None:
+                            element.getparent().remove(element)
+
+            # Handle CSS selector-based exclusion
+            excluded_selector = kwargs.get("excluded_selector", "")
+            if excluded_selector:
+                try:
+                    for element in body.cssselect(excluded_selector):
+                        if element.getparent() is not None:
+                            element.getparent().remove(element)
+                except Exception as e:
+                    self._log(
+                        "error", f"Error with excluded CSS selector: {str(e)}", "SCRAPE"
+                    )
+
+            # Extract metadata before any content filtering
+            try:
+                meta = extract_metadata_using_lxml(
+                    "", doc
+                )  # Using same function as BeautifulSoup version
+            except Exception as e:
+                self._log("error", f"Error extracting metadata: {str(e)}", "SCRAPE")
+                meta = {}
+
+            content_element = None
+            if css_selector:
+                try:
+                    selected = body.cssselect(css_selector)
+                    if selected:
+                        content_element = lhtml.Element("div")
+                        content_element.extend(copy.deepcopy(selected))
+                    else:
+                        content_element = body
+                except Exception as e:
+                    self._log("error", f"Error with css_selector: {str(e)}", "SCRAPE")
+                    content_element = body
+
+            if target_elements:
+                try:
+                    source = content_element if content_element is not None else body
+                    for_content_targeted_element = []
+                    for target_element in target_elements:
+                        for_content_targeted_element.extend(source.cssselect(target_element))
+                    content_element = lhtml.Element("div")
+                    content_element.extend(copy.deepcopy(for_content_targeted_element))
+                except Exception as e:
+                    self._log("error", f"Error with target element detection: {str(e)}", "SCRAPE")
+                    return None
+            elif content_element is None:
+                content_element = body
+
+            # Replace mermaid SVGs with text before they get stripped
+            for svg in body.xpath('.//svg[starts-with(@id, "mermaid-")]'):
+                try:
+                    diagram_type = svg.get("aria-roledescription", "diagram")
+                    labels = []
+                    seen = set()
+
+                    # Primary: foreignObject-based labels (flowchart, class, state…)
+                    for el in svg.cssselect(".nodeLabel, .label span, .edgeLabel span"):
+                        text = el.text_content().strip()
+                        if text and text not in seen:
+                            seen.add(text)
+                            labels.append(text)
+
+                    # Fallback: native SVG text/tspan elements (sequence, gantt, git…)
+                    if not labels:
+                        for el in svg.xpath(
+                            './/*[local-name()="text"] | .//*[local-name()="tspan"]'
+                        ):
+                            text = (el.text or "").strip()
+                            if text and text not in seen:
+                                seen.add(text)
+                                labels.append(text)
+
+                    if not labels:
+                        continue
+
+                    # Check whether the SVG already lives inside a 
 block.
+                    # If so, avoid creating a nested fence; just let the outer
+                    # 
 preserve the text that lxml has already flattened.
+                    ancestor = svg.getparent()
+                    inside_pre = False
+                    while ancestor is not None:
+                        if ancestor.tag == "pre":
+                            inside_pre = True
+                            break
+                        ancestor = ancestor.getparent()
+
+                    if inside_pre:
+                        # The outer 
 will format the text as a code block.
+                        # Replace the SVG with a plain span so lxml preserves
+                        # the label text without the SVG markup noise.
+                        placeholder = lhtml.Element("span")
+                        placeholder.text = "\n".join(labels)
+                        parent = svg.getparent()
+                        if parent is not None:
+                            parent.replace(svg, placeholder)
+                    else:
+                        # Build a fenced code block that survives markdown conversion
+                        placeholder = lhtml.Element("pre")
+                        placeholder.set("data-language", "mermaid")
+                        code = etree.SubElement(placeholder, "code")
+                        code.set("class", "language-mermaid")
+                        code.text = f"%% {diagram_type} diagram\n" + "\n".join(labels)
+                        parent = svg.getparent()
+                        if parent is not None:
+                            parent.replace(svg, placeholder)
+                except Exception:
+                    pass
+
+            # Remove script and style tags
+            for tag in ["style", "link", "meta", "noscript"]:
+                for element in body.xpath(f".//{tag}"):
+                    if element.getparent() is not None:
+                        element.getparent().remove(element)
+                        
+            # Handle script separately
+            for element in body.xpath(f".//script"):
+                parent = element.getparent()
+                if parent is not None:
+                    tail = element.tail  # Get the tail text
+                    if tail:
+                        prev = element.getprevious()  # Get the previous sibling node
+                        if prev is not None:
+                            if prev.tail:
+                                prev.tail += tail 
+                            else:
+                                prev.tail = tail
+                        else:
+                            if parent.text:
+                                parent.text += tail
+                            else:
+                                parent.text = tail
+                    parent.remove(element)  # Delete the element
+
+
+            # Handle social media and domain exclusions
+            kwargs["exclude_domains"] = set(kwargs.get("exclude_domains", []))
+            if kwargs.get("exclude_social_media_links", False):
+                kwargs["exclude_social_media_domains"] = set(
+                    kwargs.get("exclude_social_media_domains", [])
+                    + SOCIAL_MEDIA_DOMAINS
+                )
+                kwargs["exclude_domains"].update(kwargs["exclude_social_media_domains"])
+
+            # Process forms if needed
+            if kwargs.get("remove_forms", False):
+                for form in body.xpath(".//form"):
+                    if form.getparent() is not None:
+                        form.getparent().remove(form)
+
+            # Process content
+            media = {"images": [], "videos": [], "audios": [], "tables": []}
+            internal_links_dict = {}
+            external_links_dict = {}
+
+            self._process_element(
+                url,
+                body,
+                media,
+                internal_links_dict,
+                external_links_dict,
+                page_context=page_context,
+                base_domain=base_domain,
+                **kwargs,
+            )
+
+            # Extract tables using the table extraction strategy if provided
+            if 'table' not in excluded_tags:
+                table_extraction = kwargs.get('table_extraction')
+                if table_extraction:
+                    # Pass logger to the strategy if it doesn't have one
+                    if not table_extraction.logger:
+                        table_extraction.logger = self.logger
+                    # Extract tables using the strategy
+                    extracted_tables = table_extraction.extract_tables(body, **kwargs)
+                    media["tables"].extend(extracted_tables)
+
+            # Handle only_text option
+            if kwargs.get("only_text", False):
+                for tag in ONLY_TEXT_ELIGIBLE_TAGS:
+                    for element in body.xpath(f".//{tag}"):
+                        if element.text:
+                            new_text = lhtml.Element("span")
+                            new_text.text = element.text_content()
+                            if element.getparent() is not None:
+                                element.getparent().replace(element, new_text)
+
+            # Clean base64 images
+            for img in body.xpath(".//img[@src]"):
+                src = img.get("src", "")
+                if self.BASE64_PATTERN.match(src):
+                    img.set("src", self.BASE64_PATTERN.sub("", src))
+
+            # Remove empty elements
+            self.remove_empty_elements_fast(body, 1)
+
+            # Remove unneeded attributes
+            self.remove_unwanted_attributes_fast(
+                body, keep_data_attributes=kwargs.get("keep_data_attributes", False)
+            )
+
+            # Generate output HTML
+            cleaned_html = lhtml.tostring(
+                # body,   
+                content_element,
+                encoding="unicode",
+                pretty_print=True,
+                method="html",
+                with_tail=False,
+            ).strip()
+            
+            # Create links dictionary in the format expected by LinkPreview
+            links = {
+                "internal": list(internal_links_dict.values()),
+                "external": list(external_links_dict.values()),
+            }
+            
+            # Extract head content for links if configured
+            link_preview_config = kwargs.get("link_preview_config")
+            if link_preview_config is not None:
+                try:
+                    import asyncio
+                    from .link_preview import LinkPreview
+                    from .models import Links, Link
+                    
+                    verbose = link_preview_config.verbose
+                    
+                    if verbose:
+                        self._log("info", "Starting link head extraction for {internal} internal and {external} external links",
+                                  params={"internal": len(links["internal"]), "external": len(links["external"])}, tag="LINK_EXTRACT")
+                    
+                    # Convert dict links to Link objects
+                    internal_links = [Link(**link_data) for link_data in links["internal"]]
+                    external_links = [Link(**link_data) for link_data in links["external"]]
+                    links_obj = Links(internal=internal_links, external=external_links)
+                    
+                    # Create a config object for LinkPreview
+                    class TempCrawlerRunConfig:
+                        def __init__(self, link_config, score_links):
+                            self.link_preview_config = link_config
+                            self.score_links = score_links
+                    
+                    config = TempCrawlerRunConfig(link_preview_config, kwargs.get("score_links", False))
+                    
+                    # Extract head content (run async operation in sync context)
+                    async def extract_links():
+                        async with LinkPreview(self.logger) as extractor:
+                            return await extractor.extract_link_heads(links_obj, config)
+                    
+                    # Run the async operation
+                    try:
+                        # Check if we're already in an async context
+                        loop = asyncio.get_running_loop()
+                        # If we're in an async context, we need to run in a thread
+                        import concurrent.futures
+                        with concurrent.futures.ThreadPoolExecutor() as executor:
+                            future = executor.submit(asyncio.run, extract_links())
+                            updated_links = future.result()
+                    except RuntimeError:
+                        # No running loop, we can use asyncio.run directly
+                        updated_links = asyncio.run(extract_links())
+                    
+                    # Convert back to dict format
+                    links["internal"] = [link.dict() for link in updated_links.internal]
+                    links["external"] = [link.dict() for link in updated_links.external]
+                    
+                    if verbose:
+                        successful_internal = len([l for l in updated_links.internal if l.head_extraction_status == "valid"])
+                        successful_external = len([l for l in updated_links.external if l.head_extraction_status == "valid"])
+                        self._log("info", "Link head extraction completed: {internal_success}/{internal_total} internal, {external_success}/{external_total} external",
+                                  params={
+                                      "internal_success": successful_internal,
+                                      "internal_total": len(updated_links.internal),
+                                      "external_success": successful_external,
+                                      "external_total": len(updated_links.external)
+                                  }, tag="LINK_EXTRACT")
+                    else:
+                        self._log("info", "Link head extraction completed successfully", tag="LINK_EXTRACT")
+                        
+                except Exception as e:
+                    self._log("error", f"Error during link head extraction: {str(e)}", tag="LINK_EXTRACT")
+                    # Continue with original links if head extraction fails
+            
+            return {
+                "cleaned_html": cleaned_html,
+                "success": success,
+                "media": media,
+                "links": links,
+                "metadata": meta,
+            }
+
+        except Exception as e:
+            self._log("error", f"Error processing HTML: {str(e)}", "SCRAPE")
+            # Create error message in case of failure
+            error_body = lhtml.Element("div")
+            # Use etree.SubElement rather than lhtml.SubElement
+            error_div = etree.SubElement(error_body, "div", id="crawl4ai_error_message")
+            error_div.text = f"""
+            Crawl4AI Error: This page is not fully supported.
+            
+            Error Message: {str(e)}
+            
+            Possible reasons:
+            1. The page may have restrictions that prevent crawling.
+            2. The page might not be fully loaded.
+            
+            Suggestions:
+            - Try calling the crawl function with these parameters:
+            magic=True,
+            - Set headless=False to visualize what's happening on the page.
+            
+            If the issue persists, please check the page's structure and any potential anti-crawling measures.
+            """
+            cleaned_html = lhtml.tostring(
+                error_body, encoding="unicode", pretty_print=True
+            )
+            return {
+                "cleaned_html": cleaned_html,
+                "success": False,
+                "media": {
+                    "images": [],
+                    "videos": [],
+                    "audios": [],
+                    "tables": []
+                },
+                "links": {"internal": [], "external": []},
+                "metadata": {},
+            }
+
+
+# Backward compatibility alias
+WebScrapingStrategy = LXMLWebScrapingStrategy
diff --git a/crawl4ai/crawlers/__init__.py b/crawl4ai/crawlers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/crawl4ai/crawlers/amazon_product/__init__.py b/crawl4ai/crawlers/amazon_product/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/crawl4ai/crawlers/amazon_product/crawler.py b/crawl4ai/crawlers/amazon_product/crawler.py
new file mode 100644
index 0000000..45cc9d6
--- /dev/null
+++ b/crawl4ai/crawlers/amazon_product/crawler.py
@@ -0,0 +1,20 @@
+from crawl4ai.hub import BaseCrawler
+
+__meta__ = {
+    "version": "1.2.0",
+    "tested_on": ["amazon.com"],
+    "rate_limit": "50 RPM",
+    "schema": {"product": ["name", "price"]}
+}
+
+class AmazonProductCrawler(BaseCrawler):
+    async def run(self, url: str, **kwargs) -> str:
+        try:
+            self.logger.info(f"Crawling {url}")
+            return '{"product": {"name": "Test Amazon Product"}}'
+        except Exception as e:
+            self.logger.error(f"Crawl failed: {str(e)}")
+            return json.dumps({
+                "error": str(e),
+                "metadata": self.meta  # Include meta in error response
+            })            
\ No newline at end of file
diff --git a/crawl4ai/crawlers/google_search/__init__.py b/crawl4ai/crawlers/google_search/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/crawl4ai/crawlers/google_search/crawler.py b/crawl4ai/crawlers/google_search/crawler.py
new file mode 100644
index 0000000..1823341
--- /dev/null
+++ b/crawl4ai/crawlers/google_search/crawler.py
@@ -0,0 +1,131 @@
+from crawl4ai import BrowserConfig, AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai.hub import BaseCrawler
+from crawl4ai.utils import optimize_html, get_home_folder, preprocess_html_for_schema
+from crawl4ai import JsonCssExtractionStrategy
+from pathlib import Path
+import json
+import os
+from typing import Dict
+
+
+class GoogleSearchCrawler(BaseCrawler):
+    __meta__ = {
+        "version": "1.0.0",
+        "tested_on": ["google.com/search*"],
+        "rate_limit": "10 RPM",
+        "description": "Crawls Google Search results (text + images)",
+    }
+
+    def __init__(self):
+        super().__init__()
+        self.js_script = (Path(__file__).parent /
+                          "script.js").read_text()
+
+    async def run(self, url="", query: str = "", search_type: str = "text", schema_cache_path = None, **kwargs) -> str:
+        """Crawl Google Search results for a query"""
+        url = f"https://www.google.com/search?q={query}&gl=sg&hl=en" if search_type == "text" else f"https://www.google.com/search?q={query}&gl=sg&hl=en&tbs=qdr:d&udm=2"
+        if kwargs.get("page_start", 1) > 1:
+            url = f"{url}&start={kwargs['page_start'] * 10}"
+        if kwargs.get("page_length", 1) > 1:
+            url = f"{url}&num={kwargs['page_length']}"
+            
+        browser_config = BrowserConfig(headless=True, verbose=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            config = CrawlerRunConfig(
+                cache_mode=kwargs.get("cache_mode", CacheMode.BYPASS),
+                keep_attrs=["id", "class"],
+                keep_data_attributes=True,
+                delay_before_return_html=kwargs.get(
+                    "delay", 2 if search_type == "image" else 1),
+                js_code=self.js_script if search_type == "image" else None,
+            )
+
+            result = await crawler.arun(url=url, config=config)
+            if not result.success:
+                return json.dumps({"error": result.error})
+
+            if search_type == "image":
+                if result.js_execution_result.get("success", False) is False:
+                    return json.dumps({"error": result.js_execution_result.get("error", "Unknown error")})
+                if "results" in result.js_execution_result:
+                    image_result = result.js_execution_result['results'][0]
+                    if image_result.get("success", False) is False:
+                        return json.dumps({"error": image_result.get("error", "Unknown error")})
+                    return json.dumps(image_result["result"], indent=4)
+
+            # For text search, extract structured data
+            schemas = await self._build_schemas(result.cleaned_html, schema_cache_path)
+            extracted = {
+                key: JsonCssExtractionStrategy(schema=schemas[key]).run(
+                    url=url, sections=[result.html]
+                )
+                for key in schemas
+            }
+            return json.dumps(extracted, indent=4)
+
+    async def _build_schemas(self, html: str, schema_cache_path: str = None) -> Dict[str, Dict]:
+        """Build extraction schemas (organic, top stories, etc.)"""
+        home_dir = get_home_folder() if not schema_cache_path else schema_cache_path
+        os.makedirs(f"{home_dir}/schema", exist_ok=True)
+
+        # cleaned_html = optimize_html(html, threshold=100)
+        cleaned_html = preprocess_html_for_schema(html) 
+
+        organic_schema = None
+        if os.path.exists(f"{home_dir}/schema/organic_schema.json"):
+            with open(f"{home_dir}/schema/organic_schema.json", "r") as f:
+                organic_schema = json.load(f)
+        else:
+            organic_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "snippet": "...",
+            "date": "1 hour ago",
+        }""",
+                query="""The given html is the crawled html from Google search result. Please find the schema for organic search item in the given html, I am interested in title, link, snippet text. date."""
+            )
+
+            with open(f"{home_dir}/schema/organic_schema.json", "w") as f:
+                f.write(json.dumps(organic_schema))
+
+        top_stories_schema = None
+        if os.path.exists(f"{home_dir}/schema/top_stories_schema.json"):
+            with open(f"{home_dir}/schema/top_stories_schema.json", "r") as f:
+                top_stories_schema = json.load(f)
+        else:
+            top_stories_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "source": "Insider Monkey",
+            "date": "1 hour ago",
+        }""",
+                query="""The given html is the crawled html from Google search result. Please find the schema for Top Story item int he given html, I am interested in title, link, source. date and imageUrl."""
+            )
+
+            with open(f"{home_dir}/schema/top_stories_schema.json", "w") as f:
+                f.write(json.dumps(top_stories_schema))
+
+        suggested_query_schema = None
+        if os.path.exists(f"{home_dir}/schema/suggested_query_schema.json"):
+            with open(f"{home_dir}/schema/suggested_query_schema.json", "r") as f:
+                suggested_query_schema = json.load(f)
+        else:
+            suggested_query_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "query": "A for Apple",
+        }""",
+                query="""The given HTML contains the crawled HTML from Google search results. Please find the schema for each suggested query in the section "People also search for" within the given HTML. I am interested in the queries only."""
+            )
+            with open(f"{home_dir}/schema/suggested_query_schema.json", "w") as f:
+                f.write(json.dumps(suggested_query_schema))
+
+        return {
+            "organic_schema": organic_schema,
+            "top_stories_schema": top_stories_schema,
+            "suggested_query_schema": suggested_query_schema,
+        }
diff --git a/crawl4ai/crawlers/google_search/script.js b/crawl4ai/crawlers/google_search/script.js
new file mode 100644
index 0000000..3325746
--- /dev/null
+++ b/crawl4ai/crawlers/google_search/script.js
@@ -0,0 +1,115 @@
+(() => {
+    // Function to extract image data from Google Images page
+    function extractImageData() {
+        const keys = Object.keys(window.W_jd);
+        let allImageData = [];
+        let currentPosition = 0;
+
+        // Get the symbol we'll use (from first valid entry)
+        let targetSymbol;
+        for (let key of keys) {
+            try {
+                const symbols = Object.getOwnPropertySymbols(window.W_jd[key]);
+                if (symbols.length > 0) {
+                    targetSymbol = symbols[0];
+                    break;
+                }
+            } catch (e) {
+                continue;
+            }
+        }
+
+        if (!targetSymbol) return [];
+
+        // Iterate through ALL keys
+        for (let key of keys) {
+            try {
+                const o1 = window.W_jd[key][targetSymbol]
+                if (!o1) continue;
+                const data = Object.values(o1)[0]
+                // const data = window.W_jd[key][targetSymbol]?.Ws;
+                // Check if this is a valid image data entry
+                if (data && Array.isArray(data[1])) {
+                    const processedData = processImageEntry(data, currentPosition);
+                    if (processedData) {
+                        allImageData.push(processedData);
+                        currentPosition++;
+                    }
+                }
+            } catch (e) {
+                continue;
+            }
+        }
+
+        return allImageData;
+    }
+
+    function processImageEntry(entry, position) {
+        const imageData = entry[1];
+        if (!Array.isArray(imageData)) return null;
+
+        // Extract the image ID
+        const imageId = imageData[1];
+        if (!imageId) return null;
+
+        // Find the corresponding DOM element
+        const domElement = document.querySelector(`[data-docid="${imageId}"]`);
+        if (!domElement) return null;
+
+        // Extract data from the array structure
+        const [
+            _,
+            id,
+            thumbnailInfo,
+            imageInfo,
+            __,
+            ___,
+            rgb,
+            ____,
+            _____,
+            metadata
+        ] = imageData;
+
+        // Ensure we have the required data
+        if (!thumbnailInfo || !imageInfo) return null;
+
+        // Extract metadata from DOM
+        const title = domElement?.querySelector('.toI8Rb')?.textContent?.trim();
+        const source = domElement?.querySelector('.guK3rf')?.textContent?.trim();
+        const link = domElement?.querySelector('a.EZAeBe')?.href;
+
+        if (!link) return null;
+
+        // Build Google Image URL
+        const googleUrl = buildGoogleImageUrl(imageInfo[0], link, imageId, imageInfo[1], imageInfo[2]);
+
+        return {
+            title,
+            imageUrl: imageInfo[0],
+            imageWidth: imageInfo[2],
+            imageHeight: imageInfo[1],
+            thumbnailUrl: thumbnailInfo[0],
+            thumbnailWidth: thumbnailInfo[2],
+            thumbnailHeight: thumbnailInfo[1],
+            source,
+            domain: metadata['2000']?.[1] || new URL(link).hostname,
+            link,
+            googleUrl,
+            position: position + 1
+        };
+    }
+
+    function buildGoogleImageUrl(imgUrl, refUrl, tbnid, height, width) {
+        const params = new URLSearchParams({
+            imgurl: imgUrl,
+            tbnid: tbnid,
+            imgrefurl: refUrl,
+            docid: tbnid,
+            w: width.toString(),
+            h: height.toString(),
+        });
+
+        return `https://www.google.com/imgres?${params.toString()}`;
+    }
+    return extractImageData();
+})();
\ No newline at end of file
diff --git a/crawl4ai/deep_crawling/__init__.py b/crawl4ai/deep_crawling/__init__.py
new file mode 100644
index 0000000..4e831aa
--- /dev/null
+++ b/crawl4ai/deep_crawling/__init__.py
@@ -0,0 +1,47 @@
+# deep_crawling/__init__.py
+from .base_strategy import DeepCrawlDecorator, DeepCrawlStrategy
+from .bfs_strategy import BFSDeepCrawlStrategy
+from .bff_strategy import BestFirstCrawlingStrategy
+from .dfs_strategy import DFSDeepCrawlStrategy
+from .filters import (
+    FilterChain,
+    ContentTypeFilter,
+    DomainFilter,
+    URLFilter,
+    URLPatternFilter,
+    FilterStats,
+    ContentRelevanceFilter,
+    SEOFilter
+)
+from .scorers import (
+    KeywordRelevanceScorer,
+    URLScorer,
+    CompositeScorer,
+    DomainAuthorityScorer,
+    FreshnessScorer,
+    PathDepthScorer,
+    ContentTypeScorer
+)
+
+__all__ = [
+    "DeepCrawlDecorator",
+    "DeepCrawlStrategy",
+    "BFSDeepCrawlStrategy",
+    "BestFirstCrawlingStrategy",
+    "DFSDeepCrawlStrategy",
+    "FilterChain",
+    "ContentTypeFilter",
+    "DomainFilter",
+    "URLFilter",
+    "URLPatternFilter",
+    "FilterStats",
+    "ContentRelevanceFilter",
+    "SEOFilter",
+    "KeywordRelevanceScorer",
+    "URLScorer",
+    "CompositeScorer",
+    "DomainAuthorityScorer",
+    "FreshnessScorer",
+    "PathDepthScorer",
+    "ContentTypeScorer",
+]
diff --git a/crawl4ai/deep_crawling/base_strategy.py b/crawl4ai/deep_crawling/base_strategy.py
new file mode 100644
index 0000000..55cd6e5
--- /dev/null
+++ b/crawl4ai/deep_crawling/base_strategy.py
@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator, Optional, Set, List, Dict
+from functools import wraps
+from contextvars import ContextVar
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+
+
+class DeepCrawlDecorator:
+    """Decorator that adds deep crawling capability to arun method."""
+    deep_crawl_active = ContextVar("deep_crawl_active", default=False)
+    
+    def __init__(self, crawler: AsyncWebCrawler): 
+        self.crawler = crawler
+
+    def __call__(self, original_arun):
+        @wraps(original_arun)
+        async def wrapped_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
+            # If deep crawling is already active, call the original method to avoid recursion.
+            if config and config.deep_crawl_strategy and not self.deep_crawl_active.get():
+                token = self.deep_crawl_active.set(True)
+                # Await the arun call to get the actual result object.
+                result_obj = await config.deep_crawl_strategy.arun(
+                    crawler=self.crawler,
+                    start_url=url,
+                    config=config
+                )
+                if config.stream:
+                    async def result_wrapper():
+                        try:
+                            async for result in result_obj:
+                                yield result
+                        finally:
+                            self.deep_crawl_active.set(False)
+                    return result_wrapper()
+                else:
+                    try:
+                        return result_obj
+                    finally:
+                        self.deep_crawl_active.set(False)
+            return await original_arun(url, config=config, **kwargs)
+        return wrapped_arun
+
+class DeepCrawlStrategy(ABC):
+    """
+    Abstract base class for deep crawling strategies.
+    
+    Core functions:
+      - arun: Main entry point that returns an async generator of CrawlResults.
+      - shutdown: Clean up resources.
+      - can_process_url: Validate a URL and decide whether to process it.
+      - _process_links: Extract and process links from a CrawlResult.
+    """
+
+    @abstractmethod
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) mode:
+        Processes one BFS level at a time, then yields all the results.
+        """
+        pass
+
+    @abstractmethod
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming mode:
+        Processes one BFS level at a time and yields results immediately as they arrive.
+        """
+        pass
+    
+    async def arun(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: Optional[CrawlerRunConfig] = None,
+    ) -> RunManyReturn:
+        """
+        Traverse the given URL using the specified crawler.
+        
+        Args:
+            start_url (str): The URL from which to start crawling.
+            crawler (AsyncWebCrawler): The crawler instance to use.
+            crawler_run_config (Optional[CrawlerRunConfig]): Crawler configuration.
+        
+        Returns:
+            Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+        """
+        if config is None:
+            raise ValueError("CrawlerRunConfig must be provided")
+
+        if config.stream:
+            return self._arun_stream(start_url, crawler, config)
+        else:
+            return await self._arun_batch(start_url, crawler, config)
+
+    def __call__(self, start_url: str, crawler: AsyncWebCrawler, config: CrawlerRunConfig):
+        return self.arun(start_url, crawler, config)
+
+    @abstractmethod
+    async def shutdown(self) -> None:
+        """
+        Clean up resources used by the deep crawl strategy.
+        """
+        pass
+
+    @abstractmethod
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validate the URL format and apply custom filtering logic.
+        
+        Args:
+            url (str): The URL to validate.
+            depth (int): The current depth in the crawl.
+        
+        Returns:
+            bool: True if the URL should be processed, False otherwise.
+        """
+        pass
+
+    @abstractmethod
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_level: List[tuple],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extract and process links from the given crawl result.
+        
+        This method should:
+          - Validate each extracted URL using can_process_url.
+          - Optionally score URLs.
+          - Append valid URLs (and their parent references) to the next_level list.
+          - Update the depths dictionary with the new depth for each URL.
+        
+        Args:
+            result (CrawlResult): The result from a crawl operation.
+            source_url (str): The URL from which this result was obtained.
+            current_depth (int): The depth at which the source URL was processed.
+            visited (Set[str]): Set of already visited URLs.
+            next_level (List[tuple]): List of tuples (url, parent_url) for the next BFS level.
+            depths (Dict[str, int]): Mapping of URLs to their current depth.
+        """
+        pass
+
diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py
new file mode 100644
index 0000000..511fde6
--- /dev/null
+++ b/crawl4ai/deep_crawling/bff_strategy.py
@@ -0,0 +1,429 @@
+# best_first_crawling_strategy.py
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple, Any, Callable, Awaitable, Union
+from urllib.parse import urlparse
+
+from ..models import TraversalStats
+from .filters import FilterChain
+from .scorers import URLScorer
+from . import DeepCrawlStrategy
+
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+from ..utils import normalize_url_for_deep_crawl
+
+from math import inf as infinity
+
+# Configurable batch size for processing items from the priority queue
+BATCH_SIZE = 10
+
+
+class BestFirstCrawlingStrategy(DeepCrawlStrategy):
+    """
+    Best-First Crawling Strategy using a priority queue.
+    
+    This strategy prioritizes URLs based on their score, ensuring that higher-value
+    pages are crawled first. It reimplements the core traversal loop to use a priority
+    queue while keeping URL validation and link discovery consistent with our design.
+    
+    Core methods:
+      - arun: Returns either a list (batch mode) or an async generator (stream mode).
+      - _arun_best_first: Core generator that uses a priority queue to yield CrawlResults.
+      - can_process_url: Validates URLs and applies filtering (inherited behavior).
+      - link_discovery: Extracts and validates links from a CrawlResult.
+    """
+    def __init__(
+        self,
+        max_depth: int,
+        filter_chain: FilterChain = FilterChain(),
+        url_scorer: Optional[URLScorer] = None,
+        include_external: bool = False,
+        score_threshold: float = -infinity,
+        max_pages: int = infinity,
+        logger: Optional[logging.Logger] = None,
+        # Optional resume/callback parameters for crash recovery
+        resume_state: Optional[Dict[str, Any]] = None,
+        on_state_change: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
+        # Optional cancellation callback - checked before each URL is processed
+        should_cancel: Optional[Callable[[], Union[bool, Awaitable[bool]]]] = None,
+    ):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.url_scorer = url_scorer
+        self.include_external = include_external
+        self.score_threshold = score_threshold
+        self.max_pages = max_pages
+        # self.logger = logger or logging.getLogger(__name__)
+        # Ensure logger is always a Logger instance, not a dict from serialization
+        if isinstance(logger, logging.Logger):
+            self.logger = logger
+        else:
+            # Create a new logger if logger is None, dict, or any other non-Logger type
+            self.logger = logging.getLogger(__name__)
+        self.stats = TraversalStats(start_time=datetime.now())
+        self._cancel_event = asyncio.Event()
+        self._pages_crawled = 0
+        # Store for use in arun methods
+        self._resume_state = resume_state
+        self._on_state_change = on_state_change
+        self._should_cancel = should_cancel
+        self._last_state: Optional[Dict[str, Any]] = None
+        # Shadow list for queue items (only used when on_state_change is set)
+        self._queue_shadow: Optional[List[Tuple[float, int, str, Optional[str]]]] = None
+
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validate the URL format and apply filtering.
+        For the starting URL (depth 0), filtering is bypassed.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError("Missing scheme or netloc")
+            if parsed.scheme not in ("http", "https"):
+                raise ValueError("Invalid scheme")
+            if "." not in parsed.netloc:
+                raise ValueError("Invalid domain")
+        except Exception as e:
+            self.logger.warning(f"Invalid URL: {url}, error: {e}")
+            return False
+
+        if depth != 0 and not await self.filter_chain.apply(url):
+            return False
+
+        return True
+
+    def cancel(self) -> None:
+        """
+        Cancel the crawl. Thread-safe, can be called from any context.
+
+        The crawl will stop before processing the next URL. The current URL
+        being processed (if any) will complete before the crawl stops.
+        """
+        self._cancel_event.set()
+
+    @property
+    def cancelled(self) -> bool:
+        """
+        Check if the crawl was/is cancelled. Thread-safe.
+
+        Returns:
+            True if the crawl has been cancelled, False otherwise.
+        """
+        return self._cancel_event.is_set()
+
+    async def _check_cancellation(self) -> bool:
+        """
+        Check if crawl should be cancelled.
+
+        Handles both internal cancel flag and external should_cancel callback.
+        Supports both sync and async callbacks.
+
+        Returns:
+            True if crawl should be cancelled, False otherwise.
+        """
+        if self._cancel_event.is_set():
+            return True
+
+        if self._should_cancel:
+            try:
+                # Handle both sync and async callbacks
+                result = self._should_cancel()
+                if asyncio.iscoroutine(result):
+                    result = await result
+
+                if result:
+                    self._cancel_event.set()
+                    self.stats.end_time = datetime.now()
+                    return True
+            except Exception as e:
+                # Fail-open: log warning and continue crawling
+                self.logger.warning(f"should_cancel callback error: {e}")
+
+        return False
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_links: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extract links from the crawl result, validate them, and append new URLs
+        (with their parent references) to next_links.
+        Also updates the depths dictionary.
+        """
+        new_depth = current_depth + 1
+        if new_depth > self.max_depth:
+            return
+            
+        # If we've reached the max pages limit, don't discover new links
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            return
+
+        # Retrieve internal links; include external links if enabled.
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        # If we have more links than remaining capacity, limit how many we'll process
+        valid_links = []
+        for link in links:
+            url = link.get("href")
+            base_url = normalize_url_for_deep_crawl(url, source_url)
+            if base_url in visited:
+                continue
+            if not await self.can_process_url(base_url, new_depth):
+                self.stats.urls_skipped += 1
+                continue
+                
+            valid_links.append(base_url)
+            
+        # Record the new depths and add to next_links
+        for url in valid_links:
+            depths[url] = new_depth
+            next_links.append((url, source_url))
+
+    async def _arun_best_first(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Core best-first crawl method using a priority queue.
+
+        The queue items are tuples of (score, depth, url, parent_url). Lower scores
+        are treated as higher priority. URLs are processed in batches for efficiency.
+        """
+        # Reset cancel event for strategy reuse
+        self._cancel_event = asyncio.Event()
+
+        queue: asyncio.PriorityQueue = asyncio.PriorityQueue()
+
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+            # Restore queue from saved items
+            queue_items = self._resume_state.get("queue_items", [])
+            for item in queue_items:
+                await queue.put((item["score"], item["depth"], item["url"], item["parent_url"]))
+            # Initialize shadow list if callback is set
+            if self._on_state_change:
+                self._queue_shadow = [
+                    (item["score"], item["depth"], item["url"], item["parent_url"])
+                    for item in queue_items
+                ]
+        else:
+            # Original initialization
+            initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0
+            await queue.put((-initial_score, 0, start_url, None))
+            visited: Set[str] = set()
+            depths: Dict[str, int] = {start_url: 0}
+            # Initialize shadow list if callback is set
+            if self._on_state_change:
+                self._queue_shadow = [(-initial_score, 0, start_url, None)]
+
+        while not queue.empty() and not self._cancel_event.is_set():
+            # Stop if we've reached the max pages limit
+            if self._pages_crawled >= self.max_pages:
+                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                break
+
+            # Check external cancellation callback before processing this batch
+            if await self._check_cancellation():
+                self.logger.info("Crawl cancelled by user")
+                break
+
+            # Calculate how many more URLs we can process in this batch
+            remaining = self.max_pages - self._pages_crawled
+            batch_size = min(BATCH_SIZE, remaining)
+            if batch_size <= 0:
+                # No more pages to crawl
+                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                break
+                
+            batch: List[Tuple[float, int, str, Optional[str]]] = []
+            # Retrieve up to BATCH_SIZE items from the priority queue.
+            for _ in range(BATCH_SIZE):
+                if queue.empty():
+                    break
+                item = await queue.get()
+                # Remove from shadow list if tracking
+                if self._on_state_change and self._queue_shadow is not None:
+                    try:
+                        self._queue_shadow.remove(item)
+                    except ValueError:
+                        pass  # Item may have been removed already
+                score, depth, url, parent_url = item
+                if url in visited:
+                    continue
+                visited.add(url)
+                batch.append(item)
+
+            if not batch:
+                continue
+
+            # Process the current batch of URLs concurrently, but process the
+            # results in the original priority-queue order. arun_many streams
+            # results as requests finish, so discovering links immediately can
+            # make subsequent queue ordering depend on network timing.
+            urls = [item[2] for item in batch]
+            batch_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=urls, config=batch_config)
+            results_by_url: Dict[str, CrawlResult] = {}
+            async for result in stream_gen:
+                results_by_url[result.url] = result
+
+            for score, depth, url, parent_url in batch:
+                result = results_by_url.get(url)
+                if result is None:
+                    continue
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent_url
+                result.metadata["score"] = -score
+                
+                # Count only successful crawls toward max_pages limit
+                if result.success:
+                    self._pages_crawled += 1
+
+                # Yield the result before any limit check so the boundary page is
+                # kept (mirrors BFS/DFS, which append the result before breaking).
+                yield result
+
+                # Stop once the limit is reached, but only after yielding the
+                # successful boundary page above.
+                if result.success and self._pages_crawled >= self.max_pages:
+                    self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                    break  # Exit the generator
+
+                # Only discover links from successful crawls
+                if result.success:
+                    # Discover new links from this result
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    
+                    for new_url, new_parent in new_links:
+                        new_depth = depths.get(new_url, depth + 1)
+                        new_score = self.url_scorer.score(new_url) if self.url_scorer else 0
+                        # Skip URLs with scores below the threshold
+                        if new_score < self.score_threshold:
+                            self.logger.debug(
+                                f"URL {new_url} skipped: score {new_score} below threshold {self.score_threshold}"
+                            )
+                            self.stats.urls_skipped += 1
+                            continue
+                        queue_item = (-new_score, new_depth, new_url, new_parent)
+                        await queue.put(queue_item)
+                        # Add to shadow list if tracking
+                        if self._on_state_change and self._queue_shadow is not None:
+                            self._queue_shadow.append(queue_item)
+
+                    # Capture state after EACH URL processed (if callback set)
+                    if self._on_state_change and self._queue_shadow is not None:
+                        state = {
+                            "strategy_type": "best_first",
+                            "visited": list(visited),
+                            "queue_items": [
+                                {"score": s, "depth": d, "url": u, "parent_url": p}
+                                for s, d, u, p in self._queue_shadow
+                            ],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                            "cancelled": self._cancel_event.is_set(),
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
+
+        # Final state update if cancelled
+        if self._cancel_event.is_set() and self._on_state_change and self._queue_shadow is not None:
+            state = {
+                "strategy_type": "best_first",
+                "visited": list(visited),
+                "queue_items": [
+                    {"score": s, "depth": d, "url": u, "parent_url": p}
+                    for s, d, u, p in self._queue_shadow
+                ],
+                "depths": depths,
+                "pages_crawled": self._pages_crawled,
+                "cancelled": True,
+            }
+            self._last_state = state
+            await self._on_state_change(state)
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Best-first crawl in batch mode.
+        
+        Aggregates all CrawlResults into a list.
+        """
+        results: List[CrawlResult] = []
+        async for result in self._arun_best_first(start_url, crawler, config):
+            results.append(result)
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Best-first crawl in streaming mode.
+        
+        Yields CrawlResults as they become available.
+        """
+        async for result in self._arun_best_first(start_url, crawler, config):
+            yield result
+
+    async def arun(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: Optional[CrawlerRunConfig] = None,
+    ) -> "RunManyReturn":
+        """
+        Main entry point for best-first crawling.
+        
+        Returns either a list (batch mode) or an async generator (stream mode)
+        of CrawlResults.
+        """
+        if config is None:
+            raise ValueError("CrawlerRunConfig must be provided")
+        if config.stream:
+            return self._arun_stream(start_url, crawler, config)
+        else:
+            return await self._arun_batch(start_url, crawler, config)
+
+    async def shutdown(self) -> None:
+        """
+        Signal cancellation and clean up resources.
+        """
+        self._cancel_event.set()
+        self.stats.end_time = datetime.now()
+
+    def export_state(self) -> Optional[Dict[str, Any]]:
+        """
+        Export current crawl state for external persistence.
+
+        Note: This returns the last captured state. For real-time state,
+        use the on_state_change callback.
+
+        Returns:
+            Dict with strategy state, or None if no state captured yet.
+        """
+        return self._last_state
diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py
new file mode 100644
index 0000000..dfb7592
--- /dev/null
+++ b/crawl4ai/deep_crawling/bfs_strategy.py
@@ -0,0 +1,420 @@
+# bfs_deep_crawl_strategy.py
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple, Any, Callable, Awaitable, Union
+from urllib.parse import urlparse
+
+from ..models import TraversalStats
+from .filters import FilterChain
+from .scorers import URLScorer
+from . import DeepCrawlStrategy  
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
+from ..utils import normalize_url_for_deep_crawl, efficient_normalize_url_for_deep_crawl
+from math import inf as infinity
+
+class BFSDeepCrawlStrategy(DeepCrawlStrategy):
+    """
+    Breadth-First Search deep crawling strategy.
+    
+    Core functions:
+      - arun: Main entry point; splits execution into batch or stream modes.
+      - link_discovery: Extracts, filters, and (if needed) scores the outgoing URLs.
+      - can_process_url: Validates URL format and applies the filter chain.
+    """
+    def __init__(
+        self,
+        max_depth: int,
+        filter_chain: FilterChain = FilterChain(),
+        url_scorer: Optional[URLScorer] = None,
+        include_external: bool = False,
+        score_threshold: float = -infinity,
+        max_pages: int = infinity,
+        logger: Optional[logging.Logger] = None,
+        # Optional resume/callback parameters for crash recovery
+        resume_state: Optional[Dict[str, Any]] = None,
+        on_state_change: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
+        # Optional cancellation callback - checked before each URL is processed
+        should_cancel: Optional[Callable[[], Union[bool, Awaitable[bool]]]] = None,
+    ):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.url_scorer = url_scorer
+        self.include_external = include_external
+        self.score_threshold = score_threshold
+        self.max_pages = max_pages
+        # self.logger = logger or logging.getLogger(__name__)
+        # Ensure logger is always a Logger instance, not a dict from serialization
+        if isinstance(logger, logging.Logger):
+            self.logger = logger
+        else:
+            # Create a new logger if logger is None, dict, or any other non-Logger type
+            self.logger = logging.getLogger(__name__)
+        self.stats = TraversalStats(start_time=datetime.now())
+        self._cancel_event = asyncio.Event()
+        self._pages_crawled = 0
+        # Store for use in arun methods
+        self._resume_state = resume_state
+        self._on_state_change = on_state_change
+        self._should_cancel = should_cancel
+        self._last_state: Optional[Dict[str, Any]] = None
+
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validates the URL and applies the filter chain.
+        For the start URL (depth 0) filtering is bypassed.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError("Missing scheme or netloc")
+            if parsed.scheme not in ("http", "https"):
+                raise ValueError("Invalid scheme")
+            if "." not in parsed.netloc:
+                raise ValueError("Invalid domain")
+        except Exception as e:
+            self.logger.warning(f"Invalid URL: {url}, error: {e}")
+            return False
+
+        if depth != 0 and not await self.filter_chain.apply(url):
+            return False
+
+        return True
+
+    def cancel(self) -> None:
+        """
+        Cancel the crawl. Thread-safe, can be called from any context.
+
+        The crawl will stop before processing the next URL. The current URL
+        being processed (if any) will complete before the crawl stops.
+        """
+        self._cancel_event.set()
+
+    @property
+    def cancelled(self) -> bool:
+        """
+        Check if the crawl was/is cancelled. Thread-safe.
+
+        Returns:
+            True if the crawl has been cancelled, False otherwise.
+        """
+        return self._cancel_event.is_set()
+
+    async def _check_cancellation(self) -> bool:
+        """
+        Check if crawl should be cancelled.
+
+        Handles both internal cancel flag and external should_cancel callback.
+        Supports both sync and async callbacks.
+
+        Returns:
+            True if crawl should be cancelled, False otherwise.
+        """
+        if self._cancel_event.is_set():
+            return True
+
+        if self._should_cancel:
+            try:
+                # Handle both sync and async callbacks
+                result = self._should_cancel()
+                if asyncio.iscoroutine(result):
+                    result = await result
+
+                if result:
+                    self._cancel_event.set()
+                    self.stats.end_time = datetime.now()
+                    return True
+            except Exception as e:
+                # Fail-open: log warning and continue crawling
+                self.logger.warning(f"should_cancel callback error: {e}")
+
+        return False
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_level: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extracts links from the crawl result, validates and scores them, and
+        prepares the next level of URLs.
+        Each valid URL is appended to next_level as a tuple (url, parent_url)
+        and its depth is tracked.
+        """            
+        next_depth = current_depth + 1
+        if next_depth > self.max_depth:
+            return
+
+        # If we've reached the max pages limit, don't discover new links
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            return
+
+        # Get internal links and, if enabled, external links.
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        valid_links = []
+        
+        # First collect all valid links
+        for link in links:
+            url = link.get("href")
+            # Strip URL fragments to avoid duplicate crawling
+            # base_url = url.split('#')[0] if url else url
+            base_url = normalize_url_for_deep_crawl(url, source_url)
+            if base_url in visited:
+                continue
+            if not await self.can_process_url(base_url, next_depth):
+                self.stats.urls_skipped += 1
+                continue
+
+            # Score the URL if a scorer is provided
+            score = self.url_scorer.score(base_url) if self.url_scorer else 0
+            
+            # Skip URLs with scores below the threshold
+            if score < self.score_threshold:
+                self.logger.debug(f"URL {url} skipped: score {score} below threshold {self.score_threshold}")
+                self.stats.urls_skipped += 1
+                continue
+
+            visited.add(base_url)
+            valid_links.append((base_url, score))
+        
+        # If we have more valid links than capacity, sort by score and take the top ones
+        if len(valid_links) > remaining_capacity:
+            if self.url_scorer:
+                # Sort by score in descending order
+                valid_links.sort(key=lambda x: x[1], reverse=True)
+            # Take only as many as we have capacity for
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
+            
+        # Process the final selected links
+        for url, score in valid_links:
+            # attach the score to metadata if needed
+            if score:
+                result.metadata = result.metadata or {}
+                result.metadata["score"] = score
+            next_level.append((url, source_url))
+            depths[url] = next_depth
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) mode:
+        Processes one BFS level at a time, then yields all the results.
+        """
+        # Reset cancel event for strategy reuse
+        self._cancel_event = asyncio.Event()
+
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            current_level = [
+                (item["url"], item["parent_url"])
+                for item in self._resume_state.get("pending", [])
+            ]
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+        else:
+            # Original initialization
+            visited: Set[str] = set()
+            # current_level holds tuples: (url, parent_url)
+            current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+            depths: Dict[str, int] = {start_url: 0}
+
+        results: List[CrawlResult] = []
+
+        while current_level and not self._cancel_event.is_set():
+            # Check if we've already reached max_pages before starting a new level
+            if self._pages_crawled >= self.max_pages:
+                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                break
+
+            # Check external cancellation callback before processing this level
+            if await self._check_cancellation():
+                self.logger.info("Crawl cancelled by user")
+                break
+
+            next_level: List[Tuple[str, Optional[str]]] = []
+            urls = [url for url, _ in current_level]
+
+            # Clone the config to disable deep crawling recursion and enforce batch mode.
+            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
+            batch_results = await crawler.arun_many(urls=urls, config=batch_config)
+
+            for result in batch_results:
+                url = result.url
+                depth = depths.get(url, 0)
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                result.metadata["parent_url"] = parent_url
+                results.append(result)
+
+                # Only discover links from successful crawls
+                if result.success:
+                    # Increment pages crawled per URL for accurate state tracking
+                    self._pages_crawled += 1
+
+                    # Link discovery will handle the max pages limit internally
+                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+
+                    # Capture state after EACH URL processed (if callback set)
+                    if self._on_state_change:
+                        state = {
+                            "strategy_type": "bfs",
+                            "visited": list(visited),
+                            "pending": [{"url": u, "parent_url": p} for u, p in next_level],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                            "cancelled": self._cancel_event.is_set(),
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
+
+            current_level = next_level
+
+        # Final state update if cancelled
+        if self._cancel_event.is_set() and self._on_state_change:
+            state = {
+                "strategy_type": "bfs",
+                "visited": list(visited),
+                "pending": [{"url": u, "parent_url": p} for u, p in current_level],
+                "depths": depths,
+                "pages_crawled": self._pages_crawled,
+                "cancelled": True,
+            }
+            self._last_state = state
+            await self._on_state_change(state)
+
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming mode:
+        Processes one BFS level at a time and yields results immediately as they arrive.
+        """
+        # Reset cancel event for strategy reuse
+        self._cancel_event = asyncio.Event()
+
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            current_level = [
+                (item["url"], item["parent_url"])
+                for item in self._resume_state.get("pending", [])
+            ]
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+        else:
+            # Original initialization
+            visited: Set[str] = set()
+            current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+            depths: Dict[str, int] = {start_url: 0}
+
+        while current_level and not self._cancel_event.is_set():
+            # Check external cancellation callback before processing this level
+            if await self._check_cancellation():
+                self.logger.info("Crawl cancelled by user")
+                break
+
+            next_level: List[Tuple[str, Optional[str]]] = []
+            urls = [url for url, _ in current_level]
+            visited.update(urls)
+
+            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=urls, config=stream_config)
+            
+            # Keep track of processed results for this batch
+            results_count = 0
+            async for result in stream_gen:
+                url = result.url
+                depth = depths.get(url, 0)
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                result.metadata["parent_url"] = parent_url
+                
+                # Count only successful crawls
+                if result.success:
+                    self._pages_crawled += 1
+                    # Check if we've reached the limit during batch processing
+                    if self._pages_crawled >= self.max_pages:
+                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                        break  # Exit the generator
+                
+                results_count += 1
+                yield result
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Link discovery will handle the max pages limit internally
+                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+
+                    # Capture state after EACH URL processed (if callback set)
+                    if self._on_state_change:
+                        state = {
+                            "strategy_type": "bfs",
+                            "visited": list(visited),
+                            "pending": [{"url": u, "parent_url": p} for u, p in next_level],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                            "cancelled": self._cancel_event.is_set(),
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
+
+            # If we didn't get results back (e.g. due to errors), avoid getting stuck in an infinite loop
+            # by considering these URLs as visited but not counting them toward the max_pages limit
+            if results_count == 0 and urls:
+                self.logger.warning(f"No results returned for {len(urls)} URLs, marking as visited")
+
+            current_level = next_level
+
+        # Final state update if cancelled
+        if self._cancel_event.is_set() and self._on_state_change:
+            state = {
+                "strategy_type": "bfs",
+                "visited": list(visited),
+                "pending": [{"url": u, "parent_url": p} for u, p in current_level],
+                "depths": depths,
+                "pages_crawled": self._pages_crawled,
+                "cancelled": True,
+            }
+            self._last_state = state
+            await self._on_state_change(state)
+
+    async def shutdown(self) -> None:
+        """
+        Clean up resources and signal cancellation of the crawl.
+        """
+        self._cancel_event.set()
+        self.stats.end_time = datetime.now()
+
+    def export_state(self) -> Optional[Dict[str, Any]]:
+        """
+        Export current crawl state for external persistence.
+
+        Note: This returns the last captured state. For real-time state,
+        use the on_state_change callback.
+
+        Returns:
+            Dict with strategy state, or None if no state captured yet.
+        """
+        return self._last_state
diff --git a/crawl4ai/deep_crawling/crazy.py b/crawl4ai/deep_crawling/crazy.py
new file mode 100644
index 0000000..d2bc27e
--- /dev/null
+++ b/crawl4ai/deep_crawling/crazy.py
@@ -0,0 +1,432 @@
+from __future__ import annotations
+# I just got crazy, trying to wrute K&R C but in Python. Right now I feel like I'm in a quantum state.
+# I probably won't use this; I just want to leave it here. A century later, the future human race will be like, "WTF?"
+
+# ------ Imports That Will Make You Question Reality ------ #
+from functools import wraps
+from contextvars import ContextVar
+import inspect
+
+from crawl4ai import CacheMode
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai.models import CrawlResult, TraversalStats
+from crawl4ai.deep_crawling.filters import FilterChain
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+import time
+import logging
+from urllib.parse import urlparse
+
+from abc import ABC, abstractmethod
+from collections import deque
+import asyncio
+from typing import (
+    AsyncGenerator,
+    Dict,
+    List,
+    TypeVar,
+    Generic,
+    Tuple,
+    Callable,
+    Awaitable,
+    Union,
+)
+from functools import lru_cache
+import mmh3
+from bitarray import bitarray
+import numpy as np
+from heapq import heappush, heappop
+
+# ------ Type Algebra Mastery ------ #
+CrawlResultT = TypeVar("CrawlResultT", bound="CrawlResult")
+PriorityT = TypeVar("PriorityT")
+P = TypeVar("P")
+
+# ------ Hyperscalar Context Management ------ #
+deep_crawl_ctx = ContextVar("deep_crawl_stack", default=deque())
+
+# ------ Algebraic Crawler Monoid ------ #
+class TraversalContext:
+    __slots__ = ('visited', 'frontier', 'depths', 'priority_fn', 'current_depth')
+    
+    def __init__(self,
+                 priority_fn: Callable[[str], Awaitable[float]] = lambda _: 1.0):
+        self.visited: BloomFilter = BloomFilter(10**6, 0.01)  # 1M items, 1% FP
+        self.frontier: PriorityQueue = PriorityQueue()
+        self.depths: Dict[str, int] = {}
+        self.priority_fn = priority_fn
+        self.current_depth = 0
+
+    def clone_for_level(self) -> TraversalContext:
+        """Monadic context propagation"""
+        new_ctx = TraversalContext(self.priority_fn)
+        new_ctx.visited = self.visited.copy()
+        new_ctx.depths = self.depths.copy()
+        new_ctx.current_depth = self.current_depth
+        return new_ctx
+
+class PriorityQueue(Generic[PriorityT]):
+    """Fibonacci heap-inspired priority queue with O(1) amortized operations"""
+    __slots__ = ('_heap', '_index')
+
+    def __init__(self):
+        self._heap: List[Tuple[PriorityT, float, P]] = []
+        self._index: Dict[P, int] = {}
+
+    def insert(self, priority: PriorityT, item: P) -> None:
+        tiebreaker = time.time()  # Ensure FIFO for equal priorities
+        heappush(self._heap, (priority, tiebreaker, item))
+        self._index[item] = len(self._heap) - 1
+
+    def extract(self, top_n = 1) -> P:
+        items = []
+        for _ in range(top_n):
+            if not self._heap:
+                break
+            priority, _, item = heappop(self._heap)
+            del self._index[item]
+            items.append(item)
+        if not items:
+            raise IndexError("Priority queue empty")
+        return items
+        # while self._heap:
+        #     _, _, item = heappop(self._heap)
+        #     if item in self._index:
+        #         del self._index[item]
+        #         return item
+        raise IndexError("Priority queue empty")
+
+
+    def is_empty(self) -> bool:
+        return not bool(self._heap)
+
+class BloomFilter:
+    """Optimal Bloom filter using murmur3 hash avalanche"""
+    __slots__ = ('size', 'hashes', 'bits')
+
+    def __init__(self, capacity: int, error_rate: float):
+        self.size = self._optimal_size(capacity, error_rate)
+        self.hashes = self._optimal_hashes(capacity, self.size)
+        self.bits = bitarray(self.size)
+        self.bits.setall(False)
+
+    @staticmethod
+    def _optimal_size(n: int, p: float) -> int:
+        m = - (n * np.log(p)) / (np.log(2) ** 2)
+        return int(np.ceil(m))
+
+    @staticmethod
+    def _optimal_hashes(n: int, m: int) -> int:
+        k = (m / n) * np.log(2)
+        return int(np.ceil(k))
+
+    def add(self, item: str) -> None:
+        for seed in range(self.hashes):
+            digest = mmh3.hash(item, seed) % self.size
+            self.bits[digest] = True
+
+    def __contains__(self, item: str) -> bool:
+        return all(
+            self.bits[mmh3.hash(item, seed) % self.size]
+            for seed in range(self.hashes)
+        )
+
+    def copy(self) -> BloomFilter:
+        new = object.__new__(BloomFilter)
+        new.size = self.size
+        new.hashes = self.hashes
+        new.bits = self.bits.copy()
+        return new
+    
+    def __len__(self) -> int:
+        """
+        Estimates the number of items in the filter using the 
+        count of set bits and the formula:
+        n = -m/k * ln(1 - X/m)
+        where:
+            m = size of bit array
+            k = number of hash functions
+            X = count of set bits
+        """
+        set_bits = self.bits.count(True)
+        if set_bits == 0:
+            return 0
+            
+        # Use the inverse bloom filter formula to estimate cardinality
+        return int(
+            -(self.size / self.hashes) * 
+            np.log(1 - set_bits / self.size)
+        )
+    
+    def bit_count(self) -> int:
+        """Returns the raw count of set bits in the filter"""
+        return self.bits.count(True)
+        
+    def __repr__(self) -> str:
+        return f"BloomFilter(est_items={len(self)}, bits={self.bit_count()}/{self.size})"
+
+# ------ Hyper-Optimal Deep Crawl Core ------ #
+class DeepCrawlDecorator:
+    """Metaprogramming marvel: Zero-cost deep crawl abstraction"""
+    def __init__(self, crawler: AsyncWebCrawler):
+        self.crawler = crawler
+
+    def __call__(self, original_arun: Callable) -> Callable:
+        @wraps(original_arun)
+        async def quantum_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
+            stack = deep_crawl_ctx.get()
+            if config and config.deep_crawl_strategy and not stack:
+                stack.append(self.crawler)
+                try:
+                    deep_crawl_ctx.set(stack)
+                    async for result in config.deep_crawl_strategy.traverse(
+                        start_url=url,
+                        crawler=self.crawler,
+                        config=config
+                    ):
+                        yield result
+                finally:
+                    stack.pop()
+                    deep_crawl_ctx.set(stack)
+            else:
+                result = await original_arun(url, config=config, **kwargs)
+                yield result
+        return quantum_arun
+
+
+async def collect_results(url, crawler, config):
+    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
+        setattr(crawler, "arun", getattr(crawler, "original_arun"))
+
+    ret = crawler.arun(url, config=config)
+    # If arun is an async generator, iterate over it
+    if inspect.isasyncgen(ret):
+        return [r async for r in ret]
+    # Otherwise, await the coroutine and normalize to a list
+    result = await ret
+    return result if isinstance(result, list) else [result]
+
+async def collect_many_results(url, crawler, config):
+    # Replace back arun to its original implementation
+    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
+        setattr(crawler, "arun", getattr(crawler, "original_arun"))
+    ret = crawler.arun_many(url, config=config)
+    # If arun is an async generator, iterate over it
+    if inspect.isasyncgen(ret):
+        return [r async for r in ret]
+    # Otherwise, await the coroutine and normalize to a list
+    result = await ret
+    return result if isinstance(result, list) else [result]
+
+
+# ------ Deep Crawl Strategy Interface ------ #
+CrawlResultT = TypeVar("CrawlResultT", bound=CrawlResult)
+# In batch mode we return List[CrawlResult] and in stream mode an AsyncGenerator.
+RunManyReturn = Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+
+
+class DeepCrawlStrategy(ABC):
+    """Abstract base class that will make Dijkstra smile"""
+    @abstractmethod
+    async def traverse(self,
+                      start_url: str,
+                      crawler: AsyncWebCrawler,
+                      config: CrawlerRunConfig) -> RunManyReturn:
+        """Traverse with O(1) memory complexity via generator fusion"""
+        ...
+
+    @abstractmethod
+    def precompute_priority(self, url: str) -> Awaitable[float]:
+        """Quantum-inspired priority precomputation"""
+        pass
+
+    @abstractmethod
+    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
+        """Hilbert-curve optimized link generation"""
+        pass
+
+# ------ BFS That Would Make Knuth Proud ------ #
+
+def calculate_quantum_batch_size(
+    depth: int,
+    max_depth: int,
+    frontier_size: int,
+    visited_size: int
+) -> int:
+    """
+    Calculates optimal batch size for URL processing using quantum-inspired mathematical principles.
+    
+    This function implements a sophisticated batch size calculation using:
+    1. Golden Ratio (φ) based scaling for optimal irrationality
+    2. Depth-aware amplitude modulation
+    3. Harmonic series dampening
+    4. Logarithmic growth control
+    5. Dynamic frontier adaptation
+    
+    The formula follows the quantum harmonic oscillator principle:
+        N = ⌈φ^(2d) * log₂(|V|) * H(d)⁻¹ * min(20, |F|/10)⌉
+    where:
+        φ = Golden Ratio ((1 + √5) / 2)
+        d = depth factor (normalized remaining depth)
+        |V| = size of visited set
+        H(d) = d-th harmonic number
+        |F| = frontier size
+    
+    Args:
+        depth (int): Current traversal depth
+        max_depth (int): Maximum allowed depth
+        frontier_size (int): Current size of frontier queue
+        visited_size (int): Number of URLs visited so far
+    
+    Returns:
+        int: Optimal batch size bounded between 1 and 100
+        
+    Mathematical Properties:
+        - Maintains O(log n) growth with respect to visited size
+        - Provides φ-optimal distribution of resources
+        - Ensures quantum-like state transitions between depths
+        - Harmonically dampened to prevent exponential explosion
+    """
+    # Golden ratio φ = (1 + √5) / 2
+    φ = (1 + 5 ** 0.5) / 2
+    
+    # Calculate normalized depth factor [0, 1]
+    depth_factor = (max_depth - depth) / max_depth if depth < max_depth else 0
+    
+    # Compute harmonic number for current depth
+    harmonic = sum(1/k for k in range(1, depth + 2))
+    
+    # Calculate quantum batch size
+    batch_size = int(np.ceil(
+        (φ ** (depth_factor * 2)) *          # Golden ratio scaling
+        np.log2(visited_size + 2) *          # Logarithmic growth factor
+        (1 / harmonic) *                     # Harmonic dampening
+        max(1, min(20, frontier_size / 10))  # Frontier-aware scaling
+    ))
+    
+    # Enforce practical bounds
+    return max(1, min(100, batch_size))
+
+
+class BFSDeepCrawlStrategy(DeepCrawlStrategy):
+    """Breadth-First Search with Einstein-Rosen bridge optimization"""
+    __slots__ = ('max_depth', 'filter_chain', 'priority_fn', 'stats', '_cancel')
+
+    def __init__(self,
+                 max_depth: int,
+                 filter_chain: FilterChain = FilterChain(),
+                 priority_fn: Callable[[str], Awaitable[float]] = lambda url: 1.0,
+                 logger: logging.Logger = None):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.priority_fn = priority_fn
+        self.stats = TraversalStats()
+        self._cancel = asyncio.Event()
+        self.semaphore = asyncio.Semaphore(1000)
+
+    async def traverse(self,
+                      start_url: str,
+                      crawler: AsyncWebCrawler,
+                      config: CrawlerRunConfig) -> RunManyReturn:
+        """Non-blocking BFS with O(b^d) time complexity awareness"""
+        ctx = TraversalContext(self.priority_fn)
+        ctx.frontier.insert(self.priority_fn(start_url), (start_url, None, 0))
+        ctx.visited.add(start_url)
+        ctx.depths[start_url] = 0
+
+        while not ctx.frontier.is_empty() and not self._cancel.is_set():
+            # Use the best algorith, to find top_n value
+            top_n = calculate_quantum_batch_size(
+                depth=ctx.current_depth,
+                max_depth=self.max_depth,
+                frontier_size=len(ctx.frontier._heap),
+                visited_size=len(ctx.visited)
+            )
+
+            urls = ctx.frontier.extract(top_n=top_n)
+            # url, parent, depth = ctx.frontier.extract(top_n=top_n)
+            if urls:
+                ctx.current_depth = urls[0][2]
+
+            async with self.semaphore:
+                results = await collect_many_results([url for (url, parent, depth) in urls], crawler, config)
+                # results = await asyncio.gather(*[
+                #     collect_results(url, crawler, config) for (url, parent, depth) in urls
+                # ])
+                # result = _result[0]
+                for ix, result in enumerate(results):
+                    url, parent, depth = result.url, urls[ix][1], urls[ix][2]
+                    result.metadata['depth'] = depth
+                    result.metadata['parent'] = parent
+                    yield result
+
+                    if depth < self.max_depth:
+                        async for link in self.link_hypercube(result):
+                            if link not in ctx.visited:
+                                priority = self.priority_fn(link)
+                                ctx.frontier.insert(priority, (link, url, depth + 1))
+                                ctx.visited.add(link)
+                                ctx.depths[link] = depth + 1
+
+    @lru_cache(maxsize=65536)
+    async def validate_url(self, url: str) -> bool:
+        """Memoized URL validation with λ-calculus purity"""
+        try:
+            parsed = urlparse(url)
+            return (parsed.scheme in {'http', 'https'}
+                    and '.' in parsed.netloc
+                    and await self.filter_chain.apply(url))
+        except Exception:
+            return False
+
+    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
+        """Hilbert-ordered link generation with O(1) yield latency"""
+        links = (link['href'] for link in result.links.get('internal', []))
+        validated = filter(self.validate_url, links)
+        for link in sorted(validated, key=lambda x: -self.priority_fn(x)):
+            yield link
+
+    def __aiter__(self) -> AsyncGenerator[CrawlResult, None]:
+        """Native async iterator interface"""
+        return self.traverse()
+
+    async def __anext__(self) -> CrawlResult:
+        """True async iterator protocol implementation"""
+        result = await self.traverse().__anext__()
+        if result:
+            return result
+        raise StopAsyncIteration
+
+    async def precompute_priority(self, url):
+        return super().precompute_priority(url)
+
+    async def shutdown(self):
+        self._cancel.set()
+
+# ------ Usage That Will Drop Jaws ------ #
+async def main():
+    """Quantum crawl example"""
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=2,
+        priority_fn=lambda url: 1.0 / (len(url) + 1e-9),  # Inverse length priority
+        # filter_chain=FilterChain(...)
+    )
+
+    config: CrawlerRunConfig = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        stream=False,
+        verbose=True,
+        cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        run_decorator = DeepCrawlDecorator(crawler)
+        setattr(crawler, "original_arun", crawler.arun)
+        crawler.arun = run_decorator(crawler.arun)
+        start_time = time.perf_counter()
+        async for result in crawler.arun("https://docs.crawl4ai.com", config=config):
+            print(f"🌀 {result.url} (Depth: {result.metadata['depth']})")
+        print(f"Deep crawl completed in {time.perf_counter() - start_time:.2f}s")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/crawl4ai/deep_crawling/dfs_strategy.py b/crawl4ai/deep_crawling/dfs_strategy.py
new file mode 100644
index 0000000..3e4987f
--- /dev/null
+++ b/crawl4ai/deep_crawling/dfs_strategy.py
@@ -0,0 +1,331 @@
+# dfs_deep_crawl_strategy.py
+import asyncio
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+
+from ..models import CrawlResult
+from .bfs_strategy import BFSDeepCrawlStrategy  # noqa
+from ..types import AsyncWebCrawler, CrawlerRunConfig
+from ..utils import normalize_url_for_deep_crawl
+
+class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
+    """
+    Depth-first deep crawling with familiar BFS rules.
+
+    We reuse the same filters, scoring, and page limits from :class:`BFSDeepCrawlStrategy`,
+    but walk the graph with a stack so we fully explore one branch before hopping to the
+    next. DFS also keeps its own ``_dfs_seen`` set so we can drop duplicate links at
+    discovery time without accidentally marking them as “already crawled”.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._dfs_seen: Set[str] = set()
+
+    def _reset_seen(self, start_url: str) -> None:
+        """Start each crawl with a clean dedupe set seeded with the root URL."""
+        self._dfs_seen = {start_url}
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Crawl level-by-level but emit results at the end.
+
+        We keep a stack of ``(url, parent, depth)`` tuples, pop one at a time, and
+        hand it to ``crawler.arun_many`` with deep crawling disabled so we remain
+        in control of traversal. Every successful page bumps ``_pages_crawled`` and
+        seeds new stack items discovered via :meth:`link_discovery`.
+        """
+        # Reset cancel event for strategy reuse
+        self._cancel_event = asyncio.Event()
+
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            stack = [
+                (item["url"], item["parent_url"], item["depth"])
+                for item in self._resume_state.get("stack", [])
+            ]
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+            self._dfs_seen = set(self._resume_state.get("dfs_seen", []))
+            results: List[CrawlResult] = []
+        else:
+            # Original initialization
+            visited: Set[str] = set()
+            # Stack items: (url, parent_url, depth)
+            stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+            depths: Dict[str, int] = {start_url: 0}
+            results: List[CrawlResult] = []
+            self._reset_seen(start_url)
+
+        while stack and not self._cancel_event.is_set():
+            # Check external cancellation callback before processing this URL
+            if await self._check_cancellation():
+                self.logger.info("Crawl cancelled by user")
+                break
+
+            url, parent, depth = stack.pop()
+            if url in visited or depth > self.max_depth:
+                continue
+            visited.add(url)
+
+            # Clone config to disable recursive deep crawling.
+            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
+            url_results = await crawler.arun_many(urls=[url], config=batch_config)
+            
+            for result in url_results:
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent
+                if self.url_scorer:
+                    result.metadata["score"] = self.url_scorer.score(url)
+                results.append(result)
+                
+                # Count only successful crawls toward max_pages limit
+                if result.success:
+                    self._pages_crawled += 1
+                    # Check if we've reached the limit during batch processing
+                    if self._pages_crawled >= self.max_pages:
+                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                        break  # Exit the generator
+                    
+                    # Only discover links from successful crawls
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    
+                    # Push new links in reverse order so the first discovered is processed next.
+                    for new_url, new_parent in reversed(new_links):
+                        new_depth = depths.get(new_url, depth + 1)
+                        stack.append((new_url, new_parent, new_depth))
+
+                    # Capture state after each URL processed (if callback set)
+                    if self._on_state_change:
+                        state = {
+                            "strategy_type": "dfs",
+                            "visited": list(visited),
+                            "stack": [
+                                {"url": u, "parent_url": p, "depth": d}
+                                for u, p, d in stack
+                            ],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                            "dfs_seen": list(self._dfs_seen),
+                            "cancelled": self._cancel_event.is_set(),
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
+
+        # Final state update if cancelled
+        if self._cancel_event.is_set() and self._on_state_change:
+            state = {
+                "strategy_type": "dfs",
+                "visited": list(visited),
+                "stack": [
+                    {"url": u, "parent_url": p, "depth": d}
+                    for u, p, d in stack
+                ],
+                "depths": depths,
+                "pages_crawled": self._pages_crawled,
+                "dfs_seen": list(self._dfs_seen),
+                "cancelled": True,
+            }
+            self._last_state = state
+            await self._on_state_change(state)
+
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Same traversal as :meth:`_arun_batch`, but yield pages immediately.
+
+        Each popped URL is crawled, its metadata annotated, then the result gets
+        yielded before we even look at the next stack entry. Successful crawls
+        still feed :meth:`link_discovery`, keeping DFS order intact.
+        """
+        # Reset cancel event for strategy reuse
+        self._cancel_event = asyncio.Event()
+
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            stack = [
+                (item["url"], item["parent_url"], item["depth"])
+                for item in self._resume_state.get("stack", [])
+            ]
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+            self._dfs_seen = set(self._resume_state.get("dfs_seen", []))
+        else:
+            # Original initialization
+            visited: Set[str] = set()
+            stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+            depths: Dict[str, int] = {start_url: 0}
+            self._reset_seen(start_url)
+
+        while stack and not self._cancel_event.is_set():
+            # Check external cancellation callback before processing this URL
+            if await self._check_cancellation():
+                self.logger.info("Crawl cancelled by user")
+                break
+
+            url, parent, depth = stack.pop()
+            if url in visited or depth > self.max_depth:
+                continue
+            visited.add(url)
+
+            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=[url], config=stream_config)
+            async for result in stream_gen:
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent
+                if self.url_scorer:
+                    result.metadata["score"] = self.url_scorer.score(url)
+                yield result
+
+                # Only count successful crawls toward max_pages limit
+                # and only discover links from successful crawls
+                if result.success:
+                    self._pages_crawled += 1
+                    # Check if we've reached the limit during batch processing
+                    if self._pages_crawled >= self.max_pages:
+                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                        break  # Exit the generator
+                    
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    for new_url, new_parent in reversed(new_links):
+                        new_depth = depths.get(new_url, depth + 1)
+                        stack.append((new_url, new_parent, new_depth))
+
+                    # Capture state after each URL processed (if callback set)
+                    if self._on_state_change:
+                        state = {
+                            "strategy_type": "dfs",
+                            "visited": list(visited),
+                            "stack": [
+                                {"url": u, "parent_url": p, "depth": d}
+                                for u, p, d in stack
+                            ],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                            "dfs_seen": list(self._dfs_seen),
+                            "cancelled": self._cancel_event.is_set(),
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
+
+        # Final state update if cancelled
+        if self._cancel_event.is_set() and self._on_state_change:
+            state = {
+                "strategy_type": "dfs",
+                "visited": list(visited),
+                "stack": [
+                    {"url": u, "parent_url": p, "depth": d}
+                    for u, p, d in stack
+                ],
+                "depths": depths,
+                "pages_crawled": self._pages_crawled,
+                "dfs_seen": list(self._dfs_seen),
+                "cancelled": True,
+            }
+            self._last_state = state
+            await self._on_state_change(state)
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        _visited: Set[str],
+        next_level: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Find the next URLs we should push onto the DFS stack.
+
+        Parameters
+        ----------
+        result : CrawlResult
+            Output of the page we just crawled; its ``links`` block is our raw material.
+        source_url : str
+            URL of the parent page; stored so callers can track ancestry.
+        current_depth : int
+            Depth of the parent; children naturally sit at ``current_depth + 1``.
+        _visited : Set[str]
+            Present to match the BFS signature, but we rely on ``_dfs_seen`` instead.
+        next_level : list of tuples
+            The stack buffer supplied by the caller; we append new ``(url, parent)`` items here.
+        depths : dict
+            Shared depth map so future metadata tagging knows how deep each URL lives.
+
+        Notes
+        -----
+        - ``_dfs_seen`` keeps us from pushing duplicates without touching the traversal guard.
+        - Validation, scoring, and capacity trimming mirror the BFS version so behaviour stays consistent.
+        """
+        next_depth = current_depth + 1
+        if next_depth > self.max_depth:
+            return
+
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(
+                f"Max pages limit ({self.max_pages}) reached, stopping link discovery"
+            )
+            return
+
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        seen = self._dfs_seen
+        valid_links: List[Tuple[str, float]] = []
+
+        for link in links:
+            raw_url = link.get("href")
+            if not raw_url:
+                continue
+
+            normalized_url = normalize_url_for_deep_crawl(raw_url, source_url)
+            if not normalized_url or normalized_url in seen:
+                continue
+
+            if not await self.can_process_url(normalized_url, next_depth):
+                self.stats.urls_skipped += 1
+                continue
+
+            score = self.url_scorer.score(normalized_url) if self.url_scorer else 0
+            if score < self.score_threshold:
+                self.logger.debug(
+                    f"URL {normalized_url} skipped: score {score} below threshold {self.score_threshold}"
+                )
+                self.stats.urls_skipped += 1
+                continue
+
+            seen.add(normalized_url)
+            valid_links.append((normalized_url, score))
+
+        if len(valid_links) > remaining_capacity:
+            if self.url_scorer:
+                valid_links.sort(key=lambda x: x[1], reverse=True)
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(
+                f"Limiting to {remaining_capacity} URLs due to max_pages limit"
+            )
+
+        for url, score in valid_links:
+            if score:
+                result.metadata = result.metadata or {}
+                result.metadata["score"] = score
+            next_level.append((url, source_url))
+            depths[url] = next_depth
diff --git a/crawl4ai/deep_crawling/filters.py b/crawl4ai/deep_crawling/filters.py
new file mode 100644
index 0000000..2fb819e
--- /dev/null
+++ b/crawl4ai/deep_crawling/filters.py
@@ -0,0 +1,691 @@
+from abc import ABC, abstractmethod
+from typing import List, Pattern, Set, Union
+from urllib.parse import urlparse
+from array import array
+import re
+import logging
+from functools import lru_cache
+import fnmatch
+from dataclasses import dataclass
+import weakref
+import math
+from collections import defaultdict
+from typing import Dict
+from ..utils import HeadPeekr
+import asyncio
+import inspect
+
+
+@dataclass
+class FilterStats:
+    __slots__ = ("_counters",)
+
+    def __init__(self):
+        # Use array of unsigned ints for atomic operations
+        self._counters = array("I", [0, 0, 0])  # total, passed, rejected
+
+    @property
+    def total_urls(self):
+        return self._counters[0]
+
+    @property
+    def passed_urls(self):
+        return self._counters[1]
+
+    @property
+    def rejected_urls(self):
+        return self._counters[2]
+
+
+class URLFilter(ABC):
+    """Optimized base filter class"""
+
+    __slots__ = ("name", "stats", "_logger_ref")
+
+    def __init__(self, name: str = None):
+        self.name = name or self.__class__.__name__
+        self.stats = FilterStats()
+        # Lazy logger initialization using weakref
+        self._logger_ref = None
+
+    @property
+    def logger(self):
+        if self._logger_ref is None or self._logger_ref() is None:
+            logger = logging.getLogger(f"urlfilter.{self.name}")
+            self._logger_ref = weakref.ref(logger)
+        return self._logger_ref()
+
+    @abstractmethod
+    def apply(self, url: str) -> bool:
+        pass
+
+    def _update_stats(self, passed: bool):
+        # Use direct array index for speed
+        self.stats._counters[0] += 1  # total
+        self.stats._counters[1] += passed  # passed
+        self.stats._counters[2] += not passed  # rejected
+
+
+class FilterChain:
+    """Optimized filter chain"""
+
+    __slots__ = ("filters", "stats", "_logger_ref")
+
+    def __init__(self, filters: List[URLFilter] = None):
+        self.filters = tuple(filters or [])  # Immutable tuple for speed
+        self.stats = FilterStats()
+        self._logger_ref = None
+
+    @property
+    def logger(self):
+        if self._logger_ref is None or self._logger_ref() is None:
+            logger = logging.getLogger("urlfilter.chain")
+            self._logger_ref = weakref.ref(logger)
+        return self._logger_ref()
+
+    def add_filter(self, filter_: URLFilter) -> "FilterChain":
+        """Add a filter to the chain"""
+        self.filters = self.filters + (filter_,)
+        return self  # Enable method chaining
+
+    async def apply(self, url: str) -> bool:
+        """Apply all filters concurrently when possible"""
+        self.stats._counters[0] += 1  # Total processed URLs
+
+        tasks = []
+        for f in self.filters:
+            result = f.apply(url)
+
+            if inspect.isawaitable(result):
+                tasks.append(result)  # Collect async tasks
+            elif not result:  # Sync rejection
+                self.stats._counters[2] += 1  # Sync rejected
+                return False
+
+        if tasks:
+            results = await asyncio.gather(*tasks)
+
+            # Count how many filters rejected
+            rejections = results.count(False)
+            self.stats._counters[2] += rejections
+
+            if not all(results):
+                return False  # Stop early if any filter rejected
+
+        self.stats._counters[1] += 1  # Passed
+        return True
+
+
+class URLPatternFilter(URLFilter):
+    """Pattern filter balancing speed and completeness"""
+
+    __slots__ = (
+        "patterns",  # Store original patterns for serialization
+        "use_glob",  # Store original use_glob for serialization  
+        "reverse",   # Store original reverse for serialization
+        "_simple_suffixes",
+        "_simple_prefixes",
+        "_domain_patterns",
+        "_path_patterns",
+        "_reverse",
+    )
+
+    PATTERN_TYPES = {
+        "SUFFIX": 1,  # *.html
+        "PREFIX": 2,  # /foo/*
+        "DOMAIN": 3,  # *.example.com
+        "PATH": 4,  # Everything else
+        "REGEX": 5,
+    }
+
+    def __init__(
+        self,
+        patterns: Union[str, Pattern, List[Union[str, Pattern]]],
+        use_glob: bool = True,
+        reverse: bool = False,
+    ):
+        super().__init__()
+        # Store original constructor params for serialization
+        self.patterns = patterns
+        self.use_glob = use_glob
+        self.reverse = reverse
+        
+        self._reverse = reverse
+        patterns = [patterns] if isinstance(patterns, (str, Pattern)) else patterns
+
+        self._simple_suffixes = set()
+        self._simple_prefixes = set()
+        self._domain_patterns = []
+        self._path_patterns = []
+
+        for pattern in patterns:
+            pattern_type = self._categorize_pattern(pattern)
+            self._add_pattern(pattern, pattern_type)
+
+    def _categorize_pattern(self, pattern: str) -> int:
+        """Categorize pattern for specialized handling"""
+        if not isinstance(pattern, str):
+            return self.PATTERN_TYPES["PATH"]
+
+        # Check if it's a regex pattern
+        if pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern:
+            return self.PATTERN_TYPES["REGEX"]
+
+        if pattern.count("*") == 1:
+            if pattern.startswith("*."):
+                return self.PATTERN_TYPES["SUFFIX"]
+            if pattern.endswith("/*"):
+                return self.PATTERN_TYPES["PREFIX"]
+
+        if "://" in pattern and pattern.startswith("*."):
+            return self.PATTERN_TYPES["DOMAIN"]
+
+        return self.PATTERN_TYPES["PATH"]
+
+    def _add_pattern(self, pattern: str, pattern_type: int):
+        """Add pattern to appropriate matcher"""
+        if pattern_type == self.PATTERN_TYPES["REGEX"]:
+            # For regex patterns, compile directly without glob translation
+            if isinstance(pattern, str) and (
+                pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern
+            ):
+                self._path_patterns.append(re.compile(pattern))
+                return
+        elif pattern_type == self.PATTERN_TYPES["SUFFIX"]:
+            self._simple_suffixes.add(pattern[2:])
+        elif pattern_type == self.PATTERN_TYPES["PREFIX"]:
+            self._simple_prefixes.add(pattern[:-2])
+        elif pattern_type == self.PATTERN_TYPES["DOMAIN"]:
+            self._domain_patterns.append(re.compile(pattern.replace("*.", r"[^/]+\.")))
+        else:
+            if isinstance(pattern, str):
+                # Handle complex glob patterns
+                if "**" in pattern:
+                    pattern = pattern.replace("**", ".*")
+                if "{" in pattern:
+                    # Convert {a,b} to (a|b)
+                    pattern = re.sub(
+                        r"\{([^}]+)\}",
+                        lambda m: f'({"|".join(m.group(1).split(","))})',
+                        pattern,
+                    )
+                pattern = fnmatch.translate(pattern)
+            self._path_patterns.append(
+                pattern if isinstance(pattern, Pattern) else re.compile(pattern)
+            )
+
+    @lru_cache(maxsize=10000)
+    def apply(self, url: str) -> bool:
+        url_path = urlparse(url).path
+
+        # Quick suffix check (*.html)
+        if self._simple_suffixes:
+            if url_path.split("/")[-1].split(".")[-1] in self._simple_suffixes:
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
+
+        # Domain check
+        if self._domain_patterns:
+            for pattern in self._domain_patterns:
+                if pattern.match(url):
+                    result = True
+                    self._update_stats(result)
+                    return not result if self._reverse else result
+
+        # Prefix check (/foo/* or https://domain/foo/*)
+        if self._simple_prefixes:
+            for prefix in self._simple_prefixes:
+                # Use url_path for path-only prefixes, full URL for absolute prefixes
+                match_against = url if '://' in prefix else url_path
+                if match_against.startswith(prefix):
+                    if len(match_against) == len(prefix) or match_against[len(prefix)] in ['/', '?', '#']:
+                        result = True
+                        self._update_stats(result)
+                        return not result if self._reverse else result
+
+        # Complex patterns
+        if self._path_patterns:
+            if any(p.search(url) for p in self._path_patterns):
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
+
+        result = False
+        self._update_stats(result)
+        return not result if self._reverse else result
+
+
+class ContentTypeFilter(URLFilter):
+    """Optimized content type filter using fast lookups"""
+
+    __slots__ = ("allowed_types", "_ext_map", "_check_extension")
+
+    # Fast extension to mime type mapping
+    _MIME_MAP = {
+        # Text Formats
+        "txt": "text/plain",
+        "html": "text/html",
+        "htm": "text/html",
+        "xhtml": "application/xhtml+xml",
+        "css": "text/css",
+        "csv": "text/csv",
+        "ics": "text/calendar",
+        "js": "application/javascript",
+        # Images
+        "bmp": "image/bmp",
+        "gif": "image/gif",
+        "jpeg": "image/jpeg",
+        "jpg": "image/jpeg",
+        "png": "image/png",
+        "svg": "image/svg+xml",
+        "tiff": "image/tiff",
+        "ico": "image/x-icon",
+        "webp": "image/webp",
+        # Audio
+        "mp3": "audio/mpeg",
+        "wav": "audio/wav",
+        "ogg": "audio/ogg",
+        "m4a": "audio/mp4",
+        "aac": "audio/aac",
+        # Video
+        "mp4": "video/mp4",
+        "mpeg": "video/mpeg",
+        "webm": "video/webm",
+        "avi": "video/x-msvideo",
+        "mov": "video/quicktime",
+        "flv": "video/x-flv",
+        "wmv": "video/x-ms-wmv",
+        "mkv": "video/x-matroska",
+        # Applications
+        "json": "application/json",
+        "xml": "application/xml",
+        "pdf": "application/pdf",
+        "zip": "application/zip",
+        "gz": "application/gzip",
+        "tar": "application/x-tar",
+        "rar": "application/vnd.rar",
+        "7z": "application/x-7z-compressed",
+        "exe": "application/vnd.microsoft.portable-executable",
+        "msi": "application/x-msdownload",
+        # Fonts
+        "woff": "font/woff",
+        "woff2": "font/woff2",
+        "ttf": "font/ttf",
+        "otf": "font/otf",
+        # Microsoft Office
+        "doc": "application/msword",
+        "dot": "application/msword",
+        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "xls": "application/vnd.ms-excel",
+        "ppt": "application/vnd.ms-powerpoint",
+        "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        # OpenDocument Formats
+        "odt": "application/vnd.oasis.opendocument.text",
+        "ods": "application/vnd.oasis.opendocument.spreadsheet",
+        "odp": "application/vnd.oasis.opendocument.presentation",
+        # Archives
+        "tar.gz": "application/gzip",
+        "tgz": "application/gzip",
+        "bz2": "application/x-bzip2",
+        # Others
+        "rtf": "application/rtf",
+        "apk": "application/vnd.android.package-archive",
+        "epub": "application/epub+zip",
+        "jar": "application/java-archive",
+        "swf": "application/x-shockwave-flash",
+        "midi": "audio/midi",
+        "mid": "audio/midi",
+        "ps": "application/postscript",
+        "ai": "application/postscript",
+        "eps": "application/postscript",
+        # Custom or less common
+        "bin": "application/octet-stream",
+        "dmg": "application/x-apple-diskimage",
+        "iso": "application/x-iso9660-image",
+        "deb": "application/x-debian-package",
+        "rpm": "application/x-rpm",
+        "sqlite": "application/vnd.sqlite3",
+        # Placeholder
+        "unknown": "application/octet-stream",  # Fallback for unknown file types
+        # php
+        "php": "application/x-httpd-php",
+        "php3": "application/x-httpd-php",
+        "php4": "application/x-httpd-php",
+        "php5": "application/x-httpd-php",
+        "php7": "application/x-httpd-php",
+        "phtml": "application/x-httpd-php",
+        "phps": "application/x-httpd-php-source",
+
+    }
+
+    @staticmethod
+    @lru_cache(maxsize=1000)
+    def _extract_extension(url: str) -> str:
+        """Extracts file extension from a URL."""
+        # Remove scheme (http://, https://) if present
+        if "://" in url:
+            url = url.split("://", 1)[-1]  # Get everything after '://'
+
+        # Remove domain (everything up to the first '/')
+        path_start = url.find("/")
+        path = url[path_start:] if path_start != -1 else ""
+
+        # Extract last filename in path
+        filename = path.rsplit("/", 1)[-1] if "/" in path else ""
+
+        # Extract and validate extension
+        if "." not in filename:
+            return ""
+
+        return filename.rpartition(".")[-1].lower()
+
+    def __init__(
+        self,
+        allowed_types: Union[str, List[str]],
+        check_extension: bool = True,
+        ext_map: Dict[str, str] = _MIME_MAP,
+    ):
+        super().__init__()
+        # Normalize and store as frozenset for fast lookup
+        self.allowed_types = frozenset(
+            t.lower()
+            for t in (
+                allowed_types if isinstance(allowed_types, list) else [allowed_types]
+            )
+        )
+        self._check_extension = check_extension
+
+        # Pre-compute extension map for allowed types
+        self._ext_map = frozenset(
+            ext
+            for ext, mime in self._MIME_MAP.items()
+            if any(allowed in mime for allowed in self.allowed_types)
+        )
+
+    @lru_cache(maxsize=1000)
+    def _check_url_cached(self, url: str) -> bool:
+        """Cached URL checking"""
+        if not self._check_extension:
+            return True
+        ext = self._extract_extension(url)
+        if not ext:
+            return True
+
+        return ext in self._ext_map
+
+    def apply(self, url: str) -> bool:
+        """Fast extension check with caching"""
+        result = self._check_url_cached(url)
+        self._update_stats(result)
+        return result
+
+
+class DomainFilter(URLFilter):
+    """Optimized domain filter with fast lookups and caching"""
+
+    __slots__ = ("_allowed_domains", "_blocked_domains", "_domain_cache")
+
+    # Regex for fast domain extraction
+    _DOMAIN_REGEX = re.compile(r"://([^/]+)")
+
+    def __init__(
+        self,
+        allowed_domains: Union[str, List[str]] = None,
+        blocked_domains: Union[str, List[str]] = None,
+    ):
+        super().__init__()
+
+        # Convert inputs to frozensets for immutable, fast lookups
+        self._allowed_domains = (
+            frozenset(self._normalize_domains(allowed_domains))
+            if allowed_domains
+            else None
+        )
+        self._blocked_domains = (
+            frozenset(self._normalize_domains(blocked_domains))
+            if blocked_domains
+            else frozenset()
+        )
+
+    @staticmethod
+    def _normalize_domains(domains: Union[str, List[str]]) -> Set[str]:
+        """Fast domain normalization"""
+        if isinstance(domains, str):
+            return {domains.lower()}
+        return {d.lower() for d in domains}
+    
+    @staticmethod
+    def _is_subdomain(domain: str, parent_domain: str) -> bool:
+        """Check if domain is a subdomain of parent_domain"""
+        return domain == parent_domain or domain.endswith(f".{parent_domain}")
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _extract_domain(url: str) -> str:
+        """Ultra-fast domain extraction with regex and caching"""
+        match = DomainFilter._DOMAIN_REGEX.search(url)
+        return match.group(1).lower() if match else ""
+
+    def apply(self, url: str) -> bool:
+        """Optimized domain checking with early returns"""
+        # Skip processing if no filters
+        if not self._blocked_domains and self._allowed_domains is None:
+            self._update_stats(True)
+            return True
+
+        domain = self._extract_domain(url)
+
+        # Check for blocked domains, including subdomains
+        for blocked in self._blocked_domains:
+            if self._is_subdomain(domain, blocked):
+                self._update_stats(False)
+                return False
+
+        # If no allowed domains specified, accept all non-blocked
+        if self._allowed_domains is None:
+            self._update_stats(True)
+            return True
+
+        # Check if domain matches any allowed domain (including subdomains)
+        for allowed in self._allowed_domains:
+            if self._is_subdomain(domain, allowed):
+                self._update_stats(True)
+                return True
+
+        # No matches found
+        self._update_stats(False)
+        return False
+
+
+class ContentRelevanceFilter(URLFilter):
+    """BM25-based relevance filter using head section content"""
+
+    __slots__ = ("query_terms", "threshold", "k1", "b", "avgdl", "query")
+
+    def __init__(
+        self,
+        query: Union[str, List[str]],
+        threshold: float,
+        k1: float = 1.2,
+        b: float = 0.75,
+        avgdl: int = 1000,
+    ):
+        super().__init__(name="BM25RelevanceFilter")
+        if isinstance(query, list):
+            self.query = " ".join(query)
+        else:
+            self.query = query
+        self.query_terms = self._tokenize(self.query)
+        self.threshold = threshold
+        self.k1 = k1  # TF saturation parameter
+        self.b = b  # Length normalization parameter
+        self.avgdl = avgdl  # Average document length (empirical value)
+
+    async def apply(self, url: str) -> bool:
+        head_content = await HeadPeekr.peek_html(url)
+        if not head_content:
+            self._update_stats(False)
+            return False
+
+        # Field extraction with weighting
+        fields = {
+            "title": HeadPeekr.get_title(head_content) or "",
+            "meta": HeadPeekr.extract_meta_tags(head_content),
+        }
+        doc_text = self._build_document(fields)
+
+        score = self._bm25(doc_text)
+        decision = score >= self.threshold
+        self._update_stats(decision)
+        return decision
+
+    def _build_document(self, fields: Dict) -> str:
+        """Weighted document construction"""
+        return " ".join(
+            [
+                fields["title"] * 3,  # Title weight
+                fields["meta"].get("description", "") * 2,
+                fields["meta"].get("keywords", ""),
+                " ".join(fields["meta"].values()),
+            ]
+        )
+
+    def _tokenize(self, text: str) -> List[str]:
+        """Fast case-insensitive tokenization"""
+        return text.lower().split()
+
+    def _bm25(self, document: str) -> float:
+        """Optimized BM25 implementation for head sections"""
+        doc_terms = self._tokenize(document)
+        doc_len = len(doc_terms)
+        tf = defaultdict(int)
+
+        for term in doc_terms:
+            tf[term] += 1
+
+        score = 0.0
+        for term in set(self.query_terms):
+            term_freq = tf[term]
+            idf = math.log((1 + 1) / (term_freq + 0.5) + 1)  # Simplified IDF
+            numerator = term_freq * (self.k1 + 1)
+            denominator = term_freq + self.k1 * (
+                1 - self.b + self.b * (doc_len / self.avgdl)
+            )
+            score += idf * (numerator / denominator)
+
+        return score
+
+
+class SEOFilter(URLFilter):
+    """Quantitative SEO quality assessment filter using head section analysis"""
+
+    __slots__ = ("threshold", "_weights", "_kw_patterns")
+
+    # Based on SEMrush/Google ranking factors research
+    DEFAULT_WEIGHTS = {
+        "title_length": 0.15,
+        "title_kw": 0.18,
+        "meta_description": 0.12,
+        "canonical": 0.10,
+        "robot_ok": 0.20,  # Most critical factor
+        "schema_org": 0.10,
+        "url_quality": 0.15,
+    }
+
+    def __init__(
+        self,
+        threshold: float = 0.65,
+        keywords: List[str] = None,
+        weights: Dict[str, float] = None,
+    ):
+        super().__init__(name="SEOFilter")
+        self.threshold = threshold
+        self._weights = weights or self.DEFAULT_WEIGHTS
+        self._kw_patterns = (
+            re.compile(
+                r"\b({})\b".format("|".join(map(re.escape, keywords or []))), re.I
+            )
+            if keywords
+            else None
+        )
+
+    async def apply(self, url: str) -> bool:
+        head_content = await HeadPeekr.peek_html(url)
+        if not head_content:
+            self._update_stats(False)
+            return False
+
+        meta = HeadPeekr.extract_meta_tags(head_content)
+        title = HeadPeekr.get_title(head_content) or ""
+        parsed_url = urlparse(url)
+
+        scores = {
+            "title_length": self._score_title_length(title),
+            "title_kw": self._score_keyword_presence(title),
+            "meta_description": self._score_meta_description(
+                meta.get("description", "")
+            ),
+            "canonical": self._score_canonical(meta.get("canonical"), url),
+            "robot_ok": 1.0 if "noindex" not in meta.get("robots", "") else 0.0,
+            "schema_org": self._score_schema_org(head_content),
+            "url_quality": self._score_url_quality(parsed_url),
+        }
+
+        total_score = sum(
+            weight * scores[factor] for factor, weight in self._weights.items()
+        )
+
+        decision = total_score >= self.threshold
+        self._update_stats(decision)
+        return decision
+
+    def _score_title_length(self, title: str) -> float:
+        length = len(title)
+        if 50 <= length <= 60:
+            return 1.0
+        if 40 <= length < 50 or 60 < length <= 70:
+            return 0.7
+        return 0.3  # Poor length
+
+    def _score_keyword_presence(self, text: str) -> float:
+        if not self._kw_patterns:
+            return 0.0
+        matches = len(self._kw_patterns.findall(text))
+        return min(matches * 0.3, 1.0)  # Max 3 matches
+
+    def _score_meta_description(self, desc: str) -> float:
+        length = len(desc)
+        if 140 <= length <= 160:
+            return 1.0
+        return 0.5 if 120 <= length <= 200 else 0.2
+
+    def _score_canonical(self, canonical: str, original: str) -> float:
+        if not canonical:
+            return 0.5  # Neutral score
+        return 1.0 if canonical == original else 0.2
+
+    def _score_schema_org(self, html: str) -> float:
+        # Detect any schema.org markup in head
+        return (
+            1.0
+            if re.search(r']+type=["\']application/ld\+json', html)
+            else 0.0
+        )
+
+    def _score_url_quality(self, parsed_url) -> float:
+        score = 1.0
+        path = parsed_url.path.lower()
+
+        # Penalty factors
+        if len(path) > 80:
+            score *= 0.7
+        if re.search(r"\d{4}", path):
+            score *= 0.8  # Numbers in path
+        if parsed_url.query:
+            score *= 0.6  # URL parameters
+        if "_" in path:
+            score *= 0.9  # Underscores vs hyphens
+
+        return score
diff --git a/crawl4ai/deep_crawling/scorers.py b/crawl4ai/deep_crawling/scorers.py
new file mode 100644
index 0000000..1cd9f3e
--- /dev/null
+++ b/crawl4ai/deep_crawling/scorers.py
@@ -0,0 +1,519 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from urllib.parse import urlparse, unquote
+import re
+import logging
+from functools import lru_cache
+from array import array
+import ctypes
+import platform
+PLATFORM = platform.system()
+
+# Pre-computed scores for common year differences
+_SCORE_LOOKUP = [1.0, 0.5, 0.3333333333333333, 0.25]
+
+# Pre-computed scores for common year differences
+_FRESHNESS_SCORES = [
+   1.0,    # Current year
+   0.9,    # Last year
+   0.8,    # 2 years ago
+   0.7,    # 3 years ago
+   0.6,    # 4 years ago
+   0.5,    # 5 years ago
+]
+
+class ScoringStats:
+    __slots__ = ('_urls_scored', '_total_score', '_min_score', '_max_score')
+    
+    def __init__(self):
+        self._urls_scored = 0
+        self._total_score = 0.0
+        self._min_score = None  # Lazy initialization
+        self._max_score = None
+    
+    def update(self, score: float) -> None:
+        """Optimized update with minimal operations"""
+        self._urls_scored += 1
+        self._total_score += score
+        
+        # Lazy min/max tracking - only if actually accessed
+        if self._min_score is not None:
+            if score < self._min_score:
+                self._min_score = score
+        if self._max_score is not None:
+            if score > self._max_score:
+                self._max_score = score
+                
+    def get_average(self) -> float:
+        """Direct calculation instead of property"""
+        return self._total_score / self._urls_scored if self._urls_scored else 0.0
+    
+    def get_min(self) -> float:
+        """Lazy min calculation"""
+        if self._min_score is None:
+            self._min_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
+        return self._min_score
+        
+    def get_max(self) -> float:
+        """Lazy max calculation"""
+        if self._max_score is None:
+            self._max_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
+        return self._max_score
+class URLScorer(ABC):
+    __slots__ = ('_weight', '_stats')
+    
+    def __init__(self, weight: float = 1.0):
+        # Store weight directly as float32 for memory efficiency
+        self._weight = ctypes.c_float(weight).value
+        self._stats = ScoringStats()
+    
+    @abstractmethod
+    def _calculate_score(self, url: str) -> float:
+        """Calculate raw score for URL."""
+        pass
+    
+    def score(self, url: str) -> float:
+        """Calculate weighted score with minimal overhead."""
+        score = self._calculate_score(url) * self._weight
+        self._stats.update(score)
+        return score
+    
+    @property
+    def stats(self):
+        """Access to scoring statistics."""
+        return self._stats
+    
+    @property
+    def weight(self):
+        return self._weight
+
+class CompositeScorer(URLScorer):
+    __slots__ = ('_scorers', '_normalize', '_weights_array', '_score_array')
+    
+    def __init__(self, scorers: List[URLScorer], normalize: bool = True):
+        """Initialize composite scorer combining multiple scoring strategies.
+        
+        Optimized for:
+        - Fast parallel scoring
+        - Memory efficient score aggregation
+        - Quick short-circuit conditions
+        - Pre-allocated arrays
+        
+        Args:
+            scorers: List of scoring strategies to combine
+            normalize: Whether to normalize final score by scorer count
+        """
+        super().__init__(weight=1.0)
+        self._scorers = scorers
+        self._normalize = normalize
+        
+        # Pre-allocate arrays for scores and weights
+        self._weights_array = array('f', [s.weight for s in scorers])
+        self._score_array = array('f', [0.0] * len(scorers))
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate combined score from all scoring strategies.
+        
+        Uses:
+        1. Pre-allocated arrays for scores
+        2. Short-circuit on zero scores
+        3. Optimized normalization
+        4. Vectorized operations where possible
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Combined and optionally normalized score
+        """
+        total_score = 0.0
+        scores = self._score_array
+        
+        # Get scores from all scorers
+        for i, scorer in enumerate(self._scorers):
+            # Use public score() method which applies weight
+            scores[i] = scorer.score(url)
+            total_score += scores[i]
+            
+        # Normalize if requested
+        if self._normalize and self._scorers:
+            count = len(self._scorers)
+            return total_score / count
+            
+        return total_score
+
+    def score(self, url: str) -> float:
+        """Public scoring interface with stats tracking.
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Final combined score
+        """
+        score = self._calculate_score(url)
+        self.stats.update(score)
+        return score
+
+class KeywordRelevanceScorer(URLScorer):
+    __slots__ = ('_weight', '_stats', '_keywords', '_case_sensitive')
+    
+    def __init__(self, keywords: List[str], weight: float = 1.0, case_sensitive: bool = False):
+        super().__init__(weight=weight)
+        self._case_sensitive = case_sensitive
+        # Pre-process keywords once
+        self._keywords = [k if case_sensitive else k.lower() for k in keywords]
+    
+    @lru_cache(maxsize=10000)
+    def _url_bytes(self, url: str) -> bytes:
+        """Cache decoded URL bytes"""
+        return url.encode('utf-8') if self._case_sensitive else url.lower().encode('utf-8')
+    
+    
+    def _calculate_score(self, url: str) -> float:
+        """Fast string matching without regex or byte conversion"""
+        if not self._case_sensitive:
+            url = url.lower()
+            
+        matches = sum(1 for k in self._keywords if k in url)
+        
+        # Fast return paths
+        if not matches:
+            return 0.0
+        if matches == len(self._keywords):
+            return 1.0
+            
+        return matches / len(self._keywords)
+
+class PathDepthScorer(URLScorer):
+    __slots__ = ('_weight', '_stats', '_optimal_depth')  # Remove _url_cache
+    
+    def __init__(self, optimal_depth: int = 3, weight: float = 1.0):
+        super().__init__(weight=weight)
+        self._optimal_depth = optimal_depth
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _quick_depth(path: str) -> int:
+        """Ultra fast path depth calculation.
+        
+        Examples:
+            - "http://example.com" -> 0  # No path segments
+            - "http://example.com/" -> 0  # Empty path
+            - "http://example.com/a" -> 1
+            - "http://example.com/a/b" -> 2
+        """
+        if not path or path == '/':
+            return 0
+            
+        if '/' not in path:
+            return 0
+            
+        depth = 0
+        last_was_slash = True
+        
+        for c in path:
+            if c == '/':
+                if not last_was_slash:
+                    depth += 1
+                last_was_slash = True
+            else:
+                last_was_slash = False
+                
+        if not last_was_slash:
+            depth += 1
+            
+        return depth
+
+    @lru_cache(maxsize=10000)  # Cache the whole calculation
+    def _calculate_score(self, url: str) -> float:
+        pos = url.find('/', url.find('://') + 3)
+        if pos == -1:
+            depth = 0
+        else:
+            depth = self._quick_depth(url[pos:])
+            
+        # Use lookup table for common distances
+        distance = depth - self._optimal_depth
+        distance = distance if distance >= 0 else -distance  # Faster than abs()
+        
+        if distance < 4:
+            return _SCORE_LOOKUP[distance]
+            
+        return 1.0 / (1.0 + distance)                                             
+
+class ContentTypeScorer(URLScorer):
+    __slots__ = ('_weight', '_exact_types', '_regex_types')
+
+    def __init__(self, type_weights: Dict[str, float], weight: float = 1.0):
+        """Initialize scorer with type weights map.
+        
+        Args:
+            type_weights: Dict mapping file extensions/patterns to scores (e.g. {'.html$': 1.0})
+            weight: Overall weight multiplier for this scorer
+        """
+        super().__init__(weight=weight)
+        self._exact_types = {}  # Fast lookup for simple extensions
+        self._regex_types = []  # Fallback for complex patterns
+        
+        # Split into exact vs regex matchers for performance
+        for pattern, score in type_weights.items():
+            if pattern.startswith('.') and pattern.endswith('$'):
+                ext = pattern[1:-1]
+                self._exact_types[ext] = score
+            else:
+                self._regex_types.append((re.compile(pattern), score))
+                
+        # Sort complex patterns by score for early exit
+        self._regex_types.sort(key=lambda x: -x[1])
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _quick_extension(url: str) -> str:
+        """Extract file extension ultra-fast without regex/splits.
+        
+        Handles:
+        - Basic extensions: "example.html" -> "html"
+        - Query strings: "page.php?id=1" -> "php" 
+        - Fragments: "doc.pdf#page=1" -> "pdf"
+        - Path params: "file.jpg;width=100" -> "jpg"
+        
+        Args:
+            url: URL to extract extension from
+            
+        Returns:
+            Extension without dot, or empty string if none found
+        """
+        pos = url.rfind('.')
+        if pos == -1:
+            return ''
+        
+        # Find first non-alphanumeric char after extension
+        end = len(url)
+        for i in range(pos + 1, len(url)):
+            c = url[i]
+            # Stop at query string, fragment, path param or any non-alphanumeric
+            if c in '?#;' or not c.isalnum():
+                end = i
+                break
+                
+        return url[pos + 1:end].lower()
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate content type score for URL.
+        
+        Uses staged approach:
+        1. Try exact extension match (fast path)
+        2. Fall back to regex patterns if needed
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Score between 0.0 and 1.0 * weight
+        """
+        # Fast path: direct extension lookup
+        ext = self._quick_extension(url)
+        if ext:
+            score = self._exact_types.get(ext, None)
+            if score is not None:
+                return score
+                
+        # Slow path: regex patterns
+        for pattern, score in self._regex_types:
+            if pattern.search(url):
+                return score
+
+        return 0.0
+
+class FreshnessScorer(URLScorer):
+    __slots__ = ('_weight', '_date_pattern', '_current_year')
+
+    def __init__(self, weight: float = 1.0, current_year: int = 2024):
+        """Initialize freshness scorer.
+        
+        Extracts and scores dates from URLs using format:
+        - YYYY/MM/DD 
+        - YYYY-MM-DD
+        - YYYY_MM_DD
+        - YYYY (year only)
+        
+        Args:
+            weight: Score multiplier
+            current_year: Year to calculate freshness against (default 2024)
+        """
+        super().__init__(weight=weight)
+        self._current_year = current_year
+        
+        # Combined pattern for all date formats
+        # Uses non-capturing groups (?:) and alternation
+        self._date_pattern = re.compile(
+            r'(?:/'  # Path separator
+            r'|[-_])'  # or date separators
+            r'((?:19|20)\d{2})'  # Year group (1900-2099)
+            r'(?:'  # Optional month/day group
+            r'(?:/|[-_])'  # Date separator  
+            r'(?:\d{2})'  # Month
+            r'(?:'  # Optional day
+            r'(?:/|[-_])'  # Date separator
+            r'(?:\d{2})'  # Day
+            r')?'  # Day is optional
+            r')?'  # Month/day group is optional
+        )
+
+    @lru_cache(maxsize=10000)
+    def _extract_year(self, url: str) -> Optional[int]:
+        """Extract the most recent year from URL.
+        
+        Args:
+            url: URL to extract year from
+            
+        Returns:
+            Year as int or None if no valid year found
+        """
+        matches = self._date_pattern.finditer(url)
+        latest_year = None
+        
+        # Find most recent year
+        for match in matches:
+            year = int(match.group(1))
+            if (year <= self._current_year and  # Sanity check
+                (latest_year is None or year > latest_year)):
+                latest_year = year
+                
+        return latest_year
+
+    @lru_cache(maxsize=10000) 
+    def _calculate_score(self, url: str) -> float:
+        """Calculate freshness score based on URL date.
+        
+        More recent years score higher. Uses pre-computed scoring
+        table for common year differences.
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Score between 0.0 and 1.0 * weight
+        """
+        year = self._extract_year(url)
+        if year is None:
+            return 0.5  # Default score
+            
+        # Use lookup table for common year differences
+        year_diff = self._current_year - year
+        if year_diff < len(_FRESHNESS_SCORES):
+            return _FRESHNESS_SCORES[year_diff]
+            
+        # Fallback calculation for older content
+        return max(0.1, 1.0 - year_diff * 0.1)
+
+class DomainAuthorityScorer(URLScorer):
+    __slots__ = ('_weight', '_domain_weights', '_default_weight', '_top_domains')
+    
+    def __init__(
+        self,
+        domain_weights: Dict[str, float],
+        default_weight: float = 0.5,
+        weight: float = 1.0,
+    ):
+        """Initialize domain authority scorer.
+        
+        Args:
+            domain_weights: Dict mapping domains to authority scores
+            default_weight: Score for unknown domains
+            weight: Overall scorer weight multiplier
+            
+        Example:
+            {
+                'python.org': 1.0,
+                'github.com': 0.9,
+                'medium.com': 0.7
+            }
+        """
+        super().__init__(weight=weight)
+        
+        # Pre-process domains for faster lookup
+        self._domain_weights = {
+            domain.lower(): score 
+            for domain, score in domain_weights.items()
+        }
+        self._default_weight = default_weight
+        
+        # Cache top domains for fast path
+        self._top_domains = {
+            domain: score
+            for domain, score in sorted(
+                domain_weights.items(), 
+                key=lambda x: -x[1]
+            )[:5]  # Keep top 5 highest scoring domains
+        }
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _extract_domain(url: str) -> str:
+        """Extract domain from URL ultra-fast.
+        
+        Handles:
+        - Basic domains: "example.com"
+        - Subdomains: "sub.example.com" 
+        - Ports: "example.com:8080"
+        - IPv4: "192.168.1.1"
+        
+        Args:
+            url: Full URL to extract domain from
+            
+        Returns:
+            Lowercase domain without port
+        """
+        # Find domain start
+        start = url.find('://') 
+        if start == -1:
+            start = 0
+        else:
+            start += 3
+            
+        # Find domain end
+        end = url.find('/', start)
+        if end == -1:
+            end = url.find('?', start)
+            if end == -1:
+                end = url.find('#', start)
+                if end == -1:
+                    end = len(url)
+                    
+        # Extract domain and remove port
+        domain = url[start:end]
+        port_idx = domain.rfind(':')
+        if port_idx != -1:
+            domain = domain[:port_idx]
+            
+        return domain.lower()
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate domain authority score.
+        
+        Uses staged approach:
+        1. Check top domains (fastest)
+        2. Check full domain weights
+        3. Return default weight
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Authority score between 0.0 and 1.0 * weight
+        """
+        domain = self._extract_domain(url)
+        
+        # Fast path: check top domains first
+        score = self._top_domains.get(domain)
+        if score is not None:
+            return score
+            
+        # Regular path: check all domains
+        return self._domain_weights.get(domain, self._default_weight)
\ No newline at end of file
diff --git a/crawl4ai/docker_client.py b/crawl4ai/docker_client.py
new file mode 100644
index 0000000..6624cf0
--- /dev/null
+++ b/crawl4ai/docker_client.py
@@ -0,0 +1,219 @@
+from typing import List, Optional, Union, AsyncGenerator, Dict, Any, Callable
+import httpx
+import json
+from urllib.parse import urljoin
+import asyncio
+
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from .models import CrawlResult
+from .async_logger import AsyncLogger, LogLevel
+from .utils import hooks_to_string
+
+
+class Crawl4aiClientError(Exception):
+    """Base exception for Crawl4ai Docker client errors."""
+    pass
+
+
+class ConnectionError(Crawl4aiClientError):
+    """Raised when connection to the Docker server fails."""
+    pass
+
+
+class RequestError(Crawl4aiClientError):
+    """Raised when the server returns an error response."""
+    pass
+
+
+class Crawl4aiDockerClient:
+    """Client for interacting with Crawl4AI Docker server with token authentication."""
+    
+    def __init__(
+        self,
+        base_url: str = "http://localhost:8000",
+        timeout: float = 30.0,
+        verify_ssl: bool = True,
+        verbose: bool = True,
+        log_file: Optional[str] = None
+    ):
+        self.base_url = base_url.rstrip('/')
+        self.timeout = timeout
+        self.logger = AsyncLogger(log_file=log_file, log_level=LogLevel.DEBUG, verbose=verbose)
+        self._http_client = httpx.AsyncClient(
+            timeout=timeout,
+            verify=verify_ssl,
+            headers={"Content-Type": "application/json"}
+        )
+        self._token: Optional[str] = None
+
+    async def authenticate(self, email: str) -> None:
+        """Authenticate with the server and store the token."""
+        url = urljoin(self.base_url, "/token")
+        try:
+            self.logger.info(f"Authenticating with email: {email}", tag="AUTH")
+            response = await self._http_client.post(url, json={"email": email})
+            response.raise_for_status()
+            data = response.json()
+            self._token = data["access_token"]
+            self._http_client.headers["Authorization"] = f"Bearer {self._token}"
+            self.logger.success("Authentication successful", tag="AUTH")
+        except (httpx.RequestError, httpx.HTTPStatusError) as e:
+            error_msg = f"Authentication failed: {str(e)}"
+            self.logger.error(error_msg, tag="ERROR")
+            raise ConnectionError(error_msg)
+
+    async def _check_server(self) -> None:
+        """Check if server is reachable, raising an error if not."""
+        try:
+            await self._http_client.get(urljoin(self.base_url, "/health"))
+            self.logger.success(f"Connected to {self.base_url}", tag="READY")
+        except httpx.RequestError as e:
+            self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR")
+            raise ConnectionError(f"Cannot connect to server: {str(e)}")
+
+    def _prepare_request(
+        self,
+        urls: List[str],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None,
+        hooks_timeout: int = 30
+    ) -> Dict[str, Any]:
+        """Prepare request data from configs."""
+        if self._token:
+            self._http_client.headers["Authorization"] = f"Bearer {self._token}"
+
+        request_data = {
+            "urls": urls,
+            "browser_config": browser_config.dump() if browser_config else {},
+            "crawler_config": crawler_config.dump() if crawler_config else {}
+        }
+
+        # Handle hooks if provided
+        if hooks:
+            # Check if hooks are already strings or need conversion
+            if any(callable(v) for v in hooks.values()):
+                # Convert function objects to strings
+                hooks_code = hooks_to_string(hooks)
+            else:
+                # Already in string format
+                hooks_code = hooks
+
+            request_data["hooks"] = {
+                "code": hooks_code,
+                "timeout": hooks_timeout
+            }
+
+        return request_data
+
+    async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
+        """Make an HTTP request with error handling."""
+        url = urljoin(self.base_url, endpoint)
+        try:
+            response = await self._http_client.request(method, url, **kwargs)
+            response.raise_for_status()
+            return response
+        except httpx.TimeoutException as e:
+            raise ConnectionError(f"Request timed out: {str(e)}")
+        except httpx.RequestError as e:
+            raise ConnectionError(f"Failed to connect: {str(e)}")
+        except httpx.HTTPStatusError as e:
+            error_msg = (e.response.json().get("detail", str(e)) 
+                        if "application/json" in e.response.headers.get("content-type", "") 
+                        else str(e))
+            raise RequestError(f"Server error {e.response.status_code}: {error_msg}")
+
+    async def crawl(
+        self,
+        urls: List[str],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None,
+        hooks_timeout: int = 30
+    ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
+        """
+        Execute a crawl operation.
+
+        Args:
+            urls: List of URLs to crawl
+            browser_config: Browser configuration
+            crawler_config: Crawler configuration
+            hooks: Optional hooks - can be either:
+                   - Dict[str, Callable]: Function objects that will be converted to strings
+                   - Dict[str, str]: Already stringified hook code
+            hooks_timeout: Timeout in seconds for each hook execution (1-120)
+
+        Returns:
+            Single CrawlResult, list of results, or async generator for streaming
+
+        Example with function hooks:
+            >>> async def my_hook(page, context, **kwargs):
+            ...     await page.set_viewport_size({"width": 1920, "height": 1080})
+            ...     return page
+            >>>
+            >>> result = await client.crawl(
+            ...     ["https://example.com"],
+            ...     hooks={"on_page_context_created": my_hook}
+            ... )
+        """
+        await self._check_server()
+
+        data = self._prepare_request(urls, browser_config, crawler_config, hooks, hooks_timeout)
+        is_streaming = crawler_config and crawler_config.stream
+
+        self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL")
+
+        if is_streaming:
+            async def stream_results() -> AsyncGenerator[CrawlResult, None]:
+                async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response:
+                    response.raise_for_status()
+                    async for line in response.aiter_lines():
+                        if line.strip():
+                            result = json.loads(line)
+                            if "error" in result:
+                                self.logger.error_status(url=result.get("url", "unknown"), error=result["error"])
+                                continue
+                            self.logger.url_status(url=result.get("url", "unknown"), success=True, timing=result.get("timing", 0.0))
+                            if result.get("status") == "completed":
+                                continue
+                            else:
+                                yield CrawlResult(**result)
+            return stream_results()
+
+        response = await self._request("POST", "/crawl", json=data, timeout=hooks_timeout)
+        result_data = response.json()
+        if not result_data.get("success", False):
+            raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}")
+
+        results = [CrawlResult(**r) for r in result_data.get("results", [])]
+        self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL")
+        return results[0] if len(results) == 1 else results
+
+    async def get_schema(self) -> Dict[str, Any]:
+        """Retrieve configuration schemas."""
+        response = await self._request("GET", "/schema")
+        return response.json()
+
+    async def close(self) -> None:
+        """Close the HTTP client session."""
+        self.logger.info("Closing client", tag="CLOSE")
+        await self._http_client.aclose()
+
+    async def __aenter__(self) -> "Crawl4aiDockerClient":
+        return self
+
+    async def __aexit__(self, exc_type: Optional[type], exc_val: Optional[Exception], exc_tb: Optional[Any]) -> None:
+        await self.close()
+
+
+# Example usage
+async def main():
+    async with Crawl4aiDockerClient(verbose=True) as client:
+        await client.authenticate("user@example.com")
+        result = await client.crawl(["https://example.com"])
+        print(result)
+        schema = await client.get_schema()
+        print(schema)
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/crawl4ai/domain_mapper.py b/crawl4ai/domain_mapper.py
new file mode 100644
index 0000000..3c866ab
--- /dev/null
+++ b/crawl4ai/domain_mapper.py
@@ -0,0 +1,1132 @@
+"""
+domain_mapper.py
+Comprehensive domain URL discovery for Crawl4AI
+
+Discovers all URLs under a domain using 8 sources without deep crawling:
+  sitemap   — per-host sitemap.xml / sitemap_index.xml / robots.txt Sitemap: directives
+  cc        — Common Crawl CDX API
+  wayback   — Wayback Machine CDX API
+  crt       — Certificate Transparency (crt.sh) subdomain discovery
+  probe     — common-path probing with soft-404 detection
+  robots    — robots.txt Disallow:/Allow: path mining
+  feed      — RSS/Atom feed parsing
+  homepage  — homepage link extraction via quick_extract_links()
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import pathlib
+import re
+import time
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from urllib.parse import urljoin, urlparse, quote
+
+import httpx
+
+try:
+    from lxml import etree
+    LXML = True
+except ImportError:
+    LXML = False
+
+try:
+    import rank_bm25
+    HAS_BM25 = True
+except ImportError:
+    HAS_BM25 = False
+
+from .async_logger import AsyncLoggerBase, AsyncLogger
+from .async_url_seeder import AsyncUrlSeeder, _parse_head
+from .utils import (
+    normalize_url,
+    get_base_domain,
+    is_external_url,
+    quick_extract_links,
+)
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from .async_configs import DomainMapperConfig
+
+# ──────────────────────────────────────────────────────────────── constants
+
+WAYBACK_CDX_URL = "https://web.archive.org/cdx/search/cdx"
+CRT_SH_URL = "https://crt.sh/"
+
+DEFAULT_PROBE_PATHS = [
+    "/", "/docs", "/api", "/login", "/dashboard", "/blog",
+    "/features", "/pricing", "/about", "/contact", "/auth/login",
+    "/openapi.json", "/swagger.json", "/api-docs", "/graphql",
+    "/status", "/health", "/changelog", "/terms", "/privacy",
+    "/faq", "/help", "/support", "/products", "/services",
+]
+
+DEFAULT_COMMON_SUBDOMAINS = [
+    "www", "app", "api", "docs", "blog", "admin", "staging", "dev",
+    "cloud", "mail", "cdn", "static", "portal", "dashboard", "help",
+    "support", "shop", "store",
+]
+
+FEED_PATHS = [
+    "/feed", "/rss", "/atom.xml", "/feed.xml", "/rss.xml",
+    "/index.xml", "/feed/rss", "/blog/feed", "/feed/atom",
+]
+
+VALID_SOURCES = {"sitemap", "cc", "wayback", "crt", "probe", "robots", "feed", "homepage"}
+
+# Minimal nonsense filter — DomainMapper WANTS /login, /admin etc.
+_NONSENSE_SUFFIXES = (
+    "/robots.txt", "/sitemap.xml", "/sitemap_index.xml",
+    "/favicon.ico", "/apple-touch-icon.png", "/browserconfig.xml",
+    "/manifest.json", "/ads.txt", "/humans.txt",
+    "/crossdomain.xml",
+)
+
+_NONSENSE_CONTAINS = (
+    "/.well-known/", "/.git/", "/.svn/", "/.hg/",
+)
+
+# Static asset extensions to filter (JS, CSS, fonts, images, media)
+_ASSET_EXTENSIONS = (
+    ".js", ".css", ".scss", ".sass", ".less", ".map",
+    ".woff", ".woff2", ".ttf", ".eot", ".otf",
+    ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".svg", ".webp", ".ico", ".avif",
+    ".mp4", ".avi", ".mov", ".wmv", ".flv", ".webm",
+    ".mp3", ".wav", ".ogg", ".m4a", ".flac",
+    ".zip", ".tar", ".gz", ".rar", ".7z", ".bz2",
+    ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
+)
+
+
+# ──────────────────────────────────────────────────────────────── dataclass
+
+@dataclass
+class Soft404Fingerprint:
+    """Fingerprint of a known-bad URL response for soft-404 detection."""
+    status_code: int
+    title: Optional[str]
+    content_length: int
+    body_hash: str
+
+
+# ──────────────────────────────────────────────────────────────── class
+
+class DomainMapper:
+    """
+    Comprehensive domain URL discovery without deep crawling.
+
+    Discovers all URLs under a domain using 8 sources:
+    sitemap, cc, wayback, crt, probe, robots, feed, homepage.
+
+    Usage::
+
+        async with DomainMapper() as mapper:
+            results = await mapper.scan("example.com")
+
+        # Or via AsyncWebCrawler:
+        async with AsyncWebCrawler() as crawler:
+            results = await crawler.amap_domain("example.com")
+    """
+
+    def __init__(
+        self,
+        client: Optional[httpx.AsyncClient] = None,
+        logger: Optional[AsyncLoggerBase] = None,
+        base_directory: Optional[Union[str, Path]] = None,
+    ):
+        self._owns_client = client is None
+        self.client = client or httpx.AsyncClient(
+            http2=True,
+            timeout=15,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/123.0.0.0 Safari/537.36"
+                ),
+            },
+        )
+        self.logger = logger or AsyncLogger(verbose=False)
+        self.base_directory = Path(
+            base_directory or os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())
+        )
+        self.cache_dir = self.base_directory / ".crawl4ai" / "domain_mapper_cache"
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        self._seeder: Optional[AsyncUrlSeeder] = None
+        self._rate_sem: Optional[asyncio.Semaphore] = None
+
+    # ──────────────────────── lifecycle
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+        return False
+
+    async def close(self):
+        if self._seeder:
+            await self._seeder.close()
+            self._seeder = None
+        if self._owns_client and self.client:
+            await self.client.aclose()
+
+    # ──────────────────────── logging
+
+    def _log(self, level: str, message: str, tag: str = "MAPPER", **kwargs):
+        if self.logger:
+            fn = getattr(self.logger, level, None)
+            if fn:
+                fn(message=message, tag=tag, params=kwargs.get("params", {}))
+
+    # ──────────────────────── seeder composition
+
+    async def _get_seeder(self) -> AsyncUrlSeeder:
+        if self._seeder is None:
+            self._seeder = AsyncUrlSeeder(
+                client=self.client,
+                logger=self.logger,
+                base_directory=self.base_directory,
+            )
+        return self._seeder
+
+    # ════════════════════════════════════════════════════════════════════════
+    #  MAIN ENTRY POINT
+    # ════════════════════════════════════════════════════════════════════════
+
+    async def scan(
+        self,
+        domain: str,
+        config: Optional["DomainMapperConfig"] = None,
+        **kwargs,
+    ) -> List[Dict[str, Any]]:
+        """
+        Discover all URLs under a domain.
+
+        Args:
+            domain: Domain to scan (e.g., "superdesign.dev")
+            config: DomainMapperConfig. kwargs override config fields.
+
+        Returns:
+            List of dicts with url, host, source, status, head_data, relevance_score.
+        """
+        from .async_configs import DomainMapperConfig as _Cfg
+        if config:
+            config = config.clone(**kwargs) if kwargs else config
+        else:
+            config = _Cfg(**kwargs) if kwargs else _Cfg()
+
+        if config.verbose is not None and self.logger:
+            self.logger.verbose = config.verbose
+
+        # Parse + validate sources
+        sources = {s.strip().lower() for s in config.source.split("+") if s.strip()}
+        invalid = sources - VALID_SOURCES
+        if invalid:
+            raise ValueError(f"Invalid source(s): {invalid}. Valid: {VALID_SOURCES}")
+
+        # Rate limiter
+        if config.hits_per_sec and config.hits_per_sec > 0:
+            self._rate_sem = asyncio.Semaphore(config.hits_per_sec)
+        else:
+            self._rate_sem = None
+
+        # Normalize domain
+        base_domain = re.sub(r"^https?://", "", domain).strip("/").lower()
+
+        self._log("info", "Scanning domain: {domain} with sources: {sources}",
+                  params={"domain": base_domain, "sources": config.source})
+
+        # ── Phase 1: Host Discovery ──────────────────────────────────────
+        hosts = await self._discover_hosts(base_domain, sources, config)
+        self._log("info", "Discovered {count} live hosts",
+                  params={"count": len(hosts)})
+
+        if not hosts:
+            self._log("warning", "No live hosts found for {domain}",
+                      params={"domain": base_domain})
+            return []
+
+        # ── Phase 2: Per-Host Scanning ───────────────────────────────────
+        all_results: List[Dict[str, Any]] = []
+        scan_tasks = []
+        for host in hosts:
+            scan_tasks.append(self._scan_host(host, base_domain, sources, config))
+
+        host_results = await asyncio.gather(*scan_tasks, return_exceptions=True)
+        for i, result in enumerate(host_results):
+            if isinstance(result, Exception):
+                self._log("error", "Error scanning host {host}: {err}",
+                          params={"host": list(hosts)[i], "err": str(result)})
+            else:
+                all_results.extend(result)
+
+        self._log("info", "Collected {count} raw URLs from all hosts",
+                  params={"count": len(all_results)})
+
+        # ── Phase 3: Post-Processing ─────────────────────────────────────
+        results = self._normalize_and_dedup(all_results, base_domain)
+        self._log("info", "{count} URLs after normalization/dedup",
+                  params={"count": len(results)})
+
+        if config.filter_nonsense_urls:
+            results = [r for r in results if not self._is_nonsense(r["url"])]
+
+        # Head extraction
+        if config.extract_head:
+            results = await self._extract_heads(results, config)
+
+        # BM25 scoring
+        if config.query and config.extract_head:
+            results = await self._apply_scoring(results, config)
+
+        # Limit
+        if config.max_urls > 0:
+            results = results[:config.max_urls]
+
+        self._log("info", "Scan complete: {count} URLs found across {hosts} hosts",
+                  params={"count": len(results),
+                          "hosts": len({r["host"] for r in results})})
+        return results
+
+    # ════════════════════════════════════════════════════════════════════════
+    #  PHASE 1: HOST DISCOVERY
+    # ════════════════════════════════════════════════════════════════════════
+
+    async def _discover_hosts(
+        self, base_domain: str, sources: Set[str], config: "DomainMapperConfig"
+    ) -> Set[str]:
+        """Discover all subdomains/hosts under base_domain."""
+        hosts: Set[str] = {base_domain}
+
+        # When include_subdomains is False, skip all subdomain discovery
+        # and only scan the exact domain provided
+        if not getattr(config, "include_subdomains", True):
+            self._log("info", "Subdomain discovery disabled, scanning only {domain}",
+                      params={"domain": base_domain})
+            validated = await self._validate_hosts(hosts, config)
+            return validated
+
+        discovery_tasks = []
+
+        if "crt" in sources:
+            discovery_tasks.append(("crt", self._discover_via_crt(base_domain, config)))
+        if "wayback" in sources:
+            discovery_tasks.append(("wayback", self._discover_via_wayback(base_domain, config)))
+        if "cc" in sources:
+            discovery_tasks.append(("cc", self._discover_via_cc(base_domain, config)))
+
+        # Always guess common subdomains when crt is enabled
+        if "crt" in sources or "probe" in sources:
+            prefixes = list(DEFAULT_COMMON_SUBDOMAINS)
+            if config.common_subdomains:
+                prefixes.extend(config.common_subdomains)
+            discovery_tasks.append(("dns", self._guess_subdomains(base_domain, prefixes, config)))
+
+        # Run all discovery in parallel (per-source timeout)
+        if discovery_tasks:
+            source_timeout = getattr(config, "source_timeout", 30.0)
+            coros = [
+                asyncio.wait_for(t[1], timeout=source_timeout)
+                for t in discovery_tasks
+            ]
+            names = [t[0] for t in discovery_tasks]
+            results = await asyncio.gather(*coros, return_exceptions=True)
+            for name, result in zip(names, results):
+                if isinstance(result, asyncio.TimeoutError):
+                    self._log("warning", "{source} discovery timed out after {timeout}s, skipping",
+                              params={"source": name, "timeout": source_timeout})
+                elif isinstance(result, Exception):
+                    self._log("warning", "{source} discovery failed: {err}",
+                              params={"source": name, "err": str(result)})
+                else:
+                    self._log("info", "{source} discovered {count} hosts: {hosts}",
+                              params={"source": name, "count": len(result),
+                                      "hosts": ", ".join(sorted(result)[:10])})
+                    hosts.update(result)
+
+        # Validate all hosts are actually alive
+        validated = await self._validate_hosts(hosts, config)
+        return validated
+
+    async def _discover_via_crt(self, base_domain: str, config: "DomainMapperConfig") -> Set[str]:
+        """Discover subdomains via Certificate Transparency logs (crt.sh)."""
+        hosts: Set[str] = set()
+        url = f"{CRT_SH_URL}?q=%.{base_domain}&output=json"
+        try:
+            resp = await self.client.get(url, timeout=15, follow_redirects=True)
+            if resp.status_code != 200:
+                self._log("warning", "crt.sh returned HTTP {code}",
+                          params={"code": resp.status_code})
+                return hosts
+
+            entries = resp.json()
+            for entry in entries:
+                for field_name in ("common_name", "name_value"):
+                    val = entry.get(field_name, "")
+                    # name_value can have newline-separated SANs
+                    for name in val.split("\n"):
+                        name = name.strip().lower()
+                        # Skip wildcards
+                        if name.startswith("*."):
+                            name = name[2:]
+                        if not name:
+                            continue
+                        # Must be a subdomain of base_domain
+                        if name == base_domain or name.endswith(f".{base_domain}"):
+                            hosts.add(name)
+        except Exception as e:
+            self._log("warning", "crt.sh query failed: {err}", params={"err": str(e)})
+        return hosts
+
+    async def _discover_via_wayback(self, base_domain: str, config: "DomainMapperConfig") -> Set[str]:
+        """Discover hosts from Wayback Machine CDX API."""
+        hosts: Set[str] = set()
+        # Also store URLs for later use in Phase 2
+        self._wayback_urls: List[str] = []
+        params = {
+            "url": f"*.{base_domain}/*",
+            "output": "text",
+            "fl": "original",
+            "collapse": "urlkey",
+            "limit": "10000",
+        }
+        try:
+            resp = await self.client.get(
+                WAYBACK_CDX_URL, params=params, timeout=30, follow_redirects=True
+            )
+            if resp.status_code != 200:
+                return hosts
+            for line in resp.text.strip().splitlines():
+                url = line.strip()
+                if not url:
+                    continue
+                self._wayback_urls.append(url)
+                parsed = urlparse(url)
+                host = parsed.netloc.lower().split(":")[0]
+                if host and (host == base_domain or host.endswith(f".{base_domain}")):
+                    hosts.add(host)
+        except Exception as e:
+            self._log("warning", "Wayback CDX query failed: {err}", params={"err": str(e)})
+        return hosts
+
+    async def _discover_via_cc(self, base_domain: str, config: "DomainMapperConfig") -> Set[str]:
+        """Extract unique hostnames from Common Crawl results."""
+        hosts: Set[str] = set()
+        try:
+            seeder = await self._get_seeder()
+            # We need the CC index
+            if seeder.index_id is None:
+                seeder.index_id = await seeder._latest_index()
+
+            from .async_configs import SeedingConfig
+            cc_cfg = SeedingConfig(
+                source="cc", max_urls=-1, filter_nonsense_urls=False,
+                extract_head=False, live_check=False, force=config.force,
+            )
+            results = await seeder.urls(base_domain, cc_cfg)
+            for r in results:
+                url = r["url"] if isinstance(r, dict) else r
+                parsed = urlparse(url)
+                host = parsed.netloc.lower().split(":")[0]
+                if host and (host == base_domain or host.endswith(f".{base_domain}")):
+                    hosts.add(host)
+        except Exception as e:
+            self._log("warning", "CC host discovery failed: {err}", params={"err": str(e)})
+        return hosts
+
+    async def _guess_subdomains(
+        self, base_domain: str, prefixes: List[str], config: "DomainMapperConfig"
+    ) -> Set[str]:
+        """Guess common subdomains via DNS resolution."""
+        hosts: Set[str] = set()
+
+        async def check(prefix: str):
+            fqdn = f"{prefix}.{base_domain}"
+            try:
+                loop = asyncio.get_event_loop()
+                await asyncio.wait_for(
+                    loop.getaddrinfo(fqdn, None),
+                    timeout=config.dns_timeout,
+                )
+                return fqdn
+            except Exception:
+                return None
+
+        results = await asyncio.gather(*[check(p) for p in prefixes])
+        for r in results:
+            if r:
+                hosts.add(r)
+        return hosts
+
+    async def _validate_hosts(self, hosts: Set[str], config: "DomainMapperConfig") -> Set[str]:
+        """Validate hosts are reachable via HTTP."""
+        validated: Set[str] = set()
+        # Track redirects so we can map canonical hosts
+        self._host_redirects: Dict[str, str] = {}
+
+        async def check(host: str):
+            for scheme in ("https", "http"):
+                url = f"{scheme}://{host}/"
+                try:
+                    resp = await self.client.head(
+                        url, timeout=config.http_timeout, follow_redirects=False
+                    )
+                    # Record redirect target for dedup
+                    if resp.status_code in (301, 302, 303, 307, 308):
+                        loc = resp.headers.get("location", "")
+                        if loc:
+                            target_host = urlparse(urljoin(url, loc)).netloc.lower().split(":")[0]
+                            if target_host != host:
+                                self._host_redirects[host] = target_host
+                    return host
+                except Exception:
+                    continue
+            return None
+
+        results = await asyncio.gather(*[check(h) for h in hosts])
+        for r in results:
+            if r:
+                validated.add(r)
+        return validated
+
+    # ════════════════════════════════════════════════════════════════════════
+    #  PHASE 2: PER-HOST SCANNING
+    # ════════════════════════════════════════════════════════════════════════
+
+    async def _scan_host(
+        self, host: str, base_domain: str, sources: Set[str], config: "DomainMapperConfig"
+    ) -> List[Dict[str, Any]]:
+        """Run all enabled sources against a single host."""
+        results: List[Dict[str, Any]] = []
+
+        # Soft-404 fingerprint (must run first)
+        soft_404_fp = None
+        if config.soft_404_detection:
+            soft_404_fp = await self._fingerprint_soft_404(host, config)
+            if soft_404_fp and soft_404_fp.status_code == 200:
+                self._log("info", "Soft-404 fingerprint for {host}: title={title}",
+                          params={"host": host, "title": soft_404_fp.title or "(none)"})
+            else:
+                soft_404_fp = None  # Host returns proper 404s, no soft-404 issue
+
+        # robots.txt (feeds into sitemap + probe)
+        sitemap_urls: List[str] = []
+        disallow_paths: List[str] = []
+        if "robots" in sources or "sitemap" in sources:
+            sitemap_urls, disallow_paths = await self._scan_robots_txt(host, config)
+
+        # Gather per-host sources in parallel
+        tasks: Dict[str, Any] = {}
+
+        if "sitemap" in sources:
+            tasks["sitemap"] = self._scan_sitemaps(host, sitemap_urls, config)
+
+        if "probe" in sources or "robots" in sources:
+            # If host has soft-404 fingerprint, add robots paths to probe (which does soft-404 checking)
+            # rather than treating them as known-good
+            probe_paths = list(DEFAULT_PROBE_PATHS)
+            if config.probe_paths:
+                probe_paths.extend(config.probe_paths)
+            if "robots" in sources and disallow_paths:
+                probe_paths.extend(disallow_paths)
+            # Deduplicate
+            probe_paths = list(dict.fromkeys(probe_paths))
+            tasks["probe"] = self._probe_paths(host, probe_paths, soft_404_fp, config)
+
+        if "feed" in sources:
+            tasks["feed"] = self._discover_feeds(host, config)
+
+        if "homepage" in sources:
+            tasks["homepage"] = self._scan_homepage(host, base_domain, config)
+
+        # Run in parallel (per-source timeout)
+        if tasks:
+            source_timeout = getattr(config, "source_timeout", 30.0)
+            source_names = list(tasks.keys())
+            coros = [
+                asyncio.wait_for(c, timeout=source_timeout)
+                for c in tasks.values()
+            ]
+            task_results = await asyncio.gather(*coros, return_exceptions=True)
+
+            for name, result in zip(source_names, task_results):
+                if isinstance(result, asyncio.TimeoutError):
+                    self._log("warning", "{source} scan timed out on {host} after {timeout}s, skipping",
+                              params={"source": name, "host": host, "timeout": source_timeout})
+                    continue
+                if isinstance(result, Exception):
+                    self._log("warning", "{source} scan failed on {host}: {err}",
+                              params={"source": name, "host": host, "err": str(result)})
+                    continue
+
+                # For sitemap URLs on hosts with soft-404: sample-check a few URLs
+                # If all samples are soft-404, skip the entire batch
+                if name == "sitemap" and soft_404_fp and len(result) > 5:
+                    import random
+                    sample = random.sample(result, min(5, len(result)))
+                    soft_404_count = 0
+                    for sample_url in sample:
+                        try:
+                            resp = await self.client.get(
+                                sample_url, timeout=config.http_timeout,
+                                follow_redirects=True,
+                                headers={"Accept-Encoding": "identity"},
+                            )
+                            if self._is_soft_404(resp.status_code, resp.content, soft_404_fp):
+                                soft_404_count += 1
+                        except Exception:
+                            pass
+                    if soft_404_count == len(sample):
+                        self._log("info",
+                                  "Skipping {count} sitemap URLs from {host} (all samples are soft-404)",
+                                  params={"count": len(result), "host": host})
+                        continue
+
+                for url in result:
+                    results.append({
+                        "url": url,
+                        "host": host,
+                        "source": name,
+                        "status": "valid",
+                        "head_data": {},
+                    })
+
+        # Add Wayback URLs that belong to this host
+        if "wayback" in sources and hasattr(self, "_wayback_urls"):
+            for url in self._wayback_urls:
+                parsed = urlparse(url)
+                url_host = parsed.netloc.lower().split(":")[0]
+                if url_host == host:
+                    results.append({
+                        "url": url,
+                        "host": host,
+                        "source": "wayback",
+                        "status": "valid",
+                        "head_data": {},
+                    })
+
+        return results
+
+    # ──────────────────────── Soft-404 Detection
+
+    async def _fingerprint_soft_404(
+        self, host: str, config: "DomainMapperConfig"
+    ) -> Optional[Soft404Fingerprint]:
+        """Fetch a known-bad URL to fingerprint soft-404 responses."""
+        random_path = f"/c4ai-probe-{uuid.uuid4().hex[:12]}"
+        url = f"https://{host}{random_path}"
+        try:
+            resp = await self.client.get(
+                url, timeout=config.http_timeout, follow_redirects=True
+            )
+            body = resp.content[:2048]
+            title = None
+            m = re.search(rb"]*>(.*?)", body, re.I | re.S)
+            if m:
+                title = m.group(1).decode("utf-8", "replace").strip()
+            return Soft404Fingerprint(
+                status_code=resp.status_code,
+                title=title,
+                content_length=int(resp.headers.get("content-length", -1)),
+                body_hash=hashlib.md5(body).hexdigest(),
+            )
+        except Exception:
+            return None
+
+    def _is_soft_404(
+        self, status_code: int, body: bytes, fingerprint: Optional[Soft404Fingerprint]
+    ) -> bool:
+        """Check if a response matches the soft-404 fingerprint."""
+        if fingerprint is None:
+            return False
+        if status_code != 200:
+            return False
+
+        # Check body hash
+        body_hash = hashlib.md5(body[:2048]).hexdigest()
+        if body_hash == fingerprint.body_hash:
+            return True
+
+        # Check title
+        m = re.search(rb"]*>(.*?)", body[:2048], re.I | re.S)
+        if m and fingerprint.title:
+            title = m.group(1).decode("utf-8", "replace").strip()
+            if title == fingerprint.title:
+                return True
+
+        return False
+
+    # ──────────────────────── robots.txt
+
+    async def _scan_robots_txt(
+        self, host: str, config: "DomainMapperConfig"
+    ) -> Tuple[List[str], List[str]]:
+        """Parse robots.txt for Sitemap: and Disallow: directives."""
+        sitemap_urls: List[str] = []
+        disallow_paths: List[str] = []
+
+        for scheme in ("https", "http"):
+            url = f"{scheme}://{host}/robots.txt"
+            try:
+                resp = await self.client.get(url, timeout=config.http_timeout, follow_redirects=True)
+                if resp.status_code != 200:
+                    continue
+
+                for line in resp.text.splitlines():
+                    line = line.strip()
+                    if line.lower().startswith("sitemap:"):
+                        sm_url = line.split(":", 1)[1].strip()
+                        if sm_url:
+                            sitemap_urls.append(sm_url)
+                    elif line.lower().startswith("disallow:"):
+                        path = line.split(":", 1)[1].strip()
+                        # Keep only concrete paths (no wildcards, no empty)
+                        if path and "*" not in path and len(path) > 1:
+                            disallow_paths.append(path)
+                    elif line.lower().startswith("allow:"):
+                        path = line.split(":", 1)[1].strip()
+                        if path and "*" not in path and len(path) > 1:
+                            disallow_paths.append(path)
+
+                self._log("info", "robots.txt for {host}: {sm} sitemaps, {dp} paths",
+                          params={"host": host, "sm": len(sitemap_urls),
+                                  "dp": len(disallow_paths)})
+                return sitemap_urls, list(dict.fromkeys(disallow_paths))
+
+            except Exception:
+                continue
+
+        return sitemap_urls, disallow_paths
+
+    # ──────────────────────── Sitemaps
+
+    async def _scan_sitemaps(
+        self, host: str, sitemap_urls: List[str], config: "DomainMapperConfig"
+    ) -> List[str]:
+        """Discover URLs from sitemaps on a host."""
+        urls: List[str] = []
+
+        # If no sitemaps from robots.txt, try defaults
+        if not sitemap_urls:
+            for scheme in ("https", "http"):
+                for suffix in ("/sitemap.xml", "/sitemap_index.xml"):
+                    candidate = f"{scheme}://{host}{suffix}"
+                    try:
+                        resp = await self.client.head(
+                            candidate, timeout=config.http_timeout, follow_redirects=True
+                        )
+                        if 200 <= resp.status_code < 300:
+                            sitemap_urls.append(candidate)
+                            break
+                    except Exception:
+                        continue
+                if sitemap_urls:
+                    break
+
+        if not sitemap_urls:
+            return urls
+
+        # Use AsyncUrlSeeder's sitemap parsing
+        seeder = await self._get_seeder()
+        for sm_url in sitemap_urls:
+            try:
+                async for u in seeder._iter_sitemap(sm_url):
+                    urls.append(u)
+            except Exception as e:
+                self._log("warning", "Error parsing sitemap {url}: {err}",
+                          params={"url": sm_url, "err": str(e)})
+
+        self._log("info", "Sitemaps for {host}: {count} URLs",
+                  params={"host": host, "count": len(urls)})
+        return urls
+
+    # ──────────────────────── Path Probing
+
+    async def _probe_paths(
+        self,
+        host: str,
+        paths: List[str],
+        soft_404_fp: Optional[Soft404Fingerprint],
+        config: "DomainMapperConfig",
+    ) -> List[str]:
+        """Probe common paths on a host, filtering soft-404s."""
+        valid_urls: List[str] = []
+
+        async def probe_one(path: str) -> Optional[str]:
+            url = f"https://{host}{path}"
+            try:
+                if self._rate_sem:
+                    async with self._rate_sem:
+                        return await self._do_probe(url, soft_404_fp, config)
+                else:
+                    return await self._do_probe(url, soft_404_fp, config)
+            except Exception:
+                return None
+
+        results = await asyncio.gather(*[probe_one(p) for p in paths])
+        for r in results:
+            if r:
+                valid_urls.append(r)
+
+        self._log("info", "Probed {total} paths on {host}: {valid} valid",
+                  params={"total": len(paths), "host": host, "valid": len(valid_urls)})
+        return valid_urls
+
+    async def _do_probe(
+        self, url: str, soft_404_fp: Optional[Soft404Fingerprint], config: "DomainMapperConfig"
+    ) -> Optional[str]:
+        """Probe a single URL: HEAD then GET to check soft-404."""
+        try:
+            resp = await self.client.head(
+                url, timeout=config.http_timeout, follow_redirects=True
+            )
+        except Exception:
+            return None
+
+        if resp.status_code >= 400:
+            return None
+
+        # For 2xx responses, verify against soft-404
+        if soft_404_fp and resp.status_code == 200:
+            try:
+                get_resp = await self.client.get(
+                    url, timeout=config.http_timeout, follow_redirects=True,
+                    headers={"Accept-Encoding": "identity"},
+                )
+                body = get_resp.content[:2048]
+                if self._is_soft_404(get_resp.status_code, body, soft_404_fp):
+                    self._log("debug", "Soft-404 detected: {url}", params={"url": url})
+                    return None
+            except Exception:
+                pass
+
+        # Return the final URL after redirects
+        return str(resp.url)
+
+    # ──────────────────────── Feed Discovery
+
+    async def _discover_feeds(self, host: str, config: "DomainMapperConfig") -> List[str]:
+        """Discover URLs from RSS/Atom feeds."""
+        discovered: List[str] = []
+
+        for feed_path in FEED_PATHS:
+            url = f"https://{host}{feed_path}"
+            try:
+                resp = await self.client.get(
+                    url, timeout=config.http_timeout, follow_redirects=True
+                )
+                if resp.status_code != 200:
+                    continue
+
+                content_type = resp.headers.get("content-type", "").lower()
+                body = resp.text
+
+                # Quick check: is this XML/RSS/Atom?
+                if not any(x in content_type for x in ("xml", "rss", "atom")) and \
+                   not body.strip().startswith(" List[str]:
+        """Parse RSS or Atom feed XML to extract entry URLs."""
+        urls: List[str] = []
+        try:
+            if LXML:
+                parser = etree.XMLParser(recover=True)
+                root = etree.fromstring(xml_text.encode("utf-8", "replace"), parser=parser)
+
+                # RSS: URL
+                for link in root.xpath("//*[local-name()='item']/*[local-name()='link']"):
+                    if link.text and link.text.strip():
+                        urls.append(urljoin(base_url, link.text.strip()))
+
+                # Atom: 
+                for link in root.xpath("//*[local-name()='entry']/*[local-name()='link']"):
+                    href = link.get("href")
+                    if href:
+                        urls.append(urljoin(base_url, href.strip()))
+
+                # RSS: URL
+                if not urls:
+                    for guid in root.xpath("//*[local-name()='item']/*[local-name()='guid']"):
+                        is_permalink = guid.get("isPermaLink", "true").lower()
+                        if is_permalink == "true" and guid.text and guid.text.strip().startswith("http"):
+                            urls.append(guid.text.strip())
+            else:
+                import xml.etree.ElementTree as ET
+                root = ET.fromstring(xml_text)
+                for elem in root.iter():
+                    if "}" in elem.tag:
+                        elem.tag = elem.tag.split("}")[1]
+
+                for item in root.findall(".//item"):
+                    link = item.find("link")
+                    if link is not None and link.text:
+                        urls.append(urljoin(base_url, link.text.strip()))
+
+                for entry in root.findall(".//entry"):
+                    link = entry.find("link")
+                    if link is not None:
+                        href = link.get("href")
+                        if href:
+                            urls.append(urljoin(base_url, href.strip()))
+        except Exception:
+            pass
+        return urls
+
+    # ──────────────────────── Homepage Link Extraction
+
+    async def _scan_homepage(
+        self, host: str, base_domain: str, config: "DomainMapperConfig"
+    ) -> List[str]:
+        """Extract internal links from a host's homepage."""
+        urls: List[str] = []
+        base_url = f"https://{host}/"
+
+        try:
+            resp = await self.client.get(
+                base_url, timeout=config.http_timeout, follow_redirects=True
+            )
+            if resp.status_code != 200:
+                return urls
+
+            html = resp.text
+
+            # Extract  links
+            links = quick_extract_links(html, str(resp.url))
+            for link in links.get("internal", []):
+                href = link.get("href", "")
+                if href:
+                    urls.append(href)
+
+            # Also mine  tags from 
+            head_data = _parse_head(html)
+            for rel, entries in head_data.get("link", {}).items():
+                if rel in ("alternate", "preload", "prefetch", "next", "prev"):
+                    for entry in entries:
+                        href = entry.get("href", "")
+                        if href:
+                            full_url = urljoin(str(resp.url), href)
+                            if not is_external_url(full_url, base_domain):
+                                urls.append(full_url)
+
+            self._log("info", "Homepage {host}: {count} internal links",
+                      params={"host": host, "count": len(urls)})
+
+        except Exception as e:
+            self._log("warning", "Failed to fetch homepage for {host}: {err}",
+                      params={"host": host, "err": str(e)})
+
+        return urls
+
+    # ════════════════════════════════════════════════════════════════════════
+    #  PHASE 3: POST-PROCESSING
+    # ════════════════════════════════════════════════════════════════════════
+
+    def _normalize_and_dedup(
+        self, results: List[Dict[str, Any]], base_domain: str
+    ) -> List[Dict[str, Any]]:
+        """Normalize URLs and deduplicate, merging source attribution."""
+        seen: Dict[str, Dict[str, Any]] = {}
+
+        for r in results:
+            url = r["url"]
+            # Normalize
+            try:
+                normalized = normalize_url(url, f"https://{base_domain}/")
+            except Exception:
+                normalized = url
+
+            if not normalized:
+                continue
+
+            # Strip trailing slash for dedup (keep in output)
+            key = normalized.rstrip("/").lower()
+
+            if key in seen:
+                # Merge source
+                existing_sources = set(seen[key]["source"].split("+"))
+                existing_sources.add(r["source"])
+                seen[key]["source"] = "+".join(sorted(existing_sources))
+            else:
+                r_copy = r.copy()
+                r_copy["url"] = normalized
+                seen[key] = r_copy
+
+        return list(seen.values())
+
+    def _is_nonsense(self, url: str) -> bool:
+        """Reduced nonsense filter for DomainMapper."""
+        parsed = urlparse(url)
+        path = parsed.path.lower()
+
+        # Nonsense suffixes
+        if any(path.endswith(s) for s in _NONSENSE_SUFFIXES):
+            return True
+
+        # Nonsense contains
+        if any(c in path for c in _NONSENSE_CONTAINS):
+            return True
+
+        # Sitemap variations
+        if "/sitemap" in path and path.endswith((".xml", ".xml.gz", ".txt")):
+            return True
+
+        # Static assets (JS, CSS, fonts, images, media, archives)
+        if any(path.endswith(ext) for ext in _ASSET_EXTENSIONS):
+            return True
+
+        # Next.js / webpack chunks
+        if "/_next/" in path or "/webpack" in path:
+            return True
+
+        # Hidden dotfiles
+        parts = path.split("/")
+        if any(p.startswith(".") and p not in (".", "..") for p in parts if p):
+            return True
+
+        # Wayback garbage: URLs with encoded newlines/backslashes
+        if any(x in url for x in ("%5C", "%0A", "%0D", "\\n", "\\r")):
+            return True
+
+        return False
+
+    async def _extract_heads(
+        self, results: List[Dict[str, Any]], config: "DomainMapperConfig"
+    ) -> List[Dict[str, Any]]:
+        """Extract  metadata for all valid URLs."""
+        seeder = await self._get_seeder()
+        sem = asyncio.Semaphore(config.concurrency)
+
+        async def fetch_one(r: Dict[str, Any]):
+            if r.get("head_data"):
+                return  # Already has head data
+            async with sem:
+                if self._rate_sem:
+                    async with self._rate_sem:
+                        await self._do_fetch_head(r, seeder, config)
+                else:
+                    await self._do_fetch_head(r, seeder, config)
+
+        await asyncio.gather(*[fetch_one(r) for r in results], return_exceptions=True)
+        return results
+
+    async def _do_fetch_head(
+        self, r: Dict[str, Any], seeder: AsyncUrlSeeder, config: "DomainMapperConfig"
+    ):
+        """Fetch and parse  for a single result."""
+        try:
+            ok, html, final_url = await seeder._fetch_head(r["url"], int(config.http_timeout))
+            if ok:
+                r["head_data"] = await asyncio.to_thread(_parse_head, html)
+                if final_url and final_url != r["url"]:
+                    r["url"] = final_url
+            else:
+                r["status"] = "not_valid"
+        except Exception:
+            r["status"] = "not_valid"
+
+    async def _apply_scoring(
+        self, results: List[Dict[str, Any]], config: "DomainMapperConfig"
+    ) -> List[Dict[str, Any]]:
+        """Apply BM25 scoring to results with head data."""
+        if not HAS_BM25 or not config.query:
+            return results
+
+        seeder = await self._get_seeder()
+
+        # Build text corpus from head data
+        text_contexts = []
+        scoreable = []
+        for r in results:
+            if r.get("head_data"):
+                text = seeder._extract_text_context(r["head_data"])
+                if text:
+                    text_contexts.append(text)
+                    scoreable.append(r)
+
+        if text_contexts:
+            scores = await asyncio.to_thread(
+                seeder._calculate_bm25_score, config.query, text_contexts
+            )
+            for i, r in enumerate(scoreable):
+                if i < len(scores):
+                    r["relevance_score"] = float(scores[i])
+
+        # Filter by threshold
+        if config.score_threshold is not None:
+            results = [r for r in results if r.get("relevance_score", 0) >= config.score_threshold]
+
+        # Sort by relevance
+        results.sort(key=lambda x: x.get("relevance_score", 0.0), reverse=True)
+        return results
+
+    # ════════════════════════════════════════════════════════════════════════
+    #  CACHING
+    # ════════════════════════════════════════════════════════════════════════
+
+    def _cache_key(self, domain: str, config: "DomainMapperConfig") -> str:
+        """Generate cache key from domain + config."""
+        config_str = f"{config.source}:{config.filter_nonsense_urls}:{config.soft_404_detection}"
+        h = hashlib.md5(config_str.encode()).hexdigest()[:8]
+        safe_domain = re.sub(r"[/?#]+", "_", domain)
+        return f"scan_{safe_domain}_{h}.json"
+
+    def _read_scan_cache(self, domain: str, config: "DomainMapperConfig") -> Optional[List[Dict]]:
+        """Read cached scan results if valid."""
+        if config.force:
+            return None
+        path = self.cache_dir / self._cache_key(domain, config)
+        if not path.exists():
+            return None
+        try:
+            data = json.loads(path.read_text())
+            if data.get("version") != 1:
+                return None
+            created = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
+            age_hours = (datetime.now(timezone.utc) - created).total_seconds() / 3600
+            if age_hours > config.cache_ttl_hours:
+                return None
+            return data.get("results", [])
+        except Exception:
+            return None
+
+    def _write_scan_cache(
+        self, domain: str, config: "DomainMapperConfig", results: List[Dict]
+    ):
+        """Write scan results to cache."""
+        path = self.cache_dir / self._cache_key(domain, config)
+        try:
+            data = {
+                "version": 1,
+                "created_at": datetime.now(timezone.utc).isoformat(),
+                "domain": domain,
+                "result_count": len(results),
+                "results": results,
+            }
+            path.write_text(json.dumps(data, default=str))
+        except Exception:
+            pass
diff --git a/crawl4ai/extraction_strategy.py b/crawl4ai/extraction_strategy.py
new file mode 100644
index 0000000..ed03989
--- /dev/null
+++ b/crawl4ai/extraction_strategy.py
@@ -0,0 +1,2827 @@
+from abc import ABC, abstractmethod
+import inspect
+from typing import Any, List, Dict, Optional, Tuple, Pattern, Union
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import json
+import time
+from enum import IntFlag, auto
+
+from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH, PROMPT_EXTRACT_INFERRED_SCHEMA
+from .config import (
+    DEFAULT_PROVIDER,
+    DEFAULT_PROVIDER_API_KEY,
+    CHUNK_TOKEN_THRESHOLD,
+    OVERLAP_RATE,
+    WORD_TOKEN_RATE,
+    HTML_EXAMPLE_DELIMITER,
+)
+from .utils import *  # noqa: F403
+
+from .utils import (
+    sanitize_html,
+    escape_json_string,
+    perform_completion_with_backoff,
+    extract_xml_data,
+    split_and_parse_json_objects,
+    sanitize_input_encode,
+    merge_chunks,
+)
+from .models import * # noqa: F403
+
+from .models import TokenUsage
+
+from .model_loader import * # noqa: F403
+from .model_loader import (
+    get_device,
+    load_HF_embedding_model,
+    load_text_multilabel_classifier,
+    calculate_batch_size
+)
+
+from .types import LLMConfig, create_llm_config
+
+from functools import partial
+import numpy as np
+import re
+from bs4 import BeautifulSoup
+from lxml import html, etree
+
+
+def _strip_markdown_fences(text: str) -> str:
+    """Strip markdown code fences (e.g. ```json ... ```) from LLM responses."""
+    text = text.strip()
+    return re.sub(
+        r"^```(?:[a-zA-Z0-9_-]+)?\s*|```$", "", text, flags=re.MULTILINE
+    ).strip()
+
+
+def _get_top_level_structure(html_content: str, max_depth: int = 3) -> str:
+    """Return a compact tag outline of the HTML body up to a given depth.
+
+    Used in schema validation feedback when baseSelector matches 0 elements,
+    so the LLM can see what top-level tags actually exist.
+    """
+    try:
+        tree = html.fromstring(html_content)
+    except Exception:
+        return ""
+    body = tree.xpath("//body")
+    root = body[0] if body else tree
+    lines = []
+
+    def _walk(el, depth):
+        if depth > max_depth or not isinstance(el.tag, str):
+            return
+        classes = el.get("class", "").split()
+        cls_str = "." + ".".join(classes) if classes else ""
+        id_str = f"#{el.get('id')}" if el.get("id") else ""
+        lines.append("  " * depth + f"<{el.tag}{id_str}{cls_str}>")
+        for child in el:
+            _walk(child, depth + 1)
+
+    _walk(root, 0)
+    return "\n".join(lines[:60])
+
+
+class ExtractionStrategy(ABC):
+    """
+    Abstract base class for all extraction strategies.
+    """
+
+    def __init__(self, input_format: str = "markdown", **kwargs):
+        """
+        Initialize the extraction strategy.
+
+        Args:
+            input_format: Content format to use for extraction.
+                         Options: "markdown" (default), "html", "fit_markdown"
+            **kwargs: Additional keyword arguments
+        """
+        self.input_format = input_format
+        self.DEL = "<|DEL|>"
+        self.name = self.__class__.__name__
+        self.verbose = kwargs.get("verbose", False)
+
+    @abstractmethod
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML.
+
+        :param url: The URL of the webpage.
+        :param html: The HTML content of the webpage.
+        :return: A list of extracted blocks or chunks.
+        """
+        pass
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process sections of text in parallel by default.
+
+        :param url: The URL of the webpage.
+        :param sections: List of sections (strings) to process.
+        :return: A list of processed JSON blocks.
+        """
+        extracted_content = []
+        with ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self.extract, url, section, **kwargs)
+                for section in sections
+            ]
+            for future in as_completed(futures):
+                extracted_content.extend(future.result())
+        return extracted_content
+
+    async def arun(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Async version: Process sections of text in parallel using asyncio.
+
+        Default implementation runs the sync version in a thread pool.
+        Subclasses can override this for true async processing.
+
+        :param url: The URL of the webpage.
+        :param sections: List of sections (strings) to process.
+        :return: A list of processed JSON blocks.
+        """
+        import asyncio
+        return await asyncio.to_thread(self.run, url, sections, *q, **kwargs)
+
+
+class NoExtractionStrategy(ExtractionStrategy):
+    """
+    A strategy that does not extract any meaningful content from the HTML. It simply returns the entire HTML as a single block.
+    """
+
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML.
+        """
+        return [{"index": 0, "content": html}]
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        return [
+            {"index": i, "tags": [], "content": section}
+            for i, section in enumerate(sections)
+        ]
+
+
+#######################################################
+# Strategies using clustering for text data extraction #
+#######################################################
+
+
+class CosineStrategy(ExtractionStrategy):
+    """
+    Extract meaningful blocks or chunks from the given HTML using cosine similarity.
+
+    How it works:
+    1. Pre-filter documents using embeddings and semantic_filter.
+    2. Perform clustering using cosine similarity.
+    3. Organize texts by their cluster labels, retaining order.
+    4. Filter clusters by word count.
+    5. Extract meaningful blocks or chunks from the filtered clusters.
+
+    Attributes:
+        semantic_filter (str): A keyword filter for document filtering.
+        word_count_threshold (int): Minimum number of words per cluster.
+        max_dist (float): The maximum cophenetic distance on the dendrogram to form clusters.
+        linkage_method (str): The linkage method for hierarchical clustering.
+        top_k (int): Number of top categories to extract.
+        model_name (str): The name of the sentence-transformers model.
+        sim_threshold (float): The similarity threshold for clustering.
+    """
+
+    def __init__(
+        self,
+        semantic_filter=None,
+        word_count_threshold=10,
+        max_dist=0.2,
+        linkage_method="ward",
+        top_k=3,
+        model_name="sentence-transformers/all-MiniLM-L6-v2",
+        sim_threshold=0.3,
+        **kwargs,
+    ):
+        """
+        Initialize the strategy with clustering parameters.
+
+        Args:
+            semantic_filter (str): A keyword filter for document filtering.
+            word_count_threshold (int): Minimum number of words per cluster.
+            max_dist (float): The maximum cophenetic distance on the dendrogram to form clusters.
+            linkage_method (str): The linkage method for hierarchical clustering.
+            top_k (int): Number of top categories to extract.
+        """
+        super().__init__(**kwargs)
+
+        import numpy as np
+
+        self.semantic_filter = semantic_filter
+        self.word_count_threshold = word_count_threshold
+        self.max_dist = max_dist
+        self.linkage_method = linkage_method
+        self.top_k = top_k
+        self.sim_threshold = sim_threshold
+        self.timer = time.time()
+        self.verbose = kwargs.get("verbose", False)
+
+        self.buffer_embeddings = np.array([])
+        self.get_embedding_method = "direct"
+
+        self.device = get_device()
+        # import torch
+        # self.device = torch.device('cpu')
+
+        self.default_batch_size = calculate_batch_size(self.device)
+
+        if self.verbose:
+            print(f"[LOG] Loading Extraction Model for {self.device.type} device.")
+
+        # if False and self.device.type == "cpu":
+        #     self.model = load_onnx_all_MiniLM_l6_v2()
+        #     self.tokenizer = self.model.tokenizer
+        #     self.get_embedding_method = "direct"
+        # else:
+
+        self.tokenizer, self.model = load_HF_embedding_model(model_name)
+        self.model.to(self.device)
+        self.model.eval()
+
+        self.get_embedding_method = "batch"
+
+        self.buffer_embeddings = np.array([])
+
+        # if model_name == "bert-base-uncased":
+        #     self.tokenizer, self.model = load_bert_base_uncased()
+        #     self.model.eval()  # Ensure the model is in evaluation mode
+        #     self.get_embedding_method = "batch"
+        # elif model_name == "BAAI/bge-small-en-v1.5":
+        #     self.tokenizer, self.model = load_bge_small_en_v1_5()
+        #     self.model.eval()  # Ensure the model is in evaluation mode
+        #     self.get_embedding_method = "batch"
+        # elif model_name == "sentence-transformers/all-MiniLM-L6-v2":
+        #     self.model = load_onnx_all_MiniLM_l6_v2()
+        #     self.tokenizer = self.model.tokenizer
+        #     self.get_embedding_method = "direct"
+
+        if self.verbose:
+            print(f"[LOG] Loading Multilabel Classifier for {self.device.type} device.")
+
+        self.nlp, _ = load_text_multilabel_classifier()
+        # self.default_batch_size = 16 if self.device.type == 'cpu' else 64
+
+        if self.verbose:
+            print(
+                f"[LOG] Model loaded {model_name}, models/reuters, took "
+                + str(time.time() - self.timer)
+                + " seconds"
+            )
+
+    def filter_documents_embeddings(
+        self, documents: List[str], semantic_filter: str, at_least_k: int = 20
+    ) -> List[str]:
+        """
+        Filter and sort documents based on the cosine similarity of their embeddings with the semantic_filter embedding.
+
+        Args:
+            documents (List[str]): A list of document texts.
+            semantic_filter (str): A keyword filter for document filtering.
+            at_least_k (int): The minimum number of documents to return.
+
+        Returns:
+            List[str]: A list of filtered and sorted document texts.
+        """
+
+        if not semantic_filter:
+            return documents
+
+        if len(documents) < at_least_k:
+            at_least_k = max(1, len(documents) // 2)
+
+        from sklearn.metrics.pairwise import cosine_similarity
+
+        # Compute embedding for the keyword filter
+        query_embedding = self.get_embeddings([semantic_filter])[0]
+
+        # Compute embeddings for the documents
+        document_embeddings = self.get_embeddings(documents)
+
+        # Calculate cosine similarity between the query embedding and document embeddings
+        similarities = cosine_similarity(
+            [query_embedding], document_embeddings
+        ).flatten()
+
+        # Filter documents based on the similarity threshold
+        filtered_docs = [
+            (doc, sim)
+            for doc, sim in zip(documents, similarities)
+            if sim >= self.sim_threshold
+        ]
+
+        # If the number of filtered documents is less than at_least_k, sort remaining documents by similarity
+        if len(filtered_docs) < at_least_k:
+            remaining_docs = [
+                (doc, sim)
+                for doc, sim in zip(documents, similarities)
+                if sim < self.sim_threshold
+            ]
+            remaining_docs.sort(key=lambda x: x[1], reverse=True)
+            filtered_docs.extend(remaining_docs[: at_least_k - len(filtered_docs)])
+
+        # Extract the document texts from the tuples
+        filtered_docs = [doc for doc, _ in filtered_docs]
+
+        return filtered_docs[:at_least_k]
+
+    def get_embeddings(
+        self, sentences: List[str], batch_size=None, bypass_buffer=False
+    ):
+        """
+        Get BERT embeddings for a list of sentences.
+
+        Args:
+            sentences (List[str]): A list of text chunks (sentences).
+
+        Returns:
+            NumPy array of embeddings.
+        """
+        # if self.buffer_embeddings.any() and not bypass_buffer:
+        #     return self.buffer_embeddings
+
+        if self.device.type in ["cpu", "gpu", "cuda", "mps"]:
+            import torch
+
+            # Tokenize sentences and convert to tensor
+            if batch_size is None:
+                batch_size = self.default_batch_size
+
+            all_embeddings = []
+            for i in range(0, len(sentences), batch_size):
+                batch_sentences = sentences[i : i + batch_size]
+                encoded_input = self.tokenizer(
+                    batch_sentences, padding=True, truncation=True, return_tensors="pt"
+                )
+                encoded_input = {
+                    key: tensor.to(self.device) for key, tensor in encoded_input.items()
+                }
+
+                # Ensure no gradients are calculated
+                with torch.no_grad():
+                    model_output = self.model(**encoded_input)
+
+                # Get embeddings from the last hidden state (mean pooling)
+                embeddings = model_output.last_hidden_state.mean(dim=1).cpu().numpy()
+                all_embeddings.append(embeddings)
+
+            self.buffer_embeddings = np.vstack(all_embeddings)
+        elif self.device.type == "cpu":
+            # self.buffer_embeddings = self.model(sentences)
+            if batch_size is None:
+                batch_size = self.default_batch_size
+
+            all_embeddings = []
+            for i in range(0, len(sentences), batch_size):
+                batch_sentences = sentences[i : i + batch_size]
+                embeddings = self.model(batch_sentences)
+                all_embeddings.append(embeddings)
+
+            self.buffer_embeddings = np.vstack(all_embeddings)
+        return self.buffer_embeddings
+
+    def hierarchical_clustering(self, sentences: List[str], embeddings=None):
+        """
+        Perform hierarchical clustering on sentences and return cluster labels.
+
+        Args:
+            sentences (List[str]): A list of text chunks (sentences).
+
+        Returns:
+            NumPy array of cluster labels.
+        """
+        # Get embeddings
+        from scipy.cluster.hierarchy import linkage, fcluster
+        from scipy.spatial.distance import pdist
+
+        self.timer = time.time()
+        embeddings = self.get_embeddings(sentences, bypass_buffer=True)
+        # print(f"[LOG] 🚀 Embeddings computed in {time.time() - self.timer:.2f} seconds")
+        # Compute pairwise cosine distances
+        distance_matrix = pdist(embeddings, "cosine")
+        # Perform agglomerative clustering respecting order
+        linked = linkage(distance_matrix, method=self.linkage_method)
+        # Form flat clusters
+        labels = fcluster(linked, self.max_dist, criterion="distance")
+        return labels
+
+    def filter_clusters_by_word_count(
+        self, clusters: Dict[int, List[str]]
+    ) -> Dict[int, List[str]]:
+        """
+        Filter clusters to remove those with a word count below the threshold.
+
+        Args:
+            clusters (Dict[int, List[str]]): Dictionary of clusters.
+
+        Returns:
+            Dict[int, List[str]]: Filtered dictionary of clusters.
+        """
+        filtered_clusters = {}
+        for cluster_id, texts in clusters.items():
+            # Concatenate texts for analysis
+            full_text = " ".join(texts)
+            # Count words
+            word_count = len(full_text.split())
+
+            # Keep clusters with word count above the threshold
+            if word_count >= self.word_count_threshold:
+                filtered_clusters[cluster_id] = texts
+
+        return filtered_clusters
+
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract clusters from HTML content using hierarchical clustering.
+
+        Args:
+            url (str): The URL of the webpage.
+            html (str): The HTML content of the webpage.
+
+        Returns:
+            List[Dict[str, Any]]: A list of processed JSON blocks.
+        """
+        # Assume `html` is a list of text chunks for this strategy
+        t = time.time()
+        # Split by delimiter; fall back to double-newline splitting for raw text
+        text_chunks = html.split(self.DEL)
+        if len(text_chunks) == 1:
+            text_chunks = [chunk.strip() for chunk in html.split("\n\n") if chunk.strip()]
+
+        # Pre-filter documents using embeddings and semantic_filter
+        text_chunks = self.filter_documents_embeddings(
+            text_chunks, self.semantic_filter
+        )
+
+        if not text_chunks:
+            return []
+
+        # Perform clustering
+        labels = self.hierarchical_clustering(text_chunks)
+        # print(f"[LOG] 🚀 Clustering done in {time.time() - t:.2f} seconds")
+
+        # Organize texts by their cluster labels, retaining order
+        t = time.time()
+        clusters = {}
+        for index, label in enumerate(labels):
+            clusters.setdefault(label, []).append(text_chunks[index])
+
+        # Filter clusters by word count
+        filtered_clusters = self.filter_clusters_by_word_count(clusters)
+
+        # Convert filtered clusters to a sorted list of dictionaries
+        cluster_list = [
+            {"index": int(idx), "tags": [], "content": " ".join(filtered_clusters[idx])}
+            for idx in sorted(filtered_clusters)
+        ]
+
+        if self.verbose:
+            print(f"[LOG] 🚀 Assign tags using {self.device}")
+
+        if self.device.type in ["gpu", "cuda", "mps", "cpu"]:
+            labels = self.nlp([cluster["content"] for cluster in cluster_list])
+
+            for cluster, label in zip(cluster_list, labels):
+                cluster["tags"] = label
+        # elif self.device.type == "cpu":
+        #     # Process the text with the loaded model
+        #     texts = [cluster['content'] for cluster in cluster_list]
+        #     # Batch process texts
+        #     docs = self.nlp.pipe(texts, disable=["tagger", "parser", "ner", "lemmatizer"])
+
+        #     for doc, cluster in zip(docs, cluster_list):
+        #         tok_k = self.top_k
+        #         top_categories = sorted(doc.cats.items(), key=lambda x: x[1], reverse=True)[:tok_k]
+        #         cluster['tags'] = [cat for cat, _ in top_categories]
+
+        # for cluster in  cluster_list:
+        #     doc = self.nlp(cluster['content'])
+        #     tok_k = self.top_k
+        #     top_categories = sorted(doc.cats.items(), key=lambda x: x[1], reverse=True)[:tok_k]
+        #     cluster['tags'] = [cat for cat, _ in top_categories]
+
+        if self.verbose:
+            print(f"[LOG] 🚀 Categorization done in {time.time() - t:.2f} seconds")
+
+        return cluster_list
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process sections using hierarchical clustering.
+
+        Args:
+            url (str): The URL of the webpage.
+            sections (List[str]): List of sections (strings) to process.
+
+        Returns:
+        """
+        # This strategy processes all sections together
+
+        return self.extract(url, self.DEL.join(sections), **kwargs)
+
+
+#######################################################
+# Strategies using LLM-based extraction for text data #
+#######################################################
+class LLMExtractionStrategy(ExtractionStrategy):
+    """
+    A strategy that uses an LLM to extract meaningful content from the HTML.
+
+    Attributes:
+        llm_config: The LLM configuration object.
+        instruction: The instruction to use for the LLM model.
+        schema: Pydantic model schema for structured data.
+        extraction_type: "block" or "schema".
+        chunk_token_threshold: Maximum tokens per chunk.
+        overlap_rate: Overlap between chunks.
+        word_token_rate: Word to token conversion rate.
+        apply_chunking: Whether to apply chunking.
+        verbose: Whether to print verbose output.
+        usages: List of individual token usages.
+        total_usage: Accumulated token usage.
+    """
+    _UNWANTED_PROPS = {
+            'provider' : 'Instead, use llm_config=LLMConfig(provider="...")',
+            'api_token' : 'Instead, use llm_config=LlMConfig(api_token="...")',
+            'base_url' : 'Instead, use llm_config=LLMConfig(base_url="...")',
+            'api_base' : 'Instead, use llm_config=LLMConfig(base_url="...")',
+        }
+    def __init__(
+        self,
+        llm_config: 'LLMConfig' = None,
+        instruction: str = None,
+        schema: Dict = None,
+        extraction_type="schema",
+        chunk_token_threshold=CHUNK_TOKEN_THRESHOLD,
+        overlap_rate=OVERLAP_RATE,
+        word_token_rate=WORD_TOKEN_RATE,
+        apply_chunking=True,
+        input_format: str = "markdown",
+        force_json_response=False,
+        verbose=False,
+        # Deprecated arguments
+        provider: str = DEFAULT_PROVIDER,
+        api_token: Optional[str] = None,
+        base_url: str = None,
+        api_base: str = None,
+        **kwargs,
+    ):
+        """
+        Initialize the strategy with clustering parameters.
+
+        Args:
+            llm_config: The LLM configuration object.
+            instruction: The instruction to use for the LLM model.
+            schema: Pydantic model schema for structured data.
+            extraction_type: "block" or "schema".
+            chunk_token_threshold: Maximum tokens per chunk.
+            overlap_rate: Overlap between chunks.
+            word_token_rate: Word to token conversion rate.
+            apply_chunking: Whether to apply chunking.
+            input_format: Content format to use for extraction.
+                            Options: "markdown" (default), "html", "fit_markdown"
+            force_json_response: Whether to force a JSON response from the LLM.
+            verbose: Whether to print verbose output.
+
+            # Deprecated arguments, will be removed very soon
+            provider: The provider to use for extraction. It follows the format /, e.g., "ollama/llama3.3".
+            api_token: The API token for the provider.
+            base_url: The base URL for the API request.
+            api_base: The base URL for the API request.
+            extra_args: Additional arguments for the API request, such as temperature, max_tokens, etc.
+        """
+        super().__init__( input_format=input_format, **kwargs)
+        self.llm_config = llm_config
+        if not self.llm_config:
+            self.llm_config = create_llm_config(
+                provider=DEFAULT_PROVIDER,
+                api_token=os.environ.get(DEFAULT_PROVIDER_API_KEY),
+            )
+        self.instruction = instruction
+        self.extract_type = extraction_type
+        self.schema = schema
+        if schema:
+            self.extract_type = "schema"
+        self.force_json_response = force_json_response
+        self.chunk_token_threshold = chunk_token_threshold or CHUNK_TOKEN_THRESHOLD
+        self.overlap_rate = overlap_rate
+        self.word_token_rate = word_token_rate
+        self.apply_chunking = apply_chunking
+        self.extra_args = kwargs.get("extra_args", {})
+        if not self.apply_chunking:
+            self.chunk_token_threshold = 1e9
+        self.verbose = verbose
+        self.usages = []  # Store individual usages
+        self.total_usage = TokenUsage()  # Accumulated usage
+
+        self.provider = provider
+        self.api_token = api_token
+        self.base_url = base_url
+        self.api_base = api_base
+
+    
+    def __setattr__(self, name, value):
+        """Handle attribute setting."""
+        # TODO: Planning to set properties dynamically based on the __init__ signature
+        sig = inspect.signature(self.__init__)
+        all_params = sig.parameters  # Dictionary of parameter names and their details
+
+        if name in self._UNWANTED_PROPS and value is not all_params[name].default:
+            raise AttributeError(f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}")
+        
+        super().__setattr__(name, value)  
+        
+    def extract(self, url: str, ix: int, html: str) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML using an LLM.
+
+        How it works:
+        1. Construct a prompt with variables.
+        2. Make a request to the LLM using the prompt.
+        3. Parse the response and extract blocks or chunks.
+
+        Args:
+            url: The URL of the webpage.
+            ix: Index of the block.
+            html: The HTML content of the webpage.
+
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        if self.verbose:
+            # print("[LOG] Extracting blocks from URL:", url)
+            print(f"[LOG] Call LLM for {url} - block index: {ix}")
+
+        variable_values = {
+            "URL": url,
+            "HTML": escape_json_string(sanitize_html(html)),
+        }
+
+        prompt_with_variables = PROMPT_EXTRACT_BLOCKS
+        if self.instruction:
+            variable_values["REQUEST"] = self.instruction
+            prompt_with_variables = PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION
+
+        if self.extract_type == "schema" and self.schema:
+            variable_values["SCHEMA"] = json.dumps(self.schema, indent=2) # if type of self.schema is dict else self.schema
+            prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
+
+        if self.extract_type == "schema" and not self.schema:
+            prompt_with_variables = PROMPT_EXTRACT_INFERRED_SCHEMA
+
+        for variable in variable_values:
+            prompt_with_variables = prompt_with_variables.replace(
+                "{" + variable + "}", variable_values[variable]
+            )
+
+        try:
+            response = perform_completion_with_backoff(
+                self.llm_config.provider,
+                prompt_with_variables,
+                self.llm_config.api_token,
+                base_url=self.llm_config.base_url,
+                json_response=self.force_json_response,
+                extra_args=self.extra_args,
+                base_delay=self.llm_config.backoff_base_delay,
+                max_attempts=self.llm_config.backoff_max_attempts,
+                exponential_factor=self.llm_config.backoff_exponential_factor
+            )  # , json_response=self.extract_type == "schema")
+            # Track usage
+            usage = TokenUsage(
+                completion_tokens=response.usage.completion_tokens,
+                prompt_tokens=response.usage.prompt_tokens,
+                total_tokens=response.usage.total_tokens,
+                completion_tokens_details=response.usage.completion_tokens_details.__dict__
+                if response.usage.completion_tokens_details
+                else {},
+                prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
+                if response.usage.prompt_tokens_details
+                else {},
+            )
+            self.usages.append(usage)
+
+            # Update totals
+            self.total_usage.completion_tokens += usage.completion_tokens
+            self.total_usage.prompt_tokens += usage.prompt_tokens
+            self.total_usage.total_tokens += usage.total_tokens
+
+            try:
+                content = response.choices[0].message.content
+                blocks = None
+
+                if not content:
+                    finish_reason = getattr(response.choices[0], "finish_reason", "unknown")
+                    blocks = [{"index": 0, "error": True, "tags": ["error"],
+                               "content": f"LLM returned no content (finish_reason: {finish_reason})"}]
+                elif self.force_json_response:
+                    blocks = json.loads(_strip_markdown_fences(content))
+                    if isinstance(blocks, dict):
+                        # If it has only one key which calue is list then assign that to blocks, exampled: {"news": [..]}
+                        if len(blocks) == 1 and isinstance(list(blocks.values())[0], list):
+                            blocks = list(blocks.values())[0]
+                        else:
+                            # If it has only one key which value is not list then assign that to blocks, exampled: { "article_id": "1234", ... }
+                            blocks = [blocks]
+                    elif isinstance(blocks, list):
+                        # If it is a list then assign that to blocks
+                        blocks = blocks
+                else: 
+                    # blocks = extract_xml_data(["blocks"], response.choices[0].message.content)["blocks"]
+                    blocks = extract_xml_data(["blocks"], content)["blocks"]
+                    blocks = json.loads(blocks)
+
+                for block in blocks:
+                    block["error"] = False
+            except Exception:
+                raw_content = response.choices[0].message.content or ""
+                parsed, unparsed = split_and_parse_json_objects(raw_content)
+                blocks = parsed
+                if unparsed:
+                    blocks.append(
+                        {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
+                    )
+
+            if self.verbose:
+                print(
+                    "[LOG] Extracted",
+                    len(blocks),
+                    "blocks from URL:",
+                    url,
+                    "block index:",
+                    ix,
+                )
+            return blocks
+        except Exception as e:
+            if self.verbose:
+                print(f"[LOG] Error in LLM extraction: {e}")
+            # Add error information to extracted_content
+            return [
+                {
+                    "index": ix,
+                    "error": True,
+                    "tags": ["error"],
+                    "content": str(e),
+                }
+            ]
+
+    def _merge(self, documents, chunk_token_threshold, overlap) -> List[str]:
+        """
+        Merge documents into sections based on chunk_token_threshold and overlap.
+        """
+        sections =  merge_chunks(
+            docs = documents,
+            target_size= chunk_token_threshold,
+            overlap=overlap,
+            word_token_ratio=self.word_token_rate
+        )
+        return sections
+
+    def run(self, url: str, sections: List[str]) -> List[Dict[str, Any]]:
+        """
+        Process sections sequentially with a delay for rate limiting issues, specifically for LLMExtractionStrategy.
+
+        Args:
+            url: The URL of the webpage.
+            sections: List of sections (strings) to process.
+
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+
+        merged_sections = self._merge(
+            sections,
+            self.chunk_token_threshold,
+            overlap=int(self.chunk_token_threshold * self.overlap_rate),
+        )
+        extracted_content = []
+        if self.llm_config.provider.startswith("groq/"):
+            # Sequential processing with a delay
+            for ix, section in enumerate(merged_sections):
+                extract_func = partial(self.extract, url)
+                extracted_content.extend(
+                    extract_func(ix, sanitize_input_encode(section))
+                )
+                time.sleep(0.5)  # 500 ms delay between each processing
+        else:
+            # Parallel processing using ThreadPoolExecutor
+            # extract_func = partial(self.extract, url)
+            # for ix, section in enumerate(merged_sections):
+            #     extracted_content.append(extract_func(ix, section))
+
+            with ThreadPoolExecutor(max_workers=4) as executor:
+                extract_func = partial(self.extract, url)
+                futures = [
+                    executor.submit(extract_func, ix, sanitize_input_encode(section))
+                    for ix, section in enumerate(merged_sections)
+                ]
+
+                for future in as_completed(futures):
+                    try:
+                        extracted_content.extend(future.result())
+                    except Exception as e:
+                        if self.verbose:
+                            print(f"Error in thread execution: {e}")
+                        # Add error information to extracted_content
+                        extracted_content.append(
+                            {
+                                "index": 0,
+                                "error": True,
+                                "tags": ["error"],
+                                "content": str(e),
+                            }
+                        )
+
+        return extracted_content
+
+    async def aextract(self, url: str, ix: int, html: str) -> List[Dict[str, Any]]:
+        """
+        Async version: Extract meaningful blocks or chunks from the given HTML using an LLM.
+
+        How it works:
+        1. Construct a prompt with variables.
+        2. Make an async request to the LLM using the prompt.
+        3. Parse the response and extract blocks or chunks.
+
+        Args:
+            url: The URL of the webpage.
+            ix: Index of the block.
+            html: The HTML content of the webpage.
+
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        from .utils import aperform_completion_with_backoff
+
+        if self.verbose:
+            print(f"[LOG] Call LLM for {url} - block index: {ix}")
+
+        variable_values = {
+            "URL": url,
+            "HTML": escape_json_string(sanitize_html(html)),
+        }
+
+        prompt_with_variables = PROMPT_EXTRACT_BLOCKS
+        if self.instruction:
+            variable_values["REQUEST"] = self.instruction
+            prompt_with_variables = PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION
+
+        if self.extract_type == "schema" and self.schema:
+            variable_values["SCHEMA"] = json.dumps(self.schema, indent=2)
+            prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
+
+        if self.extract_type == "schema" and not self.schema:
+            prompt_with_variables = PROMPT_EXTRACT_INFERRED_SCHEMA
+
+        for variable in variable_values:
+            prompt_with_variables = prompt_with_variables.replace(
+                "{" + variable + "}", variable_values[variable]
+            )
+
+        try:
+            response = await aperform_completion_with_backoff(
+                self.llm_config.provider,
+                prompt_with_variables,
+                self.llm_config.api_token,
+                base_url=self.llm_config.base_url,
+                json_response=self.force_json_response,
+                extra_args=self.extra_args,
+                base_delay=self.llm_config.backoff_base_delay,
+                max_attempts=self.llm_config.backoff_max_attempts,
+                exponential_factor=self.llm_config.backoff_exponential_factor
+            )
+            # Track usage
+            usage = TokenUsage(
+                completion_tokens=response.usage.completion_tokens,
+                prompt_tokens=response.usage.prompt_tokens,
+                total_tokens=response.usage.total_tokens,
+                completion_tokens_details=response.usage.completion_tokens_details.__dict__
+                if response.usage.completion_tokens_details
+                else {},
+                prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
+                if response.usage.prompt_tokens_details
+                else {},
+            )
+            self.usages.append(usage)
+
+            # Update totals
+            self.total_usage.completion_tokens += usage.completion_tokens
+            self.total_usage.prompt_tokens += usage.prompt_tokens
+            self.total_usage.total_tokens += usage.total_tokens
+
+            try:
+                content = response.choices[0].message.content
+                blocks = None
+
+                if not content:
+                    finish_reason = getattr(response.choices[0], "finish_reason", "unknown")
+                    blocks = [{"index": 0, "error": True, "tags": ["error"],
+                               "content": f"LLM returned no content (finish_reason: {finish_reason})"}]
+                elif self.force_json_response:
+                    blocks = json.loads(_strip_markdown_fences(content))
+                    if isinstance(blocks, dict):
+                        if len(blocks) == 1 and isinstance(list(blocks.values())[0], list):
+                            blocks = list(blocks.values())[0]
+                        else:
+                            blocks = [blocks]
+                    elif isinstance(blocks, list):
+                        blocks = blocks
+                else:
+                    blocks = extract_xml_data(["blocks"], content)["blocks"]
+                    blocks = json.loads(blocks)
+
+                for block in blocks:
+                    block["error"] = False
+            except Exception:
+                raw_content = response.choices[0].message.content or ""
+                parsed, unparsed = split_and_parse_json_objects(raw_content)
+                blocks = parsed
+                if unparsed:
+                    blocks.append(
+                        {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
+                    )
+
+            if self.verbose:
+                print(
+                    "[LOG] Extracted",
+                    len(blocks),
+                    "blocks from URL:",
+                    url,
+                    "block index:",
+                    ix,
+                )
+            return blocks
+        except Exception as e:
+            if self.verbose:
+                print(f"[LOG] Error in LLM extraction: {e}")
+            return [
+                {
+                    "index": ix,
+                    "error": True,
+                    "tags": ["error"],
+                    "content": str(e),
+                }
+            ]
+
+    async def arun(self, url: str, sections: List[str]) -> List[Dict[str, Any]]:
+        """
+        Async version: Process sections with true parallelism using asyncio.gather.
+
+        Args:
+            url: The URL of the webpage.
+            sections: List of sections (strings) to process.
+
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        import asyncio
+
+        merged_sections = self._merge(
+            sections,
+            self.chunk_token_threshold,
+            overlap=int(self.chunk_token_threshold * self.overlap_rate),
+        )
+
+        extracted_content = []
+
+        # Create tasks for all sections to run in parallel
+        tasks = [
+            self.aextract(url, ix, sanitize_input_encode(section))
+            for ix, section in enumerate(merged_sections)
+        ]
+
+        # Execute all tasks concurrently
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Process results
+        for result in results:
+            if isinstance(result, Exception):
+                if self.verbose:
+                    print(f"Error in async extraction: {result}")
+                extracted_content.append(
+                    {
+                        "index": 0,
+                        "error": True,
+                        "tags": ["error"],
+                        "content": str(result),
+                    }
+                )
+            else:
+                extracted_content.extend(result)
+
+        return extracted_content
+
+    def show_usage(self) -> None:
+        """Print a detailed token usage report showing total and per-request usage."""
+        print("\n=== Token Usage Summary ===")
+        print(f"{'Type':<15} {'Count':>12}")
+        print("-" * 30)
+        print(f"{'Completion':<15} {self.total_usage.completion_tokens:>12,}")
+        print(f"{'Prompt':<15} {self.total_usage.prompt_tokens:>12,}")
+        print(f"{'Total':<15} {self.total_usage.total_tokens:>12,}")
+
+        print("\n=== Usage History ===")
+        print(f"{'Request #':<10} {'Completion':>12} {'Prompt':>12} {'Total':>12}")
+        print("-" * 48)
+        for i, usage in enumerate(self.usages, 1):
+            print(
+                f"{i:<10} {usage.completion_tokens:>12,} {usage.prompt_tokens:>12,} {usage.total_tokens:>12,}"
+            )
+
+
+#######################################################
+# New extraction strategies for JSON-based extraction #
+#######################################################
+
+
+class JsonElementExtractionStrategy(ExtractionStrategy):
+    """
+    Abstract base class for extracting structured JSON from HTML content.
+
+    How it works:
+    1. Parses HTML content using the `_parse_html` method.
+    2. Uses a schema to define base selectors, fields, and transformations.
+    3. Extracts data hierarchically, supporting nested fields and lists.
+    4. Handles computed fields with expressions or functions.
+
+    Attributes:
+        DEL (str): Delimiter used to combine HTML sections. Defaults to '\n'.
+        schema (Dict[str, Any]): The schema defining the extraction rules.
+        verbose (bool): Enables verbose logging for debugging purposes.
+
+    Methods:
+        extract(url, html_content, *q, **kwargs): Extracts structured data from HTML content.
+        _extract_item(element, fields): Extracts fields from a single element.
+        _extract_single_field(element, field): Extracts a single field based on its type.
+        _apply_transform(value, transform): Applies a transformation to a value.
+        _compute_field(item, field): Computes a field value using an expression or function.
+        run(url, sections, *q, **kwargs): Combines HTML sections and runs the extraction strategy.
+
+    Abstract Methods:
+        _parse_html(html_content): Parses raw HTML into a structured format (e.g., BeautifulSoup or lxml).
+        _get_base_elements(parsed_html, selector): Retrieves base elements using a selector.
+        _get_elements(element, selector): Retrieves child elements using a selector.
+        _get_element_text(element): Extracts text content from an element.
+        _get_element_html(element): Extracts raw HTML from an element.
+        _get_element_attribute(element, attribute): Extracts an attribute's value from an element.
+    """
+
+    DEL = "\n"
+
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        """
+        Initialize the JSON element extraction strategy with a schema.
+
+        Args:
+            schema (Dict[str, Any]): The schema defining the extraction rules.
+        """
+        super().__init__(**kwargs)
+        self.schema = schema
+        self.verbose = kwargs.get("verbose", False)
+
+    def extract(
+        self, url: str, html_content: str, *q, **kwargs
+    ) -> List[Dict[str, Any]]:
+        """
+        Extract structured data from HTML content.
+
+        How it works:
+        1. Parses the HTML content using the `_parse_html` method.
+        2. Identifies base elements using the schema's base selector.
+        3. Extracts fields from each base element using `_extract_item`.
+
+        Args:
+            url (str): The URL of the page being processed.
+            html_content (str): The raw HTML content to parse and extract.
+            *q: Additional positional arguments.
+            **kwargs: Additional keyword arguments for custom extraction.
+
+        Returns:
+            List[Dict[str, Any]]: A list of extracted items, each represented as a dictionary.
+        """
+
+        parsed_html = self._parse_html(html_content)
+        base_elements = self._get_base_elements(
+            parsed_html, self.schema["baseSelector"]
+        )
+
+        results = []
+        for element in base_elements:
+            # Extract base element attributes
+            item = {}
+            if "baseFields" in self.schema:
+                for field in self.schema["baseFields"]:
+                    value = self._extract_single_field(element, field)
+                    if value is not None:
+                        item[field["name"]] = value
+
+            # Extract child fields
+            field_data = self._extract_item(element, self.schema["fields"])
+            item.update(field_data)
+
+            if item:
+                results.append(item)
+
+        return results
+
+    @abstractmethod
+    def _parse_html(self, html_content: str):
+        """Parse HTML content into appropriate format"""
+        pass
+
+    @abstractmethod
+    def _get_base_elements(self, parsed_html, selector: str):
+        """Get all base elements using the selector"""
+        pass
+
+    @abstractmethod
+    def _get_elements(self, element, selector: str):
+        """Get child elements using the selector"""
+        pass
+
+    def _extract_field(self, element, field):
+        try:
+            if "source" in field:
+                element = self._resolve_source(element, field["source"])
+                if element is None:
+                    return field.get("default")
+
+            if field["type"] == "nested":
+                nested_elements = self._get_elements(element, field["selector"])
+                nested_element = nested_elements[0] if nested_elements else None
+                return (
+                    self._extract_item(nested_element, field["fields"])
+                    if nested_element
+                    else {}
+                )
+
+            if field["type"] == "list":
+                elements = self._get_elements(element, field["selector"])
+                return [self._extract_list_item(el, field["fields"]) for el in elements]
+
+            if field["type"] == "nested_list":
+                elements = self._get_elements(element, field["selector"])
+                return [self._extract_item(el, field["fields"]) for el in elements]
+
+            return self._extract_single_field(element, field)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error extracting field {field['name']}: {str(e)}")
+            return field.get("default")
+
+    def _extract_single_field(self, element, field):
+        """
+        Extract a single field based on its type.
+
+        How it works:
+        1. Selects the target element using the field's selector.
+        2. Extracts the field value based on its type (e.g., text, attribute, regex).
+        3. Applies transformations if defined in the schema.
+
+        Args:
+            element: The base element to extract the field from.
+            field (Dict[str, Any]): The field definition in the schema.
+
+        Returns:
+            Any: The extracted field value.
+        """
+
+        if "selector" in field:
+            selected = self._get_elements(element, field["selector"])
+            if not selected:
+                return field.get("default")
+            selected = selected[0]
+        else:
+            selected = element
+
+        type_pipeline = field["type"]
+        if not isinstance(type_pipeline, list):
+            type_pipeline = [type_pipeline]
+        value = selected
+        for step in type_pipeline:
+            if step == "text":
+                value = self._get_element_text(value)
+            elif step == "attribute":
+                value = self._get_element_attribute(value, field["attribute"])
+            elif step == "html":
+                value = self._get_element_html(value)
+            elif step == "regex":
+                pattern = field.get("pattern")
+                if pattern:
+                    # If value is still an element, extract text first (backward compat)
+                    if not isinstance(value, str):
+                        value = self._get_element_text(value)
+                    if isinstance(value, str):
+                        match = re.search(pattern, value)
+                        value = match.group(field.get("group", 1)) if match else None
+                    else:
+                        value = None
+            if value is None:
+                break
+
+        if "transform" in field:
+            value = self._apply_transform(value, field["transform"])
+
+        return value if value is not None else field.get("default")
+
+    def _extract_list_item(self, element, fields):
+        item = {}
+        for field in fields:
+            value = self._extract_single_field(element, field)
+            if value is not None:
+                item[field["name"]] = value
+        return item
+
+    def _extract_item(self, element, fields):
+        """
+        Extracts fields from a given element.
+
+        How it works:
+        1. Iterates through the fields defined in the schema.
+        2. Handles computed, single, and nested field types.
+        3. Updates the item dictionary with extracted field values.
+
+        Args:
+            element: The base element to extract fields from.
+            fields (List[Dict[str, Any]]): The list of fields to extract.
+
+        Returns:
+            Dict[str, Any]: A dictionary representing the extracted item.
+        """
+
+        item = {}
+        for field in fields:
+            if field["type"] == "computed":
+                value = self._compute_field(item, field)
+            else:
+                value = self._extract_field(element, field)
+            if value is not None:
+                item[field["name"]] = value
+        return item
+
+    def _apply_transform(self, value, transform):
+        """
+        Apply a transformation to a value.
+
+        How it works:
+        1. Checks the transformation type (e.g., `lowercase`, `strip`).
+        2. Applies the transformation to the value.
+        3. Returns the transformed value.
+
+        Args:
+            value (str): The value to transform.
+            transform (str): The type of transformation to apply.
+
+        Returns:
+            str: The transformed value.
+        """
+
+        if transform == "lowercase":
+            return value.lower()
+        elif transform == "uppercase":
+            return value.upper()
+        elif transform == "strip":
+            return value.strip()
+        return value
+
+    def _compute_field(self, item, field):
+        try:
+            if "expression" in field:
+                import logging
+                logging.getLogger(__name__).warning(
+                    "Computed field 'expression' is disabled for security "
+                    "(eval on untrusted input). Use 'function' key with a "
+                    "Python callable instead."
+                )
+                return field.get("default")
+            elif "function" in field:
+                return field["function"](item)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error computing field {field['name']}: {str(e)}")
+            return field.get("default")
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Run the extraction strategy on a combined HTML content.
+
+        How it works:
+        1. Combines multiple HTML sections using the `DEL` delimiter.
+        2. Calls the `extract` method with the combined HTML.
+
+        Args:
+            url (str): The URL of the page being processed.
+            sections (List[str]): A list of HTML sections.
+            *q: Additional positional arguments.
+            **kwargs: Additional keyword arguments for custom extraction.
+
+        Returns:
+            List[Dict[str, Any]]: A list of extracted items.
+        """
+
+        combined_html = self.DEL.join(sections)
+        return self.extract(url, combined_html, **kwargs)
+
+    @abstractmethod
+    def _get_element_text(self, element) -> str:
+        """Get text content from element"""
+        pass
+
+    @abstractmethod
+    def _get_element_html(self, element) -> str:
+        """Get HTML content from element"""
+        pass
+
+    @abstractmethod
+    def _get_element_attribute(self, element, attribute: str):
+        """Get attribute value from element"""
+        pass
+
+    @abstractmethod
+    def _resolve_source(self, element, source: str):
+        """Navigate to a sibling element relative to the base element.
+
+        Used when a field's data lives in a sibling of the base element
+        rather than a descendant. For example, Hacker News splits each
+        submission across two sibling
rows. + + Args: + element: The current base element. + source: A sibling selector string. Currently supports the + ``"+ "`` syntax which navigates to the next + sibling matching ````. + + Returns: + The resolved sibling element, or ``None`` if not found. + """ + pass + + @staticmethod + def _validate_schema( + schema: dict, + html_content: str, + schema_type: str = "CSS", + expected_fields: Optional[List[str]] = None, + ) -> dict: + """Run the generated schema against HTML and return a diagnostic result. + + Args: + schema: The extraction schema to validate. + html_content: The HTML to validate against. + schema_type: "CSS" or "XPATH". + expected_fields: When provided, enables strict mode — success + requires ALL expected fields to be present and populated. + When None, uses fuzzy mode (populated_fields > 0). + + Returns a dict with keys: success, base_elements_found, total_fields, + populated_fields, field_coverage, field_details, issues, + sample_base_html, top_level_structure. + """ + result = { + "success": False, + "base_elements_found": 0, + "total_fields": 0, + "populated_fields": 0, + "field_coverage": 0.0, + "field_details": [], + "issues": [], + "sample_base_html": "", + "top_level_structure": "", + } + + try: + StrategyClass = ( + JsonCssExtractionStrategy + if schema_type.upper() == "CSS" + else JsonXPathExtractionStrategy + ) + strategy = StrategyClass(schema=schema) + items = strategy.extract(url="", html_content=html_content) + except Exception as e: + result["issues"].append(f"Extraction crashed: {e}") + return result + + # Count base elements directly + try: + parsed = strategy._parse_html(html_content) + base_elements = strategy._get_base_elements(parsed, schema["baseSelector"]) + result["base_elements_found"] = len(base_elements) + + # Grab sample innerHTML of first base element (truncated) + if base_elements: + sample = strategy._get_element_html(base_elements[0]) + result["sample_base_html"] = sample[:2000] + except Exception: + pass + + if result["base_elements_found"] == 0: + result["issues"].append( + f"baseSelector '{schema.get('baseSelector', '')}' matched 0 elements" + ) + result["top_level_structure"] = _get_top_level_structure(html_content) + return result + + # Analyze field coverage + all_fields = schema.get("fields", []) + field_names = [f["name"] for f in all_fields] + result["total_fields"] = len(field_names) + + for fname in field_names: + values = [item.get(fname) for item in items] + populated_count = sum(1 for v in values if v is not None and v != "") + sample_val = next((v for v in values if v is not None and v != ""), None) + if sample_val is not None: + sample_val = str(sample_val)[:120] + result["field_details"].append({ + "name": fname, + "populated_count": populated_count, + "total_count": len(items), + "sample_value": sample_val, + }) + + result["populated_fields"] = sum( + 1 for fd in result["field_details"] if fd["populated_count"] > 0 + ) + if result["total_fields"] > 0: + result["field_coverage"] = result["populated_fields"] / result["total_fields"] + + # Build issues + if result["populated_fields"] == 0: + result["issues"].append( + "All fields returned None/empty — selectors likely wrong" + ) + else: + empty_fields = [ + fd["name"] + for fd in result["field_details"] + if fd["populated_count"] == 0 + ] + if empty_fields: + result["issues"].append( + f"Fields always empty: {', '.join(empty_fields)}" + ) + + # Check for missing expected fields (strict mode) + if expected_fields: + schema_field_names = {f["name"] for f in schema.get("fields", [])} + missing = [f for f in expected_fields if f not in schema_field_names] + if missing: + result["issues"].append( + f"Expected fields missing from schema: {', '.join(missing)}" + ) + + # Success criteria + if expected_fields: + # Strict: all expected fields must exist in schema AND be populated + schema_field_names = {f["name"] for f in schema.get("fields", [])} + populated_names = { + fd["name"] for fd in result["field_details"] if fd["populated_count"] > 0 + } + result["success"] = ( + result["base_elements_found"] > 0 + and all(f in populated_names for f in expected_fields) + ) + else: + # Fuzzy: at least something extracted + result["success"] = ( + result["base_elements_found"] > 0 and result["populated_fields"] > 0 + ) + return result + + @staticmethod + def _build_feedback_message( + validation_result: dict, + schema: dict, + attempt: int, + is_repeated: bool, + ) -> str: + """Build a structured feedback message from a validation result.""" + vr = validation_result + parts = [] + + parts.append(f"## Schema Validation — Attempt {attempt}") + + # Base selector + if vr["base_elements_found"] == 0: + parts.append( + f"**CRITICAL:** baseSelector `{schema.get('baseSelector', '')}` " + f"matched **0 elements**. The schema cannot extract anything." + ) + if vr["top_level_structure"]: + parts.append( + "Here is the top-level HTML structure so you can pick a valid selector:\n```\n" + + vr["top_level_structure"] + + "\n```" + ) + else: + parts.append( + f"baseSelector matched **{vr['base_elements_found']}** element(s)." + ) + + # Field coverage table + if vr["field_details"]: + parts.append( + f"\n**Field coverage:** {vr['populated_fields']}/{vr['total_fields']} fields have data\n" + ) + parts.append("| Field | Populated | Sample |") + parts.append("|-------|-----------|--------|") + for fd in vr["field_details"]: + sample = fd["sample_value"] or "*(empty)*" + parts.append( + f"| {fd['name']} | {fd['populated_count']}/{fd['total_count']} | {sample} |" + ) + + # Issues + if vr["issues"]: + parts.append("\n**Issues:**") + for issue in vr["issues"]: + parts.append(f"- {issue}") + + # Sample base HTML when all fields empty + if vr["populated_fields"] == 0 and vr["sample_base_html"]: + parts.append( + "\nHere is the innerHTML of the first base element — " + "use it to find correct child selectors:\n```html\n" + + vr["sample_base_html"] + + "\n```" + ) + + # Repeated schema warning + if is_repeated: + parts.append( + "\n**WARNING:** You returned the exact same schema as before. " + "You MUST change the selectors to fix the issues above." + ) + + parts.append( + "\nPlease fix the schema and return ONLY valid JSON, nothing else." + ) + return "\n".join(parts) + + @staticmethod + async def _infer_target_json(query: str, html_snippet: str, llm_config, url: str = None, usage: 'TokenUsage' = None) -> Optional[dict]: + """Infer a target JSON example from a query and HTML snippet via a quick LLM call. + + Args: + usage: Optional TokenUsage accumulator. If provided, token counts from + this LLM call are added to it in-place. + + Returns the parsed dict, or None if inference fails. + """ + from .utils import aperform_completion_with_backoff + + url_line = f"URL: {url}\n" if url else "" + prompt = ( + "You are given a data extraction request and a snippet of HTML from a webpage.\n" + "Your job is to produce a single example JSON object representing ONE item " + "that the user wants to extract.\n\n" + "Rules:\n" + "- Return ONLY a valid JSON object — one flat object, NOT wrapped in an array or outer key.\n" + "- The object represents a single repeated item (e.g., one product, one article, one row).\n" + "- Use clean snake_case field names matching the user's description.\n" + "- If the item has nested repeated sub-items, represent those as an array with one example inside.\n" + "- Fill values with realistic examples from the HTML so the meaning is clear.\n\n" + 'Example — if the request is "extract product name, price, and reviews":\n' + '{"name": "Widget Pro", "price": "$29.99", "reviews": [{"author": "Jane", "text": "Great product"}]}\n\n' + f"{url_line}" + f"Extraction request: {query}\n\n" + f"HTML snippet:\n```html\n{html_snippet[:2000]}\n```\n\n" + "Return ONLY the JSON object for ONE item:" + ) + + try: + response = await aperform_completion_with_backoff( + provider=llm_config.provider, + prompt_with_variables=prompt, + json_response=True, + api_token=llm_config.api_token, + base_url=llm_config.base_url, + ) + if usage is not None: + usage.completion_tokens += response.usage.completion_tokens + usage.prompt_tokens += response.usage.prompt_tokens + usage.total_tokens += response.usage.total_tokens + raw = response.choices[0].message.content + if not raw or not raw.strip(): + return None + return json.loads(_strip_markdown_fences(raw)) + except Exception: + return None + + @staticmethod + def _extract_expected_fields(target_json: dict) -> List[str]: + """Extract top-level field names from a target JSON example.""" + return list(target_json.keys()) + + _GENERATE_SCHEMA_UNWANTED_PROPS = { + 'provider': 'Instead, use llm_config=LLMConfig(provider="...")', + 'api_token': 'Instead, use llm_config=LlMConfig(api_token="...")', + } + + @staticmethod + def _build_schema_prompt(html: str, schema_type: str, query: str = None, target_json_example: str = None) -> str: + """ + Build the prompt for schema generation. Shared by sync and async methods. + + Returns: + str: Combined system and user prompt + """ + from .prompts import JSON_SCHEMA_BUILDER + + prompt_template = JSON_SCHEMA_BUILDER if schema_type == "CSS" else JSON_SCHEMA_BUILDER_XPATH + + system_content = f"""You specialize in generating special JSON schemas for web scraping. This schema uses CSS or XPATH selectors to present a repetitive pattern in crawled HTML, such as a product in a product list or a search result item in a list of search results. We use this JSON schema to pass to a language model along with the HTML content to extract structured data from the HTML. The language model uses the JSON schema to extract data from the HTML and retrieve values for fields in the JSON schema, following the schema. + +Generating this HTML manually is not feasible, so you need to generate the JSON schema using the HTML content. The HTML copied from the crawled website is provided below, which we believe contains the repetitive pattern. + +# Schema main keys: +- name: This is the name of the schema. +- baseSelector: This is the CSS or XPATH selector that identifies the base element that contains all the repetitive patterns. +- baseFields: This is a list of fields that you extract from the base element itself. +- fields: This is a list of fields that you extract from the children of the base element. {{name, selector, type}} based on the type, you may have extra keys such as "attribute" when the type is "attribute". + +# Extra Context: +In this context, the following items may or may not be present: +- Example of target JSON object: This is a sample of the final JSON object that we hope to extract from the HTML using the schema you are generating. +- Extra Instructions: This is optional instructions to consider when generating the schema provided by the user. +- Query or explanation of target/goal data item: This is a description of what data we are trying to extract from the HTML. This explanation means we're not sure about the rigid schema of the structures we want, so we leave it to you to use your expertise to create the best and most comprehensive structures aimed at maximizing data extraction from this page. You must ensure that you do not pick up nuances that may exist on a particular page. The focus should be on the data we are extracting, and it must be valid, safe, and robust based on the given HTML. + +# What if there is no example of target JSON object and also no extra instructions or even no explanation of target/goal data item? +In this scenario, use your best judgment to generate the schema. You need to examine the content of the page and understand the data it provides. If the page contains repetitive data, such as lists of items, products, jobs, places, books, or movies, focus on one single item that repeats. If the page is a detailed page about one product or item, create a schema to extract the entire structured data. At this stage, you must think and decide for yourself. Try to maximize the number of fields that you can extract from the HTML. + +# What are the instructions and details for this schema generation? +{prompt_template}""" + + user_content = f""" + HTML to analyze: + ```html + {html} + ``` + """ + + if query: + user_content += f"\n\n## Query or explanation of target/goal data item:\n{query}" + if target_json_example: + user_content += f"\n\n## Example of target JSON object:\n```json\n{target_json_example}\n```" + + if query and not target_json_example: + user_content += """IMPORTANT: To remind you, in this process, we are not providing a rigid example of the adjacent objects we seek. We rely on your understanding of the explanation provided in the above section. Make sure to grasp what we are looking for and, based on that, create the best schema..""" + elif not query and target_json_example: + user_content += """IMPORTANT: Please remember that in this process, we provided a proper example of a target JSON object. Make sure to adhere to the structure and create a schema that exactly fits this example. If you find that some elements on the page do not match completely, vote for the majority.""" + elif not query and not target_json_example: + user_content += """IMPORTANT: Since we neither have a query nor an example, it is crucial to rely solely on the HTML content provided. Leverage your expertise to determine the schema based on the repetitive patterns observed in the content.""" + + user_content += """IMPORTANT: + 0/ Ensure your schema remains reliable by avoiding selectors that appear to generate dynamically and are not dependable. You want a reliable schema, as it consistently returns the same data even after many page reloads. + 1/ DO NOT USE use base64 kind of classes, they are temporary and not reliable. + 2/ Every selector must refer to only one unique element. You should ensure your selector points to a single element and is unique to the place that contains the information. You have to use available techniques based on CSS or XPATH requested schema to make sure your selector is unique and also not fragile, meaning if we reload the page now or in the future, the selector should remain reliable. + 3/ Do not use Regex as much as possible. + + Analyze the HTML and generate a JSON schema that follows the specified format. Only output valid JSON schema, nothing else. + """ + + return "\n\n".join([system_content, user_content]) + + @staticmethod + def generate_schema( + html: str = None, + schema_type: str = "CSS", + query: str = None, + target_json_example: str = None, + llm_config: 'LLMConfig' = create_llm_config(), + provider: str = None, + api_token: str = None, + url: Union[str, List[str]] = None, + validate: bool = True, + max_refinements: int = 3, + usage: 'TokenUsage' = None, + **kwargs + ) -> dict: + """ + Generate extraction schema from HTML content or URL(s) (sync version). + + Args: + html (str, optional): The HTML content to analyze. If not provided, url must be set. + schema_type (str): "CSS" or "XPATH". Defaults to "CSS". + query (str, optional): Natural language description of what data to extract. + target_json_example (str, optional): Example of desired JSON output. + llm_config (LLMConfig): LLM configuration object. + provider (str): Legacy Parameter. LLM provider to use. + api_token (str): Legacy Parameter. API token for LLM provider. + url (str or List[str], optional): URL(s) to fetch HTML from. If provided, html parameter is ignored. + When multiple URLs are provided, HTMLs are fetched in parallel and concatenated. + validate (bool): If True, validate the schema against the HTML and + refine via LLM feedback loop. Defaults to False (zero overhead). + max_refinements (int): Max refinement rounds when validate=True. Defaults to 3. + usage (TokenUsage, optional): Token usage accumulator. If provided, + token counts from all LLM calls (including inference and + validation retries) are added to it in-place. + **kwargs: Additional args passed to LLM processor. + + Returns: + dict: Generated schema following the JsonElementExtractionStrategy format. + + Raises: + ValueError: If neither html nor url is provided. + """ + import asyncio + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + coro = JsonElementExtractionStrategy.agenerate_schema( + html=html, + schema_type=schema_type, + query=query, + target_json_example=target_json_example, + llm_config=llm_config, + provider=provider, + api_token=api_token, + url=url, + validate=validate, + max_refinements=max_refinements, + usage=usage, + **kwargs + ) + + if loop is None: + return asyncio.run(coro) + else: + import concurrent.futures + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(asyncio.run, coro) + return future.result() + + @staticmethod + async def agenerate_schema( + html: str = None, + schema_type: str = "CSS", + query: str = None, + target_json_example: str = None, + llm_config: 'LLMConfig' = None, + provider: str = None, + api_token: str = None, + url: Union[str, List[str]] = None, + validate: bool = True, + max_refinements: int = 3, + usage: 'TokenUsage' = None, + **kwargs + ) -> dict: + """ + Generate extraction schema from HTML content or URL(s) (async version). + + Use this method when calling from async contexts (e.g., FastAPI) to avoid + issues with certain LLM providers (e.g., Gemini/Vertex AI) that require + async execution. + + Args: + html (str, optional): The HTML content to analyze. If not provided, url must be set. + schema_type (str): "CSS" or "XPATH". Defaults to "CSS". + query (str, optional): Natural language description of what data to extract. + target_json_example (str, optional): Example of desired JSON output. + llm_config (LLMConfig): LLM configuration object. + provider (str): Legacy Parameter. LLM provider to use. + api_token (str): Legacy Parameter. API token for LLM provider. + url (str or List[str], optional): URL(s) to fetch HTML from. If provided, html parameter is ignored. + When multiple URLs are provided, HTMLs are fetched in parallel and concatenated. + validate (bool): If True, validate the schema against the HTML and + refine via LLM feedback loop. Defaults to False (zero overhead). + max_refinements (int): Max refinement rounds when validate=True. Defaults to 3. + usage (TokenUsage, optional): Token usage accumulator. If provided, + token counts from all LLM calls (including inference and + validation retries) are added to it in-place. + **kwargs: Additional args passed to LLM processor. + + Returns: + dict: Generated schema following the JsonElementExtractionStrategy format. + + Raises: + ValueError: If neither html nor url is provided. + """ + from .utils import aperform_completion_with_backoff, preprocess_html_for_schema + + # Validate inputs + if html is None and (url is None or (isinstance(url, list) and len(url) == 0)): + raise ValueError("Either 'html' or 'url' must be provided") + + # Check deprecated parameters + for name, message in JsonElementExtractionStrategy._GENERATE_SCHEMA_UNWANTED_PROPS.items(): + if locals()[name] is not None: + raise AttributeError(f"Setting '{name}' is deprecated. {message}") + + if llm_config is None: + llm_config = create_llm_config() + + # Save original HTML(s) before preprocessing (for validation against real HTML) + original_htmls = [] + + # Fetch HTML from URL(s) if provided + if url is not None: + from .async_webcrawler import AsyncWebCrawler + from .async_configs import BrowserConfig, CrawlerRunConfig, CacheMode + + browser_config = BrowserConfig( + headless=True, + text_mode=True, + light_mode=True, + ) + crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS) + + # Normalize to list + urls = [url] if isinstance(url, str) else url + + async with AsyncWebCrawler(config=browser_config) as crawler: + if len(urls) == 1: + result = await crawler.arun(url=urls[0], config=crawler_config) + if not result.success: + raise Exception(f"Failed to fetch URL '{urls[0]}': {result.error_message}") + if result.status_code >= 400: + raise Exception(f"HTTP {result.status_code} error for URL '{urls[0]}'") + html = result.html + original_htmls = [result.html] + else: + results = await crawler.arun_many(urls=urls, config=crawler_config) + html_parts = [] + for i, result in enumerate(results, 1): + if not result.success: + raise Exception(f"Failed to fetch URL '{result.url}': {result.error_message}") + if result.status_code >= 400: + raise Exception(f"HTTP {result.status_code} error for URL '{result.url}'") + original_htmls.append(result.html) + cleaned = preprocess_html_for_schema( + html_content=result.html, + text_threshold=2000, + attr_value_threshold=500, + max_size=500_000 + ) + header = HTML_EXAMPLE_DELIMITER.format(index=i) + html_parts.append(f"{header}\n{cleaned}") + html = "\n\n".join(html_parts) + else: + original_htmls = [html] + + # Preprocess HTML for schema generation (skip if already preprocessed from multiple URLs) + if url is None or isinstance(url, str): + html = preprocess_html_for_schema( + html_content=html, + text_threshold=2000, + attr_value_threshold=500, + max_size=500_000 + ) + + # --- Resolve expected fields for strict validation --- + expected_fields = None + if validate: + if target_json_example: + # User provided target JSON — extract field names from it + try: + if isinstance(target_json_example, str): + target_obj = json.loads(target_json_example) + else: + target_obj = target_json_example + expected_fields = JsonElementExtractionStrategy._extract_expected_fields(target_obj) + except (json.JSONDecodeError, TypeError): + pass + elif query: + # No target JSON but query describes fields — infer via quick LLM call + first_url = None + if url is not None: + first_url = url if isinstance(url, str) else url[0] + inferred = await JsonElementExtractionStrategy._infer_target_json( + query=query, html_snippet=html, llm_config=llm_config, url=first_url, usage=usage + ) + if inferred: + expected_fields = JsonElementExtractionStrategy._extract_expected_fields(inferred) + # Also inject as target_json_example for the schema prompt + if not target_json_example: + target_json_example = json.dumps(inferred, indent=2) + + prompt = JsonElementExtractionStrategy._build_schema_prompt(html, schema_type, query, target_json_example) + messages = [{"role": "user", "content": prompt}] + + prev_schema_json = None + last_schema = None + max_attempts = 1 + (max_refinements if validate else 0) + + for attempt in range(max_attempts): + try: + response = await aperform_completion_with_backoff( + provider=llm_config.provider, + prompt_with_variables=prompt, + json_response=True, + api_token=llm_config.api_token, + base_url=llm_config.base_url, + messages=messages, + extra_args=kwargs, + ) + if usage is not None: + usage.completion_tokens += response.usage.completion_tokens + usage.prompt_tokens += response.usage.prompt_tokens + usage.total_tokens += response.usage.total_tokens + raw = response.choices[0].message.content + if not raw or not raw.strip(): + raise ValueError("LLM returned an empty response") + + schema = json.loads(_strip_markdown_fences(raw)) + last_schema = schema + except json.JSONDecodeError as e: + # JSON parse failure — ask LLM to fix it + if not validate or attempt >= max_attempts - 1: + raise Exception(f"Failed to parse schema JSON: {str(e)}") + messages.append({"role": "assistant", "content": raw}) + messages.append({"role": "user", "content": ( + f"Your response was not valid JSON. Parse error: {e}\n" + "Please return ONLY valid JSON, nothing else." + )}) + continue + except Exception as e: + raise Exception(f"Failed to generate schema: {str(e)}") + + # If validation is off, return immediately (zero overhead path) + if not validate: + return schema + + # --- Validation feedback loop --- + # Validate against original HTML(s); success if works on at least one + best_result = None + for orig_html in original_htmls: + vr = JsonElementExtractionStrategy._validate_schema( + schema, orig_html, schema_type, + expected_fields=expected_fields, + ) + if best_result is None or vr["populated_fields"] > best_result["populated_fields"]: + best_result = vr + if vr["success"]: + break + + if best_result["success"]: + return schema + + # Last attempt — return best-effort + if attempt >= max_attempts - 1: + return schema + + # Detect repeated schema + current_json = json.dumps(schema, sort_keys=True) + is_repeated = current_json == prev_schema_json + prev_schema_json = current_json + + # Build feedback and extend conversation + feedback = JsonElementExtractionStrategy._build_feedback_message( + best_result, schema, attempt + 1, is_repeated + ) + messages.append({"role": "assistant", "content": raw}) + messages.append({"role": "user", "content": feedback}) + + # Should not reach here, but return last schema as safety net + if last_schema is not None: + return last_schema + raise Exception("Failed to generate schema: no attempts succeeded") + +class JsonCssExtractionStrategy(JsonElementExtractionStrategy): + """ + Concrete implementation of `JsonElementExtractionStrategy` using CSS selectors. + + How it works: + 1. Parses HTML content with BeautifulSoup. + 2. Selects elements using CSS selectors defined in the schema. + 3. Extracts field data and applies transformations as defined. + + Attributes: + schema (Dict[str, Any]): The schema defining the extraction rules. + verbose (bool): Enables verbose logging for debugging purposes. + + Methods: + _parse_html(html_content): Parses HTML content into a BeautifulSoup object. + _get_base_elements(parsed_html, selector): Selects base elements using a CSS selector. + _get_elements(element, selector): Selects child elements using a CSS selector. + _get_element_text(element): Extracts text content from a BeautifulSoup element. + _get_element_html(element): Extracts the raw HTML content of a BeautifulSoup element. + _get_element_attribute(element, attribute): Retrieves an attribute value from a BeautifulSoup element. + """ + + def __init__(self, schema: Dict[str, Any], **kwargs): + kwargs["input_format"] = "html" # Force HTML input + super().__init__(schema, **kwargs) + + def _parse_html(self, html_content: str): + # return BeautifulSoup(html_content, "html.parser") + return BeautifulSoup(html_content, "lxml") + + def _get_base_elements(self, parsed_html, selector: str): + return parsed_html.select(selector) + + def _get_elements(self, element, selector: str): + # Return all matching elements using select() instead of select_one() + # This ensures that we get all elements that match the selector, not just the first one + return element.select(selector) + + def _get_element_text(self, element) -> str: + return element.get_text(strip=True) + + def _get_element_html(self, element) -> str: + return str(element) + + def _get_element_attribute(self, element, attribute: str): + return element.get(attribute) + + def _resolve_source(self, element, source: str): + source = source.strip() + if not source.startswith("+"): + return None + sel = source[1:].strip() # e.g. "tr", "tr.subtext", ".classname" + parts = sel.split(".") + tag = parts[0].strip() or None + classes = [p.strip() for p in parts[1:] if p.strip()] + kwargs = {} + if classes: + kwargs["class_"] = lambda c, _cls=classes: c and all( + cl in c for cl in _cls + ) + return element.find_next_sibling(tag, **kwargs) + +class JsonLxmlExtractionStrategy(JsonElementExtractionStrategy): + def __init__(self, schema: Dict[str, Any], **kwargs): + kwargs["input_format"] = "html" + super().__init__(schema, **kwargs) + self._selector_cache = {} + self._xpath_cache = {} + self._result_cache = {} + + # Control selector optimization strategy + self.use_caching = kwargs.get("use_caching", True) + self.optimize_common_patterns = kwargs.get("optimize_common_patterns", True) + + # Load lxml dependencies once + from lxml import etree, html + from lxml.cssselect import CSSSelector + self.etree = etree + self.html_parser = html + self.CSSSelector = CSSSelector + + def _parse_html(self, html_content: str): + """Parse HTML content with error recovery""" + try: + parser = self.etree.HTMLParser(recover=True, remove_blank_text=True) + return self.etree.fromstring(html_content, parser) + except Exception as e: + if self.verbose: + print(f"Error parsing HTML, falling back to alternative method: {e}") + try: + return self.html_parser.fromstring(html_content) + except Exception as e2: + if self.verbose: + print(f"Critical error parsing HTML: {e2}") + # Create minimal document as fallback + return self.etree.Element("html") + + def _optimize_selector(self, selector_str): + """Optimize common selector patterns for better performance""" + if not self.optimize_common_patterns: + return selector_str + + # Handle td:nth-child(N) pattern which is very common in table scraping + import re + if re.search(r'td:nth-child\(\d+\)', selector_str): + return selector_str # Already handled specially in _apply_selector + + # Split complex selectors into parts for optimization + parts = selector_str.split() + if len(parts) <= 1: + return selector_str + + # For very long selectors, consider using just the last specific part + if len(parts) > 3 and any(p.startswith('.') or p.startswith('#') for p in parts): + specific_parts = [p for p in parts if p.startswith('.') or p.startswith('#')] + if specific_parts: + return specific_parts[-1] # Use most specific class/id selector + + return selector_str + + def _create_selector_function(self, selector_str): + """Create a selector function that handles all edge cases""" + original_selector = selector_str + + # Try to optimize the selector if appropriate + if self.optimize_common_patterns: + selector_str = self._optimize_selector(selector_str) + + try: + # Attempt to compile the CSS selector + compiled = self.CSSSelector(selector_str) + xpath = compiled.path + + # Store XPath for later use + self._xpath_cache[selector_str] = xpath + + # Create the wrapper function that implements the selection strategy + def selector_func(element, context_sensitive=True): + cache_key = None + + # Use result caching if enabled + if self.use_caching: + # Create a cache key based on element and selector + element_id = element.get('id', '') or str(hash(element)) + cache_key = f"{element_id}::{selector_str}" + + if cache_key in self._result_cache: + return self._result_cache[cache_key] + + results = [] + try: + # Strategy 1: Direct CSS selector application (fastest) + results = compiled(element) + + # If that fails and we need context sensitivity + if not results and context_sensitive: + # Strategy 2: Try XPath with context adjustment + context_xpath = self._make_context_sensitive_xpath(xpath, element) + if context_xpath: + results = element.xpath(context_xpath) + + # Strategy 3: Handle special case - nth-child + if not results and 'nth-child' in original_selector: + results = self._handle_nth_child_selector(element, original_selector) + + # Strategy 4: Direct descendant search for class/ID selectors + if not results: + results = self._fallback_class_id_search(element, original_selector) + + # Strategy 5: Last resort - tag name search for the final part + if not results: + parts = original_selector.split() + if parts: + last_part = parts[-1] + # Extract tag name from the selector + tag_match = re.match(r'^(\w+)', last_part) + if tag_match: + tag_name = tag_match.group(1) + results = element.xpath(f".//{tag_name}") + + # Cache results if caching is enabled + if self.use_caching and cache_key: + self._result_cache[cache_key] = results + + except Exception as e: + if self.verbose: + print(f"Error applying selector '{selector_str}': {e}") + + return results + + return selector_func + + except Exception as e: + if self.verbose: + print(f"Error compiling selector '{selector_str}': {e}") + + # Fallback function for invalid selectors + return lambda element, context_sensitive=True: [] + + def _make_context_sensitive_xpath(self, xpath, element): + """Convert absolute XPath to context-sensitive XPath""" + try: + # If starts with descendant-or-self, it's already context-sensitive + if xpath.startswith('descendant-or-self::'): + return xpath + + # Remove leading slash if present + if xpath.startswith('/'): + context_xpath = f".{xpath}" + else: + context_xpath = f".//{xpath}" + + # Validate the XPath by trying it + try: + element.xpath(context_xpath) + return context_xpath + except: + # If that fails, try a simpler descendant search + return f".//{xpath.split('/')[-1]}" + except: + return None + + def _handle_nth_child_selector(self, element, selector_str): + """Special handling for nth-child selectors in tables""" + import re + results = [] + + try: + # Extract the column number from td:nth-child(N) + match = re.search(r'td:nth-child\((\d+)\)', selector_str) + if match: + col_num = match.group(1) + + # Check if there's content after the nth-child part + remaining_selector = selector_str.split(f"td:nth-child({col_num})", 1)[-1].strip() + + if remaining_selector: + # If there's a specific element we're looking for after the column + # Extract any tag names from the remaining selector + tag_match = re.search(r'(\w+)', remaining_selector) + tag_name = tag_match.group(1) if tag_match else '*' + results = element.xpath(f".//td[{col_num}]//{tag_name}") + else: + # Just get the column cell + results = element.xpath(f".//td[{col_num}]") + except Exception as e: + if self.verbose: + print(f"Error handling nth-child selector: {e}") + + return results + + def _fallback_class_id_search(self, element, selector_str): + """Fallback to search by class or ID""" + results = [] + + try: + # Extract class selectors (.classname) + import re + class_matches = re.findall(r'\.([a-zA-Z0-9_-]+)', selector_str) + + # Extract ID selectors (#idname) + id_matches = re.findall(r'#([a-zA-Z0-9_-]+)', selector_str) + + # Try each class + for class_name in class_matches: + class_results = element.xpath(f".//*[contains(@class, '{class_name}')]") + results.extend(class_results) + + # Try each ID (usually more specific) + for id_name in id_matches: + id_results = element.xpath(f".//*[@id='{id_name}']") + results.extend(id_results) + except Exception as e: + if self.verbose: + print(f"Error in fallback class/id search: {e}") + + return results + + def _get_selector(self, selector_str): + """Get or create a selector function with caching""" + if selector_str not in self._selector_cache: + self._selector_cache[selector_str] = self._create_selector_function(selector_str) + return self._selector_cache[selector_str] + + def _get_base_elements(self, parsed_html, selector: str): + """Get all base elements using the selector""" + selector_func = self._get_selector(selector) + # For base elements, we don't need context sensitivity + return selector_func(parsed_html, context_sensitive=False) + + def _get_elements(self, element, selector: str): + """Get child elements using the selector with context sensitivity""" + selector_func = self._get_selector(selector) + return selector_func(element, context_sensitive=True) + + def _get_element_text(self, element) -> str: + """Extract normalized text from element""" + try: + # Get all text nodes and normalize + text = " ".join(t.strip() for t in element.xpath(".//text()") if t.strip()) + return text + except Exception as e: + if self.verbose: + print(f"Error extracting text: {e}") + # Fallback + try: + return element.text_content().strip() + except: + return "" + + def _get_element_html(self, element) -> str: + """Get HTML string representation of element""" + try: + return self.etree.tostring(element, encoding='unicode', method='html') + except Exception as e: + if self.verbose: + print(f"Error serializing HTML: {e}") + return "" + + def _get_element_attribute(self, element, attribute: str): + """Get attribute value safely""" + try: + return element.get(attribute) + except Exception as e: + if self.verbose: + print(f"Error getting attribute '{attribute}': {e}") + return None + + def _resolve_source(self, element, source: str): + source = source.strip() + if not source.startswith("+"): + return None + sel = source[1:].strip() + parts = sel.split(".") + tag = parts[0].strip() or "*" + classes = [p.strip() for p in parts[1:] if p.strip()] + xpath = f"./following-sibling::{tag}" + for cls in classes: + xpath += f"[contains(concat(' ',normalize-space(@class),' '),' {cls} ')]" + xpath += "[1]" + results = element.xpath(xpath) + return results[0] if results else None + + def _clear_caches(self): + """Clear caches to free memory""" + if self.use_caching: + self._result_cache.clear() + +class JsonLxmlExtractionStrategy_naive(JsonElementExtractionStrategy): + def __init__(self, schema: Dict[str, Any], **kwargs): + kwargs["input_format"] = "html" # Force HTML input + super().__init__(schema, **kwargs) + self._selector_cache = {} + + def _parse_html(self, html_content: str): + from lxml import etree + parser = etree.HTMLParser(recover=True) + return etree.fromstring(html_content, parser) + + def _get_selector(self, selector_str): + """Get a selector function that works within the context of an element""" + if selector_str not in self._selector_cache: + from lxml.cssselect import CSSSelector + try: + # Store both the compiled selector and its xpath translation + compiled = CSSSelector(selector_str) + + # Create a function that will apply this selector appropriately + def select_func(element): + try: + # First attempt: direct CSS selector application + results = compiled(element) + if results: + return results + + # Second attempt: contextual XPath selection + # Convert the root-based XPath to a context-based XPath + xpath = compiled.path + + # If the XPath already starts with descendant-or-self, handle it specially + if xpath.startswith('descendant-or-self::'): + context_xpath = xpath + else: + # For normal XPath expressions, make them relative to current context + context_xpath = f"./{xpath.lstrip('/')}" + + results = element.xpath(context_xpath) + if results: + return results + + # Final fallback: simple descendant search for common patterns + if 'nth-child' in selector_str: + # Handle td:nth-child(N) pattern + import re + match = re.search(r'td:nth-child\((\d+)\)', selector_str) + if match: + col_num = match.group(1) + sub_selector = selector_str.split(')', 1)[-1].strip() + if sub_selector: + return element.xpath(f".//td[{col_num}]//{sub_selector}") + else: + return element.xpath(f".//td[{col_num}]") + + # Last resort: try each part of the selector separately + parts = selector_str.split() + if len(parts) > 1 and parts[-1]: + return element.xpath(f".//{parts[-1]}") + + return [] + except Exception as e: + if self.verbose: + print(f"Error applying selector '{selector_str}': {e}") + return [] + + self._selector_cache[selector_str] = select_func + except Exception as e: + if self.verbose: + print(f"Error compiling selector '{selector_str}': {e}") + + # Fallback function for invalid selectors + def fallback_func(element): + return [] + + self._selector_cache[selector_str] = fallback_func + + return self._selector_cache[selector_str] + + def _get_base_elements(self, parsed_html, selector: str): + selector_func = self._get_selector(selector) + return selector_func(parsed_html) + + def _get_elements(self, element, selector: str): + selector_func = self._get_selector(selector) + return selector_func(element) + + def _get_element_text(self, element) -> str: + return "".join(element.xpath(".//text()")).strip() + + def _get_element_html(self, element) -> str: + from lxml import etree + return etree.tostring(element, encoding='unicode') + + def _get_element_attribute(self, element, attribute: str): + return element.get(attribute) + + def _resolve_source(self, element, source: str): + source = source.strip() + if not source.startswith("+"): + return None + sel = source[1:].strip() + parts = sel.split(".") + tag = parts[0].strip() or "*" + classes = [p.strip() for p in parts[1:] if p.strip()] + xpath = f"./following-sibling::{tag}" + for cls in classes: + xpath += f"[contains(concat(' ',normalize-space(@class),' '),' {cls} ')]" + xpath += "[1]" + results = element.xpath(xpath) + return results[0] if results else None + +class JsonXPathExtractionStrategy(JsonElementExtractionStrategy): + """ + Concrete implementation of `JsonElementExtractionStrategy` using XPath selectors. + + How it works: + 1. Parses HTML content into an lxml tree. + 2. Selects elements using XPath expressions. + 3. Converts CSS selectors to XPath when needed. + + Attributes: + schema (Dict[str, Any]): The schema defining the extraction rules. + verbose (bool): Enables verbose logging for debugging purposes. + + Methods: + _parse_html(html_content): Parses HTML content into an lxml tree. + _get_base_elements(parsed_html, selector): Selects base elements using an XPath selector. + _css_to_xpath(css_selector): Converts a CSS selector to an XPath expression. + _get_elements(element, selector): Selects child elements using an XPath selector. + _get_element_text(element): Extracts text content from an lxml element. + _get_element_html(element): Extracts the raw HTML content of an lxml element. + _get_element_attribute(element, attribute): Retrieves an attribute value from an lxml element. + """ + + def __init__(self, schema: Dict[str, Any], **kwargs): + kwargs["input_format"] = "html" # Force HTML input + super().__init__(schema, **kwargs) + + def _parse_html(self, html_content: str): + return html.fromstring(html_content) + + def _get_base_elements(self, parsed_html, selector: str): + return parsed_html.xpath(selector) + + def _css_to_xpath(self, css_selector: str) -> str: + """Convert CSS selector to XPath if needed""" + if "/" in css_selector: # Already an XPath + return css_selector + return self._basic_css_to_xpath(css_selector) + + def _basic_css_to_xpath(self, css_selector: str) -> str: + """Basic CSS to XPath conversion for common cases""" + if " > " in css_selector: + parts = css_selector.split(" > ") + return "//" + "/".join(parts) + if " " in css_selector: + parts = css_selector.split(" ") + return "//" + "//".join(parts) + return "//" + css_selector + + def _get_elements(self, element, selector: str): + xpath = self._css_to_xpath(selector) + if not xpath.startswith("."): + xpath = "." + xpath + return element.xpath(xpath) + + def _get_element_text(self, element) -> str: + return "".join(element.xpath(".//text()")).strip() + + def _get_element_html(self, element) -> str: + return etree.tostring(element, encoding="unicode") + + def _get_element_attribute(self, element, attribute: str): + return element.get(attribute) + + def _resolve_source(self, element, source: str): + source = source.strip() + if not source.startswith("+"): + return None + sel = source[1:].strip() + parts = sel.split(".") + tag = parts[0].strip() or "*" + classes = [p.strip() for p in parts[1:] if p.strip()] + xpath = f"./following-sibling::{tag}" + for cls in classes: + xpath += f"[contains(concat(' ',normalize-space(@class),' '),' {cls} ')]" + xpath += "[1]" + results = element.xpath(xpath) + return results[0] if results else None + +""" +RegexExtractionStrategy +Fast, zero-LLM extraction of common entities via regular expressions. +""" + +_CTRL = {c: rf"\x{ord(c):02x}" for c in map(chr, range(32)) if c not in "\t\n\r"} + +_WB_FIX = re.compile(r"\x08") # stray back-space → word-boundary +_NEEDS_ESCAPE = re.compile(r"(? Dict[str, str]: + """Fix common JSON-escape goofs coming from LLMs or manual edits.""" + safe = {} + for label, pat in schema.items(): + # 1️⃣ replace accidental control chars (inc. the infamous back-space) + pat = _WB_FIX.sub(r"\\b", pat).translate(_CTRL) + + # 2️⃣ double any single backslash that JSON kept single + pat = _NEEDS_ESCAPE.sub(r"\\\\", pat) + + # 3️⃣ quick sanity compile + try: + re.compile(pat) + except re.error as e: + raise ValueError(f"Regex for '{label}' won’t compile after fix: {e}") from None + + safe[label] = pat + return safe + + +class RegexExtractionStrategy(ExtractionStrategy): + """ + A lean strategy that finds e-mails, phones, URLs, dates, money, etc., + using nothing but pre-compiled regular expressions. + + Extraction returns:: + + { + "url": "", + "label": "", + "value": "", + "span": [start, end] + } + + Only `generate_schema()` touches an LLM, extraction itself is pure Python. + """ + + # -------------------------------------------------------------- # + # Built-in patterns exposed as IntFlag so callers can bit-OR them + # -------------------------------------------------------------- # + class _B(IntFlag): + EMAIL = auto() + PHONE_INTL = auto() + PHONE_US = auto() + URL = auto() + IPV4 = auto() + IPV6 = auto() + UUID = auto() + CURRENCY = auto() + PERCENTAGE = auto() + NUMBER = auto() + DATE_ISO = auto() + DATE_US = auto() + TIME_24H = auto() + POSTAL_US = auto() + POSTAL_UK = auto() + HTML_COLOR_HEX = auto() + TWITTER_HANDLE = auto() + HASHTAG = auto() + MAC_ADDR = auto() + IBAN = auto() + CREDIT_CARD = auto() + NOTHING = auto() + ALL = ( + EMAIL | PHONE_INTL | PHONE_US | URL | IPV4 | IPV6 | UUID + | CURRENCY | PERCENTAGE | NUMBER | DATE_ISO | DATE_US | TIME_24H + | POSTAL_US | POSTAL_UK | HTML_COLOR_HEX | TWITTER_HANDLE + | HASHTAG | MAC_ADDR | IBAN | CREDIT_CARD + ) + + # user-friendly aliases (RegexExtractionStrategy.Email, .IPv4, …) + Email = _B.EMAIL + PhoneIntl = _B.PHONE_INTL + PhoneUS = _B.PHONE_US + Url = _B.URL + IPv4 = _B.IPV4 + IPv6 = _B.IPV6 + Uuid = _B.UUID + Currency = _B.CURRENCY + Percentage = _B.PERCENTAGE + Number = _B.NUMBER + DateIso = _B.DATE_ISO + DateUS = _B.DATE_US + Time24h = _B.TIME_24H + PostalUS = _B.POSTAL_US + PostalUK = _B.POSTAL_UK + HexColor = _B.HTML_COLOR_HEX + TwitterHandle = _B.TWITTER_HANDLE + Hashtag = _B.HASHTAG + MacAddr = _B.MAC_ADDR + Iban = _B.IBAN + CreditCard = _B.CREDIT_CARD + All = _B.ALL + Nothing = _B(0) # no patterns + + # ------------------------------------------------------------------ # + # Built-in pattern catalog + # ------------------------------------------------------------------ # + DEFAULT_PATTERNS: Dict[str, str] = { + # Communication + "email": r"[\w.+-]+@[\w-]+\.[\w.-]+", + "phone_intl": r"\+?\d[\d .()-]{7,}\d", + "phone_us": r"\(?\d{3}\)?[ -. ]?\d{3}[ -. ]?\d{4}", + # Web + "url": r"https?://[^\s\"'<>]+", + "ipv4": r"(?:\d{1,3}\.){3}\d{1,3}", + "ipv6": r"[A-F0-9]{1,4}(?::[A-F0-9]{1,4}){7}", + # IDs + "uuid": r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}", + # Money / numbers + "currency": r"(?:USD|EUR|RM|\$|€|£)\s?\d+(?:[.,]\d{2})?", + "percentage": r"\d+(?:\.\d+)?%", + "number": r"\b\d{1,3}(?:[,.\s]\d{3})*(?:\.\d+)?\b", + # Dates / Times + "date_iso": r"\d{4}-\d{2}-\d{2}", + "date_us": r"\d{1,2}/\d{1,2}/\d{2,4}", + "time_24h": r"\b(?:[01]?\d|2[0-3]):[0-5]\d(?:[:.][0-5]\d)?\b", + # Misc + "postal_us": r"\b\d{5}(?:-\d{4})?\b", + "postal_uk": r"\b[A-Z]{1,2}\d[A-Z\d]? ?\d[A-Z]{2}\b", + "html_color_hex": r"#[0-9A-Fa-f]{6}\b", + "twitter_handle": r"@[\w]{1,15}", + "hashtag": r"#[\w-]+", + "mac_addr": r"(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}", + "iban": r"[A-Z]{2}\d{2}[A-Z0-9]{11,30}", + "credit_card": r"\b(?:4\d{12}(?:\d{3})?|5[1-5]\d{14}|3[47]\d{13}|6(?:011|5\d{2})\d{12})\b", + } + + _FLAGS = re.IGNORECASE | re.MULTILINE + _UNWANTED_PROPS = { + "provider": "Use llm_config instead", + "api_token": "Use llm_config instead", + } + + # ------------------------------------------------------------------ # + # Construction + # ------------------------------------------------------------------ # + def __init__( + self, + pattern: "_B" = _B.NOTHING, + *, + custom: Optional[Union[Dict[str, str], List[Tuple[str, str]]]] = None, + input_format: str = "fit_html", + **kwargs, + ) -> None: + """ + Args: + patterns: Custom patterns overriding or extending defaults. + Dict[label, regex] or list[tuple(label, regex)]. + input_format: "html", "markdown" or "text". + **kwargs: Forwarded to ExtractionStrategy. + """ + super().__init__(input_format=input_format, **kwargs) + + # 1️⃣ take only the requested built-ins + merged: Dict[str, str] = { + key: rx + for key, rx in self.DEFAULT_PATTERNS.items() + if getattr(self._B, key.upper()).value & pattern + } + + # 2️⃣ apply user overrides / additions + if custom: + if isinstance(custom, dict): + merged.update(custom) + else: # iterable of (label, regex) + merged.update({lbl: rx for lbl, rx in custom}) + + self._compiled: Dict[str, Pattern] = { + lbl: re.compile(rx, self._FLAGS) for lbl, rx in merged.items() + } + + # ------------------------------------------------------------------ # + # Extraction + # ------------------------------------------------------------------ # + def extract(self, url: str, content: str, *q, **kw) -> List[Dict[str, Any]]: + # text = self._plain_text(html) + out: List[Dict[str, Any]] = [] + + for label, cre in self._compiled.items(): + for m in cre.finditer(content): + out.append( + { + "url": url, + "label": label, + "value": m.group(0), + "span": [m.start(), m.end()], + } + ) + return out + + # ------------------------------------------------------------------ # + # Helpers + # ------------------------------------------------------------------ # + def _plain_text(self, content: str) -> str: + if self.input_format == "text": + return content + return BeautifulSoup(content, "lxml").get_text(" ", strip=True) + + # ------------------------------------------------------------------ # + # LLM-assisted pattern generator + # ------------------------------------------------------------------ # + # ------------------------------------------------------------------ # + # LLM-assisted one-off pattern builder + # ------------------------------------------------------------------ # + @staticmethod + def generate_pattern( + label: str, + html: str, + *, + query: Optional[str] = None, + examples: Optional[List[str]] = None, + llm_config: Optional[LLMConfig] = None, + **kwargs, + ) -> Dict[str, str]: + """ + Ask an LLM for a single page-specific regex and return + {label: pattern} ── ready for RegexExtractionStrategy(custom=…) + """ + + # ── guard deprecated kwargs + for k in RegexExtractionStrategy._UNWANTED_PROPS: + if k in kwargs: + raise AttributeError( + f"{k} is deprecated, {RegexExtractionStrategy._UNWANTED_PROPS[k]}" + ) + + # ── default LLM config + if llm_config is None: + llm_config = create_llm_config() + + # ── system prompt – hardened + system_msg = ( + "You are an expert Python-regex engineer.\n" + f"Return **one** JSON object whose single key is exactly \"{label}\", " + "and whose value is a raw-string regex pattern that works with " + "the standard `re` module in Python.\n\n" + "Strict rules (obey every bullet):\n" + "• If a *user query* is supplied, treat it as the precise semantic target and optimise the " + " pattern to capture ONLY text that answers that query. If the query conflicts with the " + " sample HTML, the HTML wins.\n" + "• Tailor the pattern to the *sample HTML* – reproduce its exact punctuation, spacing, " + " symbols, capitalisation, etc. Do **NOT** invent a generic form.\n" + "• Keep it minimal and fast: avoid unnecessary capturing, prefer non-capturing `(?: … )`, " + " and guard against catastrophic backtracking.\n" + "• Anchor with `^`, `$`, or `\\b` only when it genuinely improves precision.\n" + "• Use inline flags like `(?i)` when needed; no verbose flag comments.\n" + "• Output must be valid JSON – no markdown, code fences, comments, or extra keys.\n" + "• The regex value must be a Python string literal: **double every backslash** " + "(e.g. `\\\\b`, `\\\\d`, `\\\\\\\\`).\n\n" + "Example valid output:\n" + f"{{\"{label}\": \"(?:RM|rm)\\\\s?\\\\d{{1,3}}(?:,\\\\d{{3}})*(?:\\\\.\\\\d{{2}})?\"}}" + ) + + # ── user message: cropped HTML + optional hints + user_parts = ["```html", html[:5000], "```"] # protect token budget + if query: + user_parts.append(f"\n\n## Query\n{query.strip()}") + if examples: + user_parts.append("## Examples\n" + "\n".join(examples[:20])) + user_msg = "\n\n".join(user_parts) + + # ── LLM call (with retry/backoff) + resp = perform_completion_with_backoff( + provider=llm_config.provider, + prompt_with_variables="\n\n".join([system_msg, user_msg]), + json_response=True, + api_token=llm_config.api_token, + base_url=llm_config.base_url, + extra_args=kwargs, + ) + + # ── clean & load JSON (fix common escape mistakes *before* json.loads) + raw = resp.choices[0].message.content + raw = raw.replace("\x08", "\\b") # stray back-space → \b + raw = re.sub(r'(? None: + """ + Input parameters: + out: possible custom replacement for self.outtextf (which + appends lines of text). + baseurl: base URL of the document we process + """ + super().__init__(convert_charrefs=False) + + # Config options + self.split_next_td = False + self.td_count = 0 + self.table_start = False + self.unicode_snob = config.UNICODE_SNOB # covered in cli + + self.escape_snob = config.ESCAPE_SNOB # covered in cli + self.escape_backslash = config.ESCAPE_BACKSLASH # covered in cli + self.escape_dot = config.ESCAPE_DOT # covered in cli + self.escape_plus = config.ESCAPE_PLUS # covered in cli + self.escape_dash = config.ESCAPE_DASH # covered in cli + + self.links_each_paragraph = config.LINKS_EACH_PARAGRAPH + self.body_width = bodywidth # covered in cli + self.skip_internal_links = config.SKIP_INTERNAL_LINKS # covered in cli + self.inline_links = config.INLINE_LINKS # covered in cli + self.protect_links = config.PROTECT_LINKS # covered in cli + self.google_list_indent = config.GOOGLE_LIST_INDENT # covered in cli + self.ignore_links = config.IGNORE_ANCHORS # covered in cli + self.ignore_mailto_links = config.IGNORE_MAILTO_LINKS # covered in cli + self.ignore_images = config.IGNORE_IMAGES # covered in cli + self.images_as_html = config.IMAGES_AS_HTML # covered in cli + self.images_to_alt = config.IMAGES_TO_ALT # covered in cli + self.images_with_size = config.IMAGES_WITH_SIZE # covered in cli + self.ignore_emphasis = config.IGNORE_EMPHASIS # covered in cli + self.bypass_tables = config.BYPASS_TABLES # covered in cli + self.ignore_tables = config.IGNORE_TABLES # covered in cli + self.google_doc = False # covered in cli + self.ul_item_mark = "*" # covered in cli + self.emphasis_mark = "_" # covered in cli + self.strong_mark = "**" + self.single_line_break = config.SINGLE_LINE_BREAK # covered in cli + self.use_automatic_links = config.USE_AUTOMATIC_LINKS # covered in cli + self.hide_strikethrough = False # covered in cli + self.mark_code = config.MARK_CODE + self.wrap_list_items = config.WRAP_LIST_ITEMS # covered in cli + self.wrap_links = config.WRAP_LINKS # covered in cli + self.wrap_tables = config.WRAP_TABLES + self.pad_tables = config.PAD_TABLES # covered in cli + self.default_image_alt = config.DEFAULT_IMAGE_ALT # covered in cli + self.tag_callback = None + self.open_quote = config.OPEN_QUOTE # covered in cli + self.close_quote = config.CLOSE_QUOTE # covered in cli + self.include_sup_sub = config.INCLUDE_SUP_SUB # covered in cli + + if out is None: + self.out = self.outtextf + else: + self.out = out + + # empty list to store output characters before they are "joined" + self.outtextlist: List[str] = [] + + self.quiet = 0 + self.p_p = 0 # number of newline character to print before next output + self.outcount = 0 + self.start = True + self.space = False + self.a: List[AnchorElement] = [] + self.astack: List[Optional[Dict[str, Optional[str]]]] = [] + self.maybe_automatic_link: Optional[str] = None + self.empty_link = False + self.absolute_url_matcher = re.compile(r"^[a-zA-Z+]+://") + self.acount = 0 + self.list: List[ListElement] = [] + self.blockquote = 0 + self.pre = False + self.startpre = False + self.code = False + self.quote = False + self.br_toggle = "" + self.lastWasNL = False + self.lastWasList = False + self.style = 0 + self.style_def: Dict[str, Dict[str, str]] = {} + self.tag_stack: List[Tuple[str, Dict[str, Optional[str]], Dict[str, str]]] = [] + self.emphasis = 0 + self.drop_white_space = 0 + self.inheader = False + # Current abbreviation definition + self.abbr_title: Optional[str] = None + # Last inner HTML (for abbr being defined) + self.abbr_data: Optional[str] = None + # Stack of abbreviations to write later + self.abbr_list: Dict[str, str] = {} + self.baseurl = baseurl + self.stressed = False + self.preceding_stressed = False + self.preceding_data = "" + self.current_tag = "" + + config.UNIFIABLE["nbsp"] = " _place_holder;" + + def update_params(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + def feed(self, data: str) -> None: + data = data.replace("", "") + super().feed(data) + + def handle(self, data: str) -> str: + self.start = True + self.feed(data) + self.feed("") + markdown = self.optwrap(self.finish()) + if self.pad_tables: + return pad_tables_in_text(markdown) + else: + return markdown + + def outtextf(self, s: str) -> None: + self.outtextlist.append(s) + if s: + self.lastWasNL = s[-1] == "\n" + + def finish(self) -> str: + self.close() + + self.pbr() + self.o("", force="end") + + outtext = "".join(self.outtextlist) + + if self.unicode_snob: + nbsp = html.entities.html5["nbsp;"] + else: + nbsp = " " + outtext = outtext.replace(" _place_holder;", nbsp) + + # Clear self.outtextlist to avoid memory leak of its content to + # the next handling. + self.outtextlist = [] + + return outtext + + def handle_charref(self, c: str) -> None: + self.handle_data(self.charref(c), True) + + def handle_entityref(self, c: str) -> None: + ref = self.entityref(c) + + # ref may be an empty string (e.g. for ‎/‏ markers that should + # not contribute to the final output). + # self.handle_data cannot handle a zero-length string right after a + # stressed tag or mid-text within a stressed tag (text get split and + # self.stressed/self.preceding_stressed gets switched after the first + # part of that text). + if ref: + self.handle_data(ref, True) + + def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None: + self.handle_tag(tag, dict(attrs), start=True) + + def handle_endtag(self, tag: str) -> None: + self.handle_tag(tag, {}, start=False) + + def previousIndex(self, attrs: Dict[str, Optional[str]]) -> Optional[int]: + """ + :type attrs: dict + + :returns: The index of certain set of attributes (of a link) in the + self.a list. If the set of attributes is not found, returns None + :rtype: int + """ + if "href" not in attrs: + return None + + match = False + for i, a in enumerate(self.a): + if "href" in a.attrs and a.attrs["href"] == attrs["href"]: + if "title" in a.attrs or "title" in attrs: + if ( + "title" in a.attrs + and "title" in attrs + and a.attrs["title"] == attrs["title"] + ): + match = True + else: + match = True + + if match: + return i + return None + + def handle_emphasis( + self, start: bool, tag_style: Dict[str, str], parent_style: Dict[str, str] + ) -> None: + """ + Handles various text emphases + """ + tag_emphasis = google_text_emphasis(tag_style) + parent_emphasis = google_text_emphasis(parent_style) + + # handle Google's text emphasis + strikethrough = "line-through" in tag_emphasis and self.hide_strikethrough + + # google and others may mark a font's weight as `bold` or `700` + bold = False + for bold_marker in config.BOLD_TEXT_STYLE_VALUES: + bold = bold_marker in tag_emphasis and bold_marker not in parent_emphasis + if bold: + break + + italic = "italic" in tag_emphasis and "italic" not in parent_emphasis + fixed = ( + google_fixed_width_font(tag_style) + and not google_fixed_width_font(parent_style) + and not self.pre + ) + + if start: + # crossed-out text must be handled before other attributes + # in order not to output qualifiers unnecessarily + if bold or italic or fixed: + self.emphasis += 1 + if strikethrough: + self.quiet += 1 + if italic: + self.o(self.emphasis_mark) + self.drop_white_space += 1 + if bold: + self.o(self.strong_mark) + self.drop_white_space += 1 + if fixed: + self.o("`") + self.drop_white_space += 1 + self.code = True + else: + if bold or italic or fixed: + # there must not be whitespace before closing emphasis mark + self.emphasis -= 1 + self.space = False + if fixed: + if self.drop_white_space: + # empty emphasis, drop it + self.drop_white_space -= 1 + else: + self.o("`") + self.code = False + if bold: + if self.drop_white_space: + # empty emphasis, drop it + self.drop_white_space -= 1 + else: + self.o(self.strong_mark) + if italic: + if self.drop_white_space: + # empty emphasis, drop it + self.drop_white_space -= 1 + else: + self.o(self.emphasis_mark) + # space is only allowed after *all* emphasis marks + if (bold or italic) and not self.emphasis: + self.o(" ") + if strikethrough: + self.quiet -= 1 + + def handle_tag( + self, tag: str, attrs: Dict[str, Optional[str]], start: bool + ) -> None: + self.current_tag = tag + + if self.tag_callback is not None: + if self.tag_callback(self, tag, attrs, start) is True: + return + + # Handle tag to update base URL for relative links + if tag == "base" and start: + href = attrs.get("href") + if href: + self.baseurl = href + + # first thing inside the anchor tag is another tag + # that produces some output + if ( + start + and self.maybe_automatic_link is not None + and tag not in ["p", "div", "style", "dl", "dt"] + and (tag != "img" or self.ignore_images) + ): + self.o("[") + self.maybe_automatic_link = None + self.empty_link = False + + if self.google_doc: + # the attrs parameter is empty for a closing tag. in addition, we + # need the attributes of the parent nodes in order to get a + # complete style description for the current element. we assume + # that google docs export well formed html. + parent_style: Dict[str, str] = {} + if start: + if self.tag_stack: + parent_style = self.tag_stack[-1][2] + tag_style = element_style(attrs, self.style_def, parent_style) + self.tag_stack.append((tag, attrs, tag_style)) + else: + dummy, attrs, tag_style = ( + self.tag_stack.pop() if self.tag_stack else (None, {}, {}) + ) + if self.tag_stack: + parent_style = self.tag_stack[-1][2] + + if hn(tag): + # check if nh is inside of an 'a' tag (incorrect but found in the wild) + if self.astack: + if start: + self.inheader = True + # are inside link name, so only add '#' if it can appear before '[' + if self.outtextlist and self.outtextlist[-1] == "[": + self.outtextlist.pop() + self.space = False + self.o(hn(tag) * "#" + " ") + self.o("[") + else: + self.p_p = 0 # don't break up link name + self.inheader = False + return # prevent redundant emphasis marks on headers + else: + self.p() + if start: + self.inheader = True + self.o(hn(tag) * "#" + " ") + else: + self.inheader = False + return # prevent redundant emphasis marks on headers + + if tag in ["p", "div"]: + if self.google_doc: + if start and google_has_height(tag_style): + self.p() + else: + self.soft_br() + elif self.astack: + pass + elif self.split_next_td: + pass + else: + self.p() + + if tag == "br" and start: + if self.blockquote > 0: + self.o(" \n> ") + else: + self.o(" \n") + + if tag == "hr" and start: + self.p() + self.o("* * *") + self.p() + + if tag in ["head", "style", "script"]: + if start: + self.quiet += 1 + else: + self.quiet -= 1 + + if tag == "style": + if start: + self.style += 1 + else: + self.style -= 1 + + if tag in ["body"]: + self.quiet = 0 # sites like 9rules.com never close + + if tag == "blockquote": + if start: + self.p() + self.o("> ", force=True) + self.start = True + self.blockquote += 1 + else: + self.blockquote -= 1 + self.p() + + if tag in ["em", "i", "u"] and not self.ignore_emphasis: + # Separate with a space if we immediately follow an alphanumeric + # character, since otherwise Markdown won't render the emphasis + # marks, and we'll be left with eg 'foo_bar_' visible. + # (Don't add a space otherwise, though, since there isn't one in the + # original HTML.) + if ( + start + and self.preceding_data + and self.preceding_data[-1] not in string.whitespace + and self.preceding_data[-1] not in string.punctuation + ): + emphasis = " " + self.emphasis_mark + self.preceding_data += " " + else: + emphasis = self.emphasis_mark + + self.o(emphasis) + if start: + self.stressed = True + + if tag in ["strong", "b"] and not self.ignore_emphasis: + # Separate with space if we immediately follow an * character, since + # without it, Markdown won't render the resulting *** correctly. + # (Don't add a space otherwise, though, since there isn't one in the + # original HTML.) + if ( + start + and self.preceding_data + # When `self.strong_mark` is set to empty, the next condition + # will cause IndexError since it's trying to match the data + # with the first character of the `self.strong_mark`. + and len(self.strong_mark) > 0 + and self.preceding_data[-1] == self.strong_mark[0] + ): + strong = " " + self.strong_mark + self.preceding_data += " " + else: + strong = self.strong_mark + + self.o(strong) + if start: + self.stressed = True + + if tag in ["del", "strike", "s"]: + if start and self.preceding_data and self.preceding_data[-1] == "~": + strike = " ~~" + self.preceding_data += " " + else: + strike = "~~" + + self.o(strike) + if start: + self.stressed = True + + if self.google_doc: + if not self.inheader: + # handle some font attributes, but leave headers clean + self.handle_emphasis(start, tag_style, parent_style) + + if tag in ["kbd", "code", "tt"] and not self.pre: + self.o("`") # TODO: `` `this` `` + self.code = not self.code + + if tag == "abbr": + if start: + self.abbr_title = None + self.abbr_data = "" + if "title" in attrs: + self.abbr_title = attrs["title"] + else: + if self.abbr_title is not None: + assert self.abbr_data is not None + self.abbr_list[self.abbr_data] = self.abbr_title + self.abbr_title = None + self.abbr_data = None + + if tag == "q": + if not self.quote: + self.o(self.open_quote) + else: + self.o(self.close_quote) + self.quote = not self.quote + + def link_url(self: HTML2Text, link: str, title: str = "") -> None: + url = urlparse.urljoin(self.baseurl, link) + title = ' "{}"'.format(title) if title.strip() else "" + self.o("]({url}{title})".format(url=escape_md(url), title=title)) + + if tag == "a" and not self.ignore_links: + if start: + self.inside_link = True + if ( + "href" in attrs + and attrs["href"] is not None + and not (self.skip_internal_links and attrs["href"].startswith("#")) + and not ( + self.ignore_mailto_links and attrs["href"].startswith("mailto:") + ) + ): + self.astack.append(attrs) + self.maybe_automatic_link = attrs["href"] + self.empty_link = True + if self.protect_links: + attrs["href"] = "<" + attrs["href"] + ">" + else: + self.astack.append(None) + else: + self.inside_link = False + if self.astack: + a = self.astack.pop() + if self.maybe_automatic_link and not self.empty_link: + self.maybe_automatic_link = None + elif a: + assert a["href"] is not None + if self.empty_link: + self.o("[") + self.empty_link = False + self.maybe_automatic_link = None + if self.inline_links: + self.p_p = 0 + title = a.get("title") or "" + title = escape_md(title) + link_url(self, a["href"], title) + else: + i = self.previousIndex(a) + if i is not None: + a_props = self.a[i] + else: + self.acount += 1 + a_props = AnchorElement(a, self.acount, self.outcount) + self.a.append(a_props) + self.o("][" + str(a_props.count) + "]") + + if tag == "img" and start and not self.ignore_images: + if "src" in attrs and attrs["src"] is not None: + if not self.images_to_alt: + attrs["href"] = attrs["src"] + alt = attrs.get("alt") or self.default_image_alt + + # If we have images_with_size, write raw html including width, + # height, and alt attributes + if self.images_as_html or ( + self.images_with_size and ("width" in attrs or "height" in attrs) + ): + self.o("") + return + + # If we have a link to create, output the start + if self.maybe_automatic_link is not None: + href = self.maybe_automatic_link + if ( + self.images_to_alt + and escape_md(alt) == href + and self.absolute_url_matcher.match(href) + ): + self.o("<" + escape_md(alt) + ">") + self.empty_link = False + return + else: + self.o("[") + self.maybe_automatic_link = None + self.empty_link = False + + # If we have images_to_alt, we discard the image itself, + # considering only the alt text. + if self.images_to_alt: + self.o(escape_md(alt)) + else: + self.o("![" + escape_md(alt) + "]") + if self.inline_links: + href = attrs.get("href") or "" + self.o( + "(" + escape_md(urlparse.urljoin(self.baseurl, href)) + ")" + ) + else: + i = self.previousIndex(attrs) + if i is not None: + a_props = self.a[i] + else: + self.acount += 1 + a_props = AnchorElement(attrs, self.acount, self.outcount) + self.a.append(a_props) + self.o("[" + str(a_props.count) + "]") + + if tag == "dl" and start: + self.p() # Add paragraph break before list starts + self.p_p = 0 # Reset paragraph state + + elif tag == "dt" and start: + if self.p_p == 0: # If not first term + self.o("\n\n") # Add spacing before new term-definition pair + self.p_p = 0 # Reset paragraph state + + elif tag == "dt" and not start: + self.o("\n") # Single newline between term and definition + + elif tag == "dd" and start: + self.o(" ") # Indent definition + + elif tag == "dd" and not start: + self.p_p = 0 + + if tag in ["ol", "ul"]: + # Google Docs create sub lists as top level lists + if not self.list and not self.lastWasList: + self.p() + if start: + if self.google_doc: + list_style = google_list_style(tag_style) + else: + list_style = tag + numbering_start = list_numbering_start(attrs) + self.list.append(ListElement(list_style, numbering_start)) + else: + if self.list: + self.list.pop() + if not self.google_doc and not self.list: + self.o("\n") + self.lastWasList = True + else: + self.lastWasList = False + + if tag == "li": + self.pbr() + if start: + if self.list: + li = self.list[-1] + else: + li = ListElement("ul", 0) + if self.google_doc: + self.o(" " * self.google_nest_count(tag_style)) + else: + # Indent two spaces per list, except use three spaces for an + # unordered list inside an ordered list. + # https://spec.commonmark.org/0.28/#motivation + # TODO: line up
  1. s > 9 correctly. + parent_list = None + for list in self.list: + self.o( + " " if parent_list == "ol" and list.name == "ul" else " " + ) + parent_list = list.name + + if li.name == "ul": + self.o(self.ul_item_mark + " ") + elif li.name == "ol": + li.num += 1 + self.o(str(li.num) + ". ") + self.start = True + + if tag == "caption": + if not start: + # Ensure caption text ends on its own line before table rows + self.soft_br() + + if tag in ["table", "tr", "td", "th"]: + if self.ignore_tables: + if tag == "tr": + if start: + pass + else: + self.soft_br() + else: + pass + + elif self.bypass_tables: + if start: + self.soft_br() + attr_str = "" + if attrs: + attr_str = "".join( + ' {}="{}"'.format(k, v) if v is not None else " {}".format(k) + for k, v in attrs.items() + ) + if tag in ["td", "th"]: + self.o("<{}{}>\n\n".format(tag, attr_str)) + else: + self.o("<{}{}>".format(tag, attr_str)) + else: + if tag in ["td", "th"]: + self.o("\n".format(tag)) + else: + self.o("".format(tag)) + + else: + if tag == "table": + if start: + self.table_start = True + if self.pad_tables: + self.o("<" + config.TABLE_MARKER_FOR_PAD + ">") + self.o(" \n") + else: + # Ensure table starts on its own line (GFM requirement) + self.soft_br() + else: + if self.pad_tables: + # add break in case the table is empty or its 1 row table + self.soft_br() + self.o("") + self.o(" \n") + if tag in ["td", "th"] and start: + if self.pad_tables: + # pad_tables mode: keep upstream inter-cell delimiter only + # (pad post-processor adds leading/trailing pipes and alignment) + if self.split_next_td: + self.o("| ") + else: + # GFM mode: leading pipe on first cell, spaced pipes between cells + if self.split_next_td: + self.o(" | ") + else: + self.o("| ") + self.split_next_td = True + + if tag == "tr" and start: + self.td_count = 0 + if tag == "tr" and not start: + if not self.pad_tables: + # Add trailing pipe for GFM compliance + self.o(" |") + self.split_next_td = False + self.soft_br() + if tag == "tr" and not start and self.table_start: + if self.pad_tables: + # pad_tables: plain separator (post-processor reformats) + self.o("|".join(["---"] * self.td_count)) + else: + # GFM: separator with leading/trailing pipes + self.o("| " + " | ".join(["---"] * self.td_count) + " |") + self.soft_br() + self.table_start = False + if tag in ["td", "th"] and start: + self.td_count += 1 + + if tag == "pre": + if start: + self.startpre = True + self.pre = True + else: + self.pre = False + if self.mark_code: + self.out("\n[/code]") + self.p() + + if tag in ["sup", "sub"] and self.include_sup_sub: + if start: + self.o("<{}>".format(tag)) + else: + self.o("".format(tag)) + + # TODO: Add docstring for these one letter functions + def pbr(self) -> None: + "Pretty print has a line break" + if self.p_p == 0: + self.p_p = 1 + + def p(self) -> None: + "Set pretty print to 1 or 2 lines" + self.p_p = 1 if self.single_line_break else 2 + + def soft_br(self) -> None: + "Soft breaks" + self.pbr() + self.br_toggle = " " + + def o( + self, data: str, puredata: bool = False, force: Union[bool, str] = False + ) -> None: + """ + Deal with indentation and whitespace + """ + if self.abbr_data is not None: + self.abbr_data += data + + if not self.quiet: + if self.google_doc: + # prevent white space immediately after 'begin emphasis' + # marks ('**' and '_') + lstripped_data = data.lstrip() + if self.drop_white_space and not (self.pre or self.code): + data = lstripped_data + if lstripped_data != "": + self.drop_white_space = 0 + + if puredata and not self.pre: + # This is a very dangerous call ... it could mess up + # all handling of   when not handled properly + # (see entityref) + data = re.sub(r"\s+", r" ", data) + if data and data[0] == " ": + self.space = True + data = data[1:] + if not data and not force: + return + + if self.startpre: + # self.out(" :") #TODO: not output when already one there + if not data.startswith("\n") and not data.startswith("\r\n"): + # 
    stuff...
+                    data = "\n" + data
+                if self.mark_code:
+                    self.out("\n[code]")
+                    self.p_p = 0
+
+            bq = ">" * self.blockquote
+            if not (force and data and data[0] == ">") and self.blockquote:
+                bq += " "
+
+            if self.pre:
+                if not self.list:
+                    bq += "    "
+                # else: list content is already partially indented
+                bq += "    " * len(self.list)
+                data = data.replace("\n", "\n" + bq)
+
+            if self.startpre:
+                self.startpre = False
+                if self.list:
+                    # use existing initial indentation
+                    data = data.lstrip("\n")
+
+            if self.start:
+                self.space = False
+                self.p_p = 0
+                self.start = False
+
+            if force == "end":
+                # It's the end.
+                self.p_p = 0
+                self.out("\n")
+                self.space = False
+
+            if self.p_p:
+                self.out((self.br_toggle + "\n" + bq) * self.p_p)
+                self.space = False
+                self.br_toggle = ""
+
+            if self.space:
+                if not self.lastWasNL:
+                    self.out(" ")
+                self.space = False
+
+            if self.a and (
+                (self.p_p == 2 and self.links_each_paragraph) or force == "end"
+            ):
+                if force == "end":
+                    self.out("\n")
+
+                newa = []
+                for link in self.a:
+                    if self.outcount > link.outcount:
+                        self.out(
+                            "   ["
+                            + str(link.count)
+                            + "]: "
+                            + urlparse.urljoin(self.baseurl, link.attrs["href"])
+                        )
+                        if "title" in link.attrs and link.attrs["title"] is not None:
+                            self.out(" (" + link.attrs["title"] + ")")
+                        self.out("\n")
+                    else:
+                        newa.append(link)
+
+                # Don't need an extra line when nothing was done.
+                if self.a != newa:
+                    self.out("\n")
+
+                self.a = newa
+
+            if self.abbr_list and force == "end":
+                for abbr, definition in self.abbr_list.items():
+                    self.out("  *[" + abbr + "]: " + definition + "\n")
+
+            self.p_p = 0
+            self.out(data)
+            self.outcount += 1
+
+    def handle_data(self, data: str, entity_char: bool = False) -> None:
+        if not data:
+            # Data may be empty for some HTML entities. For example,
+            # LEFT-TO-RIGHT MARK.
+            return
+
+        if self.stressed:
+            data = data.strip()
+            self.stressed = False
+            self.preceding_stressed = True
+        elif self.preceding_stressed:
+            if (
+                re.match(r"[^][(){}\s.!?]", data[0])
+                and not hn(self.current_tag)
+                and self.current_tag not in ["a", "code", "pre"]
+            ):
+                # should match a letter or common punctuation
+                data = " " + data
+            self.preceding_stressed = False
+
+        if self.style:
+            self.style_def.update(dumb_css_parser(data))
+
+        if self.maybe_automatic_link is not None:
+            href = self.maybe_automatic_link
+            if (
+                href == data
+                and self.absolute_url_matcher.match(href)
+                and self.use_automatic_links
+            ):
+                self.o("<" + data + ">")
+                self.empty_link = False
+                return
+            else:
+                self.o("[")
+                self.maybe_automatic_link = None
+                self.empty_link = False
+
+        if not self.code and not self.pre and not entity_char:
+            data = escape_md_section(
+                data,
+                snob=self.escape_snob,
+                escape_dot=self.escape_dot,
+                escape_plus=self.escape_plus,
+                escape_dash=self.escape_dash,
+            )
+        self.preceding_data = data
+        self.o(data, puredata=True)
+
+    def charref(self, name: str) -> str:
+        if name[0] in ["x", "X"]:
+            c = int(name[1:], 16)
+        else:
+            c = int(name)
+
+        if not self.unicode_snob and c in unifiable_n:
+            return unifiable_n[c]
+        else:
+            try:
+                return chr(c)
+            except ValueError:  # invalid unicode
+                return ""
+
+    def entityref(self, c: str) -> str:
+        if not self.unicode_snob and c in config.UNIFIABLE:
+            return config.UNIFIABLE[c]
+        try:
+            ch = html.entities.html5[c + ";"]
+        except KeyError:
+            return "&" + c + ";"
+        return config.UNIFIABLE[c] if c == "nbsp" else ch
+
+    def google_nest_count(self, style: Dict[str, str]) -> int:
+        """
+        Calculate the nesting count of google doc lists
+
+        :type style: dict
+
+        :rtype: int
+        """
+        nest_count = 0
+        if "margin-left" in style:
+            nest_count = int(style["margin-left"][:-2]) // self.google_list_indent
+
+        return nest_count
+
+    def optwrap(self, text: str) -> str:
+        """
+        Wrap all paragraphs in the provided text.
+
+        :type text: str
+
+        :rtype: str
+        """
+        if not self.body_width:
+            return text
+
+        result = ""
+        newlines = 0
+        # I cannot think of a better solution for now.
+        # To avoid the non-wrap behaviour for entire paras
+        # because of the presence of a link in it
+        if not self.wrap_links:
+            self.inline_links = False
+        for para in text.split("\n"):
+            if len(para) > 0:
+                if not skipwrap(
+                    para, self.wrap_links, self.wrap_list_items, self.wrap_tables
+                ):
+                    indent = ""
+                    if para.startswith("  " + self.ul_item_mark):
+                        # list item continuation: add a double indent to the
+                        # new lines
+                        indent = "    "
+                    elif para.startswith("> "):
+                        # blockquote continuation: add the greater than symbol
+                        # to the new lines
+                        indent = "> "
+                    wrapped = wrap(
+                        para,
+                        self.body_width,
+                        break_long_words=False,
+                        subsequent_indent=indent,
+                    )
+                    result += "\n".join(wrapped)
+                    if para.endswith("  "):
+                        result += "  \n"
+                        newlines = 1
+                    elif indent:
+                        result += "\n"
+                        newlines = 1
+                    else:
+                        result += "\n\n"
+                        newlines = 2
+                else:
+                    # Warning for the tempted!!!
+                    # Be aware that obvious replacement of this with
+                    # line.isspace()
+                    # DOES NOT work! Explanations are welcome.
+                    if not config.RE_SPACE.match(para):
+                        result += para + "\n"
+                        newlines = 1
+            else:
+                if newlines < 2:
+                    result += "\n"
+                    newlines += 1
+        return result
+
+
+def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = None) -> str:
+    if bodywidth is None:
+        bodywidth = config.BODY_WIDTH
+    h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
+
+    return h.handle(html)
+
+
+class CustomHTML2Text(HTML2Text):
+    def __init__(self, *args, handle_code_in_pre=False, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.inside_pre = False
+        self.inside_code = False
+        self.inside_link = False
+        self.preserve_tags = set()  # Set of tags to preserve
+        self.current_preserved_tag = None
+        self.preserved_content = []
+        self.preserve_depth = 0
+        self.handle_code_in_pre = handle_code_in_pre
+
+        # Configuration options
+        self.skip_internal_links = False
+        self.single_line_break = False
+        self.mark_code = False
+        self.include_sup_sub = False
+        self.body_width = 0
+        self.ignore_mailto_links = True
+        self.ignore_links = False
+        self.escape_backslash = False
+        self.escape_dot = False
+        self.escape_plus = False
+        self.escape_dash = False
+        self.escape_snob = False
+
+    def update_params(self, **kwargs):
+        """Update parameters and set preserved tags."""
+        for key, value in kwargs.items():
+            if key == "preserve_tags":
+                self.preserve_tags = set(value)
+            elif key == "handle_code_in_pre":
+                self.handle_code_in_pre = value
+            else:
+                setattr(self, key, value)
+
+    def handle_tag(self, tag, attrs, start):
+        # Handle  tag to update base URL for relative links
+        # Must be handled before preserved tags since  is in 
+        if tag == "base" and start:
+            href = attrs.get("href") if attrs else None
+            if href:
+                self.baseurl = href
+            # Also let parent class handle it
+            return super().handle_tag(tag, attrs, start)
+
+        # Handle preserved tags
+        if tag in self.preserve_tags:
+            if start:
+                if self.preserve_depth == 0:
+                    self.current_preserved_tag = tag
+                    self.preserved_content = []
+                    # Format opening tag with attributes
+                    attr_str = "".join(
+                        f' {k}="{v}"' for k, v in attrs.items() if v is not None
+                    )
+                    self.preserved_content.append(f"<{tag}{attr_str}>")
+                self.preserve_depth += 1
+                return
+            else:
+                self.preserve_depth -= 1
+                if self.preserve_depth == 0:
+                    self.preserved_content.append(f"")
+                    # Output the preserved HTML block with proper spacing
+                    preserved_html = "".join(self.preserved_content)
+                    self.o("\n" + preserved_html + "\n")
+                    self.current_preserved_tag = None
+                return
+
+        # If we're inside a preserved tag, collect all content
+        if self.preserve_depth > 0:
+            if start:
+                # Format nested tags with attributes
+                attr_str = "".join(
+                    f' {k}="{v}"' for k, v in attrs.items() if v is not None
+                )
+                self.preserved_content.append(f"<{tag}{attr_str}>")
+            else:
+                self.preserved_content.append(f"")
+            return
+
+        # Handle pre tags
+        if tag == "pre":
+            if start:
+                lang = attrs.get("data-language", "")
+                self.o(f"\n```{lang}\n")  # Markdown code block start
+                self.inside_pre = True
+            else:
+                self.o("\n```\n")  # Markdown code block end
+                self.inside_pre = False
+        elif tag == "code":
+            if self.inside_pre and not self.handle_code_in_pre:
+                # Ignore code tags inside pre blocks if handle_code_in_pre is False
+                return
+            if start:
+                if not self.inside_link:
+                    self.o("`")  # Only output backtick if not inside a link
+                self.inside_code = True
+            else:
+                if not self.inside_link:
+                    self.o("`")  # Only output backtick if not inside a link
+                self.inside_code = False
+
+            # If inside a link, let the parent class handle the content
+            if self.inside_link:
+                super().handle_tag(tag, attrs, start) 
+        else:
+            super().handle_tag(tag, attrs, start)
+
+    def handle_data(self, data, entity_char=False):
+        """Override handle_data to capture content within preserved tags."""
+        if self.preserve_depth > 0:
+            self.preserved_content.append(data)
+            return
+
+        if self.inside_pre:
+            # Output the raw content for pre blocks, including content inside code tags
+            self.o(data)  # Directly output the data as-is (preserve newlines)
+            return
+        if self.inside_code:
+            # Inline code: no newlines allowed
+            self.o(data.replace("\n", " "))
+            return
+
+        # Default behavior for other tags
+        super().handle_data(data, entity_char)
+
+    #     # Handle pre tags
+    #     if tag == 'pre':
+    #         if start:
+    #             self.o('```\n')
+    #             self.inside_pre = True
+    #         else:
+    #             self.o('\n```')
+    #             self.inside_pre = False
+    #     # elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+    #     #     pass
+    #     else:
+    #         super().handle_tag(tag, attrs, start)
+
+    # def handle_data(self, data, entity_char=False):
+    #     """Override handle_data to capture content within preserved tags."""
+    #     if self.preserve_depth > 0:
+    #         self.preserved_content.append(data)
+    #         return
+    #     super().handle_data(data, entity_char)
diff --git a/crawl4ai/html2text/__main__.py b/crawl4ai/html2text/__main__.py
new file mode 100644
index 0000000..4e28416
--- /dev/null
+++ b/crawl4ai/html2text/__main__.py
@@ -0,0 +1,3 @@
+from .cli import main
+
+main()
diff --git a/crawl4ai/html2text/_typing.py b/crawl4ai/html2text/_typing.py
new file mode 100644
index 0000000..6e17fed
--- /dev/null
+++ b/crawl4ai/html2text/_typing.py
@@ -0,0 +1,3 @@
+class OutCallback:
+    def __call__(self, s: str) -> None:
+        ...
diff --git a/crawl4ai/html2text/cli.py b/crawl4ai/html2text/cli.py
new file mode 100644
index 0000000..0153227
--- /dev/null
+++ b/crawl4ai/html2text/cli.py
@@ -0,0 +1,330 @@
+import argparse
+import sys
+
+from . import HTML2Text, __version__, config
+
+
+def main() -> None:
+    baseurl = ""
+
+    class bcolors:
+        HEADER = "\033[95m"
+        OKBLUE = "\033[94m"
+        OKGREEN = "\033[92m"
+        WARNING = "\033[93m"
+        FAIL = "\033[91m"
+        ENDC = "\033[0m"
+        BOLD = "\033[1m"
+        UNDERLINE = "\033[4m"
+
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "--default-image-alt",
+        dest="default_image_alt",
+        default=config.DEFAULT_IMAGE_ALT,
+        help="The default alt string for images with missing ones",
+    )
+    p.add_argument(
+        "--pad-tables",
+        dest="pad_tables",
+        action="store_true",
+        default=config.PAD_TABLES,
+        help="pad the cells to equal column width in tables",
+    )
+    p.add_argument(
+        "--no-wrap-links",
+        dest="wrap_links",
+        action="store_false",
+        default=config.WRAP_LINKS,
+        help="don't wrap links during conversion",
+    )
+    p.add_argument(
+        "--wrap-list-items",
+        dest="wrap_list_items",
+        action="store_true",
+        default=config.WRAP_LIST_ITEMS,
+        help="wrap list items during conversion",
+    )
+    p.add_argument(
+        "--wrap-tables",
+        dest="wrap_tables",
+        action="store_true",
+        default=config.WRAP_TABLES,
+        help="wrap tables",
+    )
+    p.add_argument(
+        "--ignore-emphasis",
+        dest="ignore_emphasis",
+        action="store_true",
+        default=config.IGNORE_EMPHASIS,
+        help="don't include any formatting for emphasis",
+    )
+    p.add_argument(
+        "--reference-links",
+        dest="inline_links",
+        action="store_false",
+        default=config.INLINE_LINKS,
+        help="use reference style links instead of inline links",
+    )
+    p.add_argument(
+        "--ignore-links",
+        dest="ignore_links",
+        action="store_true",
+        default=config.IGNORE_ANCHORS,
+        help="don't include any formatting for links",
+    )
+    p.add_argument(
+        "--ignore-mailto-links",
+        action="store_true",
+        dest="ignore_mailto_links",
+        default=config.IGNORE_MAILTO_LINKS,
+        help="don't include mailto: links",
+    )
+    p.add_argument(
+        "--protect-links",
+        dest="protect_links",
+        action="store_true",
+        default=config.PROTECT_LINKS,
+        help="protect links from line breaks surrounding them with angle brackets",
+    )
+    p.add_argument(
+        "--ignore-images",
+        dest="ignore_images",
+        action="store_true",
+        default=config.IGNORE_IMAGES,
+        help="don't include any formatting for images",
+    )
+    p.add_argument(
+        "--images-as-html",
+        dest="images_as_html",
+        action="store_true",
+        default=config.IMAGES_AS_HTML,
+        help=(
+            "Always write image tags as raw html; preserves `height`, `width` and "
+            "`alt` if possible."
+        ),
+    )
+    p.add_argument(
+        "--images-to-alt",
+        dest="images_to_alt",
+        action="store_true",
+        default=config.IMAGES_TO_ALT,
+        help="Discard image data, only keep alt text",
+    )
+    p.add_argument(
+        "--images-with-size",
+        dest="images_with_size",
+        action="store_true",
+        default=config.IMAGES_WITH_SIZE,
+        help=(
+            "Write image tags with height and width attrs as raw html to retain "
+            "dimensions"
+        ),
+    )
+    p.add_argument(
+        "-g",
+        "--google-doc",
+        action="store_true",
+        dest="google_doc",
+        default=False,
+        help="convert an html-exported Google Document",
+    )
+    p.add_argument(
+        "-d",
+        "--dash-unordered-list",
+        action="store_true",
+        dest="ul_style_dash",
+        default=False,
+        help="use a dash rather than a star for unordered list items",
+    )
+    p.add_argument(
+        "-e",
+        "--asterisk-emphasis",
+        action="store_true",
+        dest="em_style_asterisk",
+        default=False,
+        help="use an asterisk rather than an underscore for emphasized text",
+    )
+    p.add_argument(
+        "-b",
+        "--body-width",
+        dest="body_width",
+        type=int,
+        default=config.BODY_WIDTH,
+        help="number of characters per output line, 0 for no wrap",
+    )
+    p.add_argument(
+        "-i",
+        "--google-list-indent",
+        dest="list_indent",
+        type=int,
+        default=config.GOOGLE_LIST_INDENT,
+        help="number of pixels Google indents nested lists",
+    )
+    p.add_argument(
+        "-s",
+        "--hide-strikethrough",
+        action="store_true",
+        dest="hide_strikethrough",
+        default=False,
+        help="hide strike-through text. only relevant when -g is " "specified as well",
+    )
+    p.add_argument(
+        "--escape-all",
+        action="store_true",
+        dest="escape_snob",
+        default=False,
+        help=(
+            "Escape all special characters.  Output is less readable, but avoids "
+            "corner case formatting issues."
+        ),
+    )
+    p.add_argument(
+        "--bypass-tables",
+        action="store_true",
+        dest="bypass_tables",
+        default=config.BYPASS_TABLES,
+        help="Format tables in HTML rather than Markdown syntax.",
+    )
+    p.add_argument(
+        "--ignore-tables",
+        action="store_true",
+        dest="ignore_tables",
+        default=config.IGNORE_TABLES,
+        help="Ignore table-related tags (table, th, td, tr) " "while keeping rows.",
+    )
+    p.add_argument(
+        "--single-line-break",
+        action="store_true",
+        dest="single_line_break",
+        default=config.SINGLE_LINE_BREAK,
+        help=(
+            "Use a single line break after a block element rather than two line "
+            "breaks. NOTE: Requires --body-width=0"
+        ),
+    )
+    p.add_argument(
+        "--unicode-snob",
+        action="store_true",
+        dest="unicode_snob",
+        default=config.UNICODE_SNOB,
+        help="Use unicode throughout document",
+    )
+    p.add_argument(
+        "--no-automatic-links",
+        action="store_false",
+        dest="use_automatic_links",
+        default=config.USE_AUTOMATIC_LINKS,
+        help="Do not use automatic links wherever applicable",
+    )
+    p.add_argument(
+        "--no-skip-internal-links",
+        action="store_false",
+        dest="skip_internal_links",
+        default=config.SKIP_INTERNAL_LINKS,
+        help="Do not skip internal links",
+    )
+    p.add_argument(
+        "--links-after-para",
+        action="store_true",
+        dest="links_each_paragraph",
+        default=config.LINKS_EACH_PARAGRAPH,
+        help="Put links after each paragraph instead of document",
+    )
+    p.add_argument(
+        "--mark-code",
+        action="store_true",
+        dest="mark_code",
+        default=config.MARK_CODE,
+        help="Mark program code blocks with [code]...[/code]",
+    )
+    p.add_argument(
+        "--decode-errors",
+        dest="decode_errors",
+        default=config.DECODE_ERRORS,
+        help=(
+            "What to do in case of decode errors.'ignore', 'strict' and 'replace' are "
+            "acceptable values"
+        ),
+    )
+    p.add_argument(
+        "--open-quote",
+        dest="open_quote",
+        default=config.OPEN_QUOTE,
+        help="The character used to open quotes",
+    )
+    p.add_argument(
+        "--close-quote",
+        dest="close_quote",
+        default=config.CLOSE_QUOTE,
+        help="The character used to close quotes",
+    )
+    p.add_argument(
+        "--version", action="version", version=".".join(map(str, __version__))
+    )
+    p.add_argument("filename", nargs="?")
+    p.add_argument("encoding", nargs="?", default="utf-8")
+    p.add_argument(
+        "--include-sup-sub",
+        dest="include_sup_sub",
+        action="store_true",
+        default=config.INCLUDE_SUP_SUB,
+        help="Include the sup and sub tags",
+    )
+    args = p.parse_args()
+
+    if args.filename and args.filename != "-":
+        with open(args.filename, "rb") as fp:
+            data = fp.read()
+    else:
+        data = sys.stdin.buffer.read()
+
+    try:
+        html = data.decode(args.encoding, args.decode_errors)
+    except UnicodeDecodeError as err:
+        warning = bcolors.WARNING + "Warning:" + bcolors.ENDC
+        warning += " Use the " + bcolors.OKGREEN
+        warning += "--decode-errors=ignore" + bcolors.ENDC + " flag."
+        print(warning)
+        raise err
+
+    h = HTML2Text(baseurl=baseurl)
+    # handle options
+    if args.ul_style_dash:
+        h.ul_item_mark = "-"
+    if args.em_style_asterisk:
+        h.emphasis_mark = "*"
+        h.strong_mark = "__"
+
+    h.body_width = args.body_width
+    h.google_list_indent = args.list_indent
+    h.ignore_emphasis = args.ignore_emphasis
+    h.ignore_links = args.ignore_links
+    h.ignore_mailto_links = args.ignore_mailto_links
+    h.protect_links = args.protect_links
+    h.ignore_images = args.ignore_images
+    h.images_as_html = args.images_as_html
+    h.images_to_alt = args.images_to_alt
+    h.images_with_size = args.images_with_size
+    h.google_doc = args.google_doc
+    h.hide_strikethrough = args.hide_strikethrough
+    h.escape_snob = args.escape_snob
+    h.bypass_tables = args.bypass_tables
+    h.ignore_tables = args.ignore_tables
+    h.single_line_break = args.single_line_break
+    h.inline_links = args.inline_links
+    h.unicode_snob = args.unicode_snob
+    h.use_automatic_links = args.use_automatic_links
+    h.skip_internal_links = args.skip_internal_links
+    h.links_each_paragraph = args.links_each_paragraph
+    h.mark_code = args.mark_code
+    h.wrap_links = args.wrap_links
+    h.wrap_list_items = args.wrap_list_items
+    h.wrap_tables = args.wrap_tables
+    h.pad_tables = args.pad_tables
+    h.default_image_alt = args.default_image_alt
+    h.open_quote = args.open_quote
+    h.close_quote = args.close_quote
+    h.include_sup_sub = args.include_sup_sub
+
+    sys.stdout.write(h.handle(html))
diff --git a/crawl4ai/html2text/config.py b/crawl4ai/html2text/config.py
new file mode 100644
index 0000000..d14ed64
--- /dev/null
+++ b/crawl4ai/html2text/config.py
@@ -0,0 +1,172 @@
+import re
+
+# Use Unicode characters instead of their ascii pseudo-replacements
+UNICODE_SNOB = False
+
+# Marker to use for marking tables for padding post processing
+TABLE_MARKER_FOR_PAD = "special_marker_for_table_padding"
+# Escape all special characters.  Output is less readable, but avoids
+# corner case formatting issues.
+ESCAPE_SNOB = False
+ESCAPE_BACKSLASH = False
+ESCAPE_DOT = False
+ESCAPE_PLUS = False
+ESCAPE_DASH = False
+
+# Put the links after each paragraph instead of at the end.
+LINKS_EACH_PARAGRAPH = False
+
+# Wrap long lines at position. 0 for no wrapping.
+BODY_WIDTH = 78
+
+# Don't show internal links (href="#local-anchor") -- corresponding link
+# targets won't be visible in the plain text file anyway.
+SKIP_INTERNAL_LINKS = True
+
+# Use inline, rather than reference, formatting for images and links
+INLINE_LINKS = True
+
+# Protect links from line breaks surrounding them with angle brackets (in
+# addition to their square brackets)
+PROTECT_LINKS = False
+# WRAP_LINKS = True
+WRAP_LINKS = True
+
+# Wrap list items.
+WRAP_LIST_ITEMS = False
+
+# Wrap tables
+WRAP_TABLES = False
+
+# Number of pixels Google indents nested lists
+GOOGLE_LIST_INDENT = 36
+
+# Values Google and others may use to indicate bold text
+BOLD_TEXT_STYLE_VALUES = ("bold", "700", "800", "900")
+
+IGNORE_ANCHORS = False
+IGNORE_MAILTO_LINKS = False
+IGNORE_IMAGES = False
+IMAGES_AS_HTML = False
+IMAGES_TO_ALT = False
+IMAGES_WITH_SIZE = False
+IGNORE_EMPHASIS = False
+MARK_CODE = False
+DECODE_ERRORS = "strict"
+DEFAULT_IMAGE_ALT = ""
+PAD_TABLES = False
+
+# Convert links with same href and text to  format
+# if they are absolute links
+USE_AUTOMATIC_LINKS = True
+
+# For checking space-only lines on line 771
+RE_SPACE = re.compile(r"\s\+")
+
+RE_ORDERED_LIST_MATCHER = re.compile(r"\d+\.\s")
+RE_UNORDERED_LIST_MATCHER = re.compile(r"[-\*\+]\s")
+RE_MD_CHARS_MATCHER = re.compile(r"([\\\[\]\(\)])")
+RE_MD_CHARS_MATCHER_ALL = re.compile(r"([`\*_{}\[\]\(\)#!])")
+
+# to find links in the text
+RE_LINK = re.compile(r"(\[.*?\] ?\(.*?\))|(\[.*?\]:.*?)")
+
+# to find table separators
+RE_TABLE = re.compile(r" \| ")
+
+RE_MD_DOT_MATCHER = re.compile(
+    r"""
+    ^             # start of line
+    (\s*\d+)      # optional whitespace and a number
+    (\.)          # dot
+    (?=\s)        # lookahead assert whitespace
+    """,
+    re.MULTILINE | re.VERBOSE,
+)
+RE_MD_PLUS_MATCHER = re.compile(
+    r"""
+    ^
+    (\s*)
+    (\+)
+    (?=\s)
+    """,
+    flags=re.MULTILINE | re.VERBOSE,
+)
+RE_MD_DASH_MATCHER = re.compile(
+    r"""
+    ^
+    (\s*)
+    (-)
+    (?=\s|\-)     # followed by whitespace (bullet list, or spaced out hr)
+                  # or another dash (header or hr)
+    """,
+    flags=re.MULTILINE | re.VERBOSE,
+)
+RE_SLASH_CHARS = r"\`*_{}[]()#+-.!"
+RE_MD_BACKSLASH_MATCHER = re.compile(
+    r"""
+    (\\)          # match one slash
+    (?=[%s])      # followed by a char that requires escaping
+    """
+    % re.escape(RE_SLASH_CHARS),
+    flags=re.VERBOSE,
+)
+
+UNIFIABLE = {
+    "rsquo": "'",
+    "lsquo": "'",
+    "rdquo": '"',
+    "ldquo": '"',
+    "copy": "(C)",
+    "mdash": "--",
+    "nbsp": " ",
+    "rarr": "->",
+    "larr": "<-",
+    "middot": "*",
+    "ndash": "-",
+    "oelig": "oe",
+    "aelig": "ae",
+    "agrave": "a",
+    "aacute": "a",
+    "acirc": "a",
+    "atilde": "a",
+    "auml": "a",
+    "aring": "a",
+    "egrave": "e",
+    "eacute": "e",
+    "ecirc": "e",
+    "euml": "e",
+    "igrave": "i",
+    "iacute": "i",
+    "icirc": "i",
+    "iuml": "i",
+    "ograve": "o",
+    "oacute": "o",
+    "ocirc": "o",
+    "otilde": "o",
+    "ouml": "o",
+    "ugrave": "u",
+    "uacute": "u",
+    "ucirc": "u",
+    "uuml": "u",
+    "lrm": "",
+    "rlm": "",
+}
+
+# Format tables in HTML rather than Markdown syntax
+BYPASS_TABLES = False
+# Ignore table-related tags (table, th, td, tr) while keeping rows
+IGNORE_TABLES = False
+
+
+# Use a single line break after a block element rather than two line breaks.
+# NOTE: Requires body width setting to be 0.
+SINGLE_LINE_BREAK = False
+
+
+# Use double quotation marks when converting the  tag.
+OPEN_QUOTE = '"'
+CLOSE_QUOTE = '"'
+
+# Include the  and  tags
+INCLUDE_SUP_SUB = False
diff --git a/crawl4ai/html2text/elements.py b/crawl4ai/html2text/elements.py
new file mode 100644
index 0000000..2533ec0
--- /dev/null
+++ b/crawl4ai/html2text/elements.py
@@ -0,0 +1,18 @@
+from typing import Dict, Optional
+
+
+class AnchorElement:
+    __slots__ = ["attrs", "count", "outcount"]
+
+    def __init__(self, attrs: Dict[str, Optional[str]], count: int, outcount: int):
+        self.attrs = attrs
+        self.count = count
+        self.outcount = outcount
+
+
+class ListElement:
+    __slots__ = ["name", "num"]
+
+    def __init__(self, name: str, num: int):
+        self.name = name
+        self.num = num
diff --git a/crawl4ai/html2text/utils.py b/crawl4ai/html2text/utils.py
new file mode 100644
index 0000000..21bf98f
--- /dev/null
+++ b/crawl4ai/html2text/utils.py
@@ -0,0 +1,304 @@
+import html.entities
+from typing import Dict, List, Optional
+
+from . import config
+
+unifiable_n = {
+    html.entities.name2codepoint[k]: v
+    for k, v in config.UNIFIABLE.items()
+    if k != "nbsp"
+}
+
+
+def hn(tag: str) -> int:
+    if tag[0] == "h" and len(tag) == 2:
+        n = tag[1]
+        if "0" < n <= "9":
+            return int(n)
+    return 0
+
+
+def dumb_property_dict(style: str) -> Dict[str, str]:
+    """
+    :returns: A hash of css attributes
+    """
+    return {
+        x.strip().lower(): y.strip().lower()
+        for x, y in [z.split(":", 1) for z in style.split(";") if ":" in z]
+    }
+
+
+def dumb_css_parser(data: str) -> Dict[str, Dict[str, str]]:
+    """
+    :type data: str
+
+    :returns: A hash of css selectors, each of which contains a hash of
+    css attributes.
+    :rtype: dict
+    """
+    # remove @import sentences
+    data += ";"
+    importIndex = data.find("@import")
+    while importIndex != -1:
+        data = data[0:importIndex] + data[data.find(";", importIndex) + 1 :]
+        importIndex = data.find("@import")
+
+    # parse the css. reverted from dictionary comprehension in order to
+    # support older pythons
+    pairs = [x.split("{") for x in data.split("}") if "{" in x.strip()]
+    try:
+        elements = {a.strip(): dumb_property_dict(b) for a, b in pairs}
+    except ValueError:
+        elements = {}  # not that important
+
+    return elements
+
+
+def element_style(
+    attrs: Dict[str, Optional[str]],
+    style_def: Dict[str, Dict[str, str]],
+    parent_style: Dict[str, str],
+) -> Dict[str, str]:
+    """
+    :type attrs: dict
+    :type style_def: dict
+    :type style_def: dict
+
+    :returns: A hash of the 'final' style attributes of the element
+    :rtype: dict
+    """
+    style = parent_style.copy()
+    if "class" in attrs:
+        assert attrs["class"] is not None
+        for css_class in attrs["class"].split():
+            css_style = style_def.get("." + css_class, {})
+            style.update(css_style)
+    if "style" in attrs:
+        assert attrs["style"] is not None
+        immediate_style = dumb_property_dict(attrs["style"])
+        style.update(immediate_style)
+
+    return style
+
+
+def google_list_style(style: Dict[str, str]) -> str:
+    """
+    Finds out whether this is an ordered or unordered list
+
+    :type style: dict
+
+    :rtype: str
+    """
+    if "list-style-type" in style:
+        list_style = style["list-style-type"]
+        if list_style in ["disc", "circle", "square", "none"]:
+            return "ul"
+
+    return "ol"
+
+
+def google_has_height(style: Dict[str, str]) -> bool:
+    """
+    Check if the style of the element has the 'height' attribute
+    explicitly defined
+
+    :type style: dict
+
+    :rtype: bool
+    """
+    return "height" in style
+
+
+def google_text_emphasis(style: Dict[str, str]) -> List[str]:
+    """
+    :type style: dict
+
+    :returns: A list of all emphasis modifiers of the element
+    :rtype: list
+    """
+    emphasis = []
+    if "text-decoration" in style:
+        emphasis.append(style["text-decoration"])
+    if "font-style" in style:
+        emphasis.append(style["font-style"])
+    if "font-weight" in style:
+        emphasis.append(style["font-weight"])
+
+    return emphasis
+
+
+def google_fixed_width_font(style: Dict[str, str]) -> bool:
+    """
+    Check if the css of the current element defines a fixed width font
+
+    :type style: dict
+
+    :rtype: bool
+    """
+    font_family = ""
+    if "font-family" in style:
+        font_family = style["font-family"]
+    return "courier new" == font_family or "consolas" == font_family
+
+
+def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
+    """
+    Extract numbering from list element attributes
+
+    :type attrs: dict
+
+    :rtype: int or None
+    """
+    if "start" in attrs:
+        assert attrs["start"] is not None
+        try:
+            return int(attrs["start"]) - 1
+        except ValueError:
+            pass
+
+    return 0
+
+
+def skipwrap(
+    para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool
+) -> bool:
+    # If it appears to contain a link
+    # don't wrap
+    if not wrap_links and config.RE_LINK.search(para):
+        return True
+    # If the text begins with four spaces or one tab, it's a code block;
+    # don't wrap
+    if para[0:4] == "    " or para[0] == "\t":
+        return True
+
+    # If the text begins with only two "--", possibly preceded by
+    # whitespace, that's an emdash; so wrap.
+    stripped = para.lstrip()
+    if stripped[0:2] == "--" and len(stripped) > 2 and stripped[2] != "-":
+        return False
+
+    # I'm not sure what this is for; I thought it was to detect lists,
+    # but there's a 
    -inside- case in one of the tests that
+    # also depends upon it.
+    if stripped[0:1] in ("-", "*") and not stripped[0:2] == "**":
+        return not wrap_list_items
+
+    # If text contains a pipe character it is likely a table
+    if not wrap_tables and config.RE_TABLE.search(para):
+        return True
+
+    # If the text begins with a single -, *, or +, followed by a space,
+    # or an integer, followed by a ., followed by a space (in either
+    # case optionally proceeded by whitespace), it's a list; don't wrap.
+    return bool(
+        config.RE_ORDERED_LIST_MATCHER.match(stripped)
+        or config.RE_UNORDERED_LIST_MATCHER.match(stripped)
+    )
+
+
+def escape_md(text: str) -> str:
+    """
+    Escapes markdown-sensitive characters within other markdown
+    constructs.
+    """
+    return config.RE_MD_CHARS_MATCHER.sub(r"\\\1", text)
+
+
+def escape_md_section(
+    text: str,
+    escape_backslash: bool = True,
+    snob: bool = False,
+    escape_dot: bool = True,
+    escape_plus: bool = True,
+    escape_dash: bool = True,
+) -> str:
+    """
+    Escapes markdown-sensitive characters across whole document sections.
+    Each escaping operation can be controlled individually.
+    """
+    if escape_backslash:
+        text = config.RE_MD_BACKSLASH_MATCHER.sub(r"\\\1", text)
+
+    if snob:
+        text = config.RE_MD_CHARS_MATCHER_ALL.sub(r"\\\1", text)
+
+    if escape_dot:
+        text = config.RE_MD_DOT_MATCHER.sub(r"\1\\\2", text)
+
+    if escape_plus:
+        text = config.RE_MD_PLUS_MATCHER.sub(r"\1\\\2", text)
+
+    if escape_dash:
+        text = config.RE_MD_DASH_MATCHER.sub(r"\1\\\2", text)
+
+    return text
+
+
+def reformat_table(lines: List[str], right_margin: int) -> List[str]:
+    """
+    Given the lines of a table
+    padds the cells and returns the new lines
+    """
+    # find the maximum width of the columns
+    max_width = [len(x.rstrip()) + right_margin for x in lines[0].split("|")]
+    max_cols = len(max_width)
+    for line in lines:
+        cols = [x.rstrip() for x in line.split("|")]
+        num_cols = len(cols)
+
+        # don't drop any data if colspan attributes result in unequal lengths
+        if num_cols < max_cols:
+            cols += [""] * (max_cols - num_cols)
+        elif max_cols < num_cols:
+            max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
+            max_cols = num_cols
+
+        max_width = [
+            max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
+        ]
+
+    # reformat
+    new_lines = []
+    for line in lines:
+        cols = [x.rstrip() for x in line.split("|")]
+        if set(line.strip()) == set("-|"):
+            filler = "-"
+            new_cols = [
+                x.rstrip() + (filler * (M - len(x.rstrip())))
+                for x, M in zip(cols, max_width)
+            ]
+            new_lines.append("|-" + "|".join(new_cols) + "|")
+        else:
+            filler = " "
+            new_cols = [
+                x.rstrip() + (filler * (M - len(x.rstrip())))
+                for x, M in zip(cols, max_width)
+            ]
+            new_lines.append("| " + "|".join(new_cols) + "|")
+    return new_lines
+
+
+def pad_tables_in_text(text: str, right_margin: int = 1) -> str:
+    """
+    Provide padding for tables in the text
+    """
+    lines = text.split("\n")
+    table_buffer = []  # type: List[str]
+    table_started = False
+    new_lines = []
+    for line in lines:
+        # Toggle table started
+        if config.TABLE_MARKER_FOR_PAD in line:
+            table_started = not table_started
+            if not table_started:
+                table = reformat_table(table_buffer, right_margin)
+                new_lines.extend(table)
+                table_buffer = []
+                new_lines.append("")
+            continue
+        # Process lines
+        if table_started:
+            table_buffer.append(line)
+        else:
+            new_lines.append(line)
+    return "\n".join(new_lines)
diff --git a/crawl4ai/hub.py b/crawl4ai/hub.py
new file mode 100644
index 0000000..75056df
--- /dev/null
+++ b/crawl4ai/hub.py
@@ -0,0 +1,69 @@
+# crawl4ai/hub.py
+from abc import ABC, abstractmethod
+from typing import Dict, Type, Union
+import logging
+import importlib
+from pathlib import Path
+import inspect
+
+logger = logging.getLogger(__name__)
+
+
+class BaseCrawler(ABC):
+    def __init__(self):
+        self.logger = logging.getLogger(self.__class__.__name__)
+        
+    @abstractmethod
+    async def run(self, url: str = "", **kwargs) -> str:
+        """
+        Implement this method to return JSON string.
+        Must accept URL + arbitrary kwargs for flexibility.
+        """
+        pass
+
+    def __init_subclass__(cls, **kwargs):
+        """Enforce interface validation on subclassing"""
+        super().__init_subclass__(**kwargs)
+        
+        # Verify run method signature
+        run_method = cls.run
+        if not run_method.__code__.co_argcount >= 2:  # self + url
+            raise TypeError(f"{cls.__name__} must implement 'run(self, url: str, **kwargs)'")
+            
+        # Verify async nature
+        if not inspect.iscoroutinefunction(run_method):
+            raise TypeError(f"{cls.__name__}.run must be async")
+
+class CrawlerHub:
+    _crawlers: Dict[str, Type[BaseCrawler]] = {}
+
+    @classmethod
+    def _discover_crawlers(cls):
+        """Dynamically load crawlers from /crawlers in 3 lines"""
+        base_path = Path(__file__).parent / "crawlers"
+        for crawler_dir in base_path.iterdir():
+            if crawler_dir.is_dir():
+                try:
+                    module = importlib.import_module(
+                        f"crawl4ai.crawlers.{crawler_dir.name}.crawler"
+                    )
+                    for attr in dir(module):
+                        cls._maybe_register_crawler(
+                            getattr(module, attr), crawler_dir.name
+                        )
+                except Exception as e:
+                    logger.warning(f"Failed {crawler_dir.name}: {str(e)}")
+
+    @classmethod
+    def _maybe_register_crawler(cls, obj, name: str):
+        """Brilliant one-liner registration"""
+        if isinstance(obj, type) and issubclass(obj, BaseCrawler) and obj != BaseCrawler:
+            module = importlib.import_module(obj.__module__)
+            obj.meta = getattr(module, "__meta__", {})
+            cls._crawlers[name] = obj
+
+    @classmethod
+    def get(cls, name: str) -> Union[Type[BaseCrawler], None]:
+        if not cls._crawlers:
+            cls._discover_crawlers()
+        return cls._crawlers.get(name)
\ No newline at end of file
diff --git a/crawl4ai/install.py b/crawl4ai/install.py
new file mode 100644
index 0000000..68726ed
--- /dev/null
+++ b/crawl4ai/install.py
@@ -0,0 +1,212 @@
+import subprocess
+import sys
+import asyncio
+from .async_logger import AsyncLogger, LogLevel
+from pathlib import Path
+import os
+import shutil
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+def setup_home_directory():
+    """Set up the .crawl4ai folder structure in the user's home directory."""
+    base_dir = os.getenv("CRAWL4_AI_BASE_DIRECTORY")
+    crawl4ai_folder = Path(base_dir) if base_dir else Path.home()
+    crawl4ai_config = crawl4ai_folder / "global.yml"
+    crawl4ai_folder = crawl4ai_folder / ".crawl4ai"
+    cache_folder = crawl4ai_folder / "cache"
+    content_folders = [
+        "html_content",
+        "cleaned_html",
+        "markdown_content",
+        "extracted_content",
+        "screenshots",
+    ]
+
+    # Clean up old cache if exists
+    if cache_folder.exists():
+        shutil.rmtree(cache_folder)
+
+    # Create new folder structure
+    crawl4ai_folder.mkdir(exist_ok=True)
+    cache_folder.mkdir(exist_ok=True)
+    for folder in content_folders:
+        (crawl4ai_folder / folder).mkdir(exist_ok=True)
+    
+    # If config file does not exist, create it
+    if not crawl4ai_config.exists():
+        with open(crawl4ai_config, "w") as f:
+            f.write("")
+
+def post_install():
+    """
+    Run all post-installation tasks.
+    Checks CRAWL4AI_MODE environment variable. If set to 'api',
+    skips Playwright browser installation.
+    """
+    logger.info("Running post-installation setup...", tag="INIT")
+    setup_home_directory()
+
+    # Check environment variable to conditionally skip Playwright install
+    run_mode = os.getenv('CRAWL4AI_MODE')
+    if run_mode == 'api':
+        logger.warning(
+            "CRAWL4AI_MODE=api detected. Skipping Playwright browser installation.",
+            tag="SETUP"
+        )
+    else:
+        # Proceed with installation only if mode is not 'api'
+        install_playwright()
+
+    run_migration()
+    # TODO: Will be added in the future
+    # setup_builtin_browser()
+    logger.success("Post-installation setup completed!", tag="COMPLETE")
+    
+def setup_builtin_browser():
+    """Set up a builtin browser for use with Crawl4AI"""
+    try:
+        logger.info("Setting up builtin browser...", tag="INIT")
+        asyncio.run(_setup_builtin_browser())
+        logger.success("Builtin browser setup completed!", tag="COMPLETE")
+    except Exception as e:
+        logger.warning(f"Failed to set up builtin browser: {e}")
+        logger.warning("You can manually set up a builtin browser using 'crawl4ai-doctor builtin-browser-start'")
+    
+async def _setup_builtin_browser():
+    try:
+        # Import BrowserProfiler here to avoid circular imports
+        from .browser_profiler import BrowserProfiler
+        profiler = BrowserProfiler(logger=logger)
+        
+        # Launch the builtin browser
+        cdp_url = await profiler.launch_builtin_browser(headless=True)
+        if cdp_url:
+            logger.success(f"Builtin browser launched at {cdp_url}", tag="BROWSER")
+        else:
+            logger.warning("Failed to launch builtin browser", tag="BROWSER")
+    except Exception as e:
+        logger.warning(f"Error setting up builtin browser: {e}", tag="BROWSER")
+        raise
+
+
+def install_playwright():
+    logger.info("Installing Playwright browsers...", tag="INIT")
+    try:
+        # subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chrome"])
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                "playwright",
+                "install",
+                "--with-deps",
+                "--force",
+                "chromium",
+            ]
+        )
+        logger.success(
+            "Playwright installation completed successfully.", tag="COMPLETE"
+        )
+    except subprocess.CalledProcessError:
+        # logger.error(f"Error during Playwright installation: {e}", tag="ERROR")
+        logger.warning(
+            f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
+        )
+    except Exception:
+        # logger.error(f"Unexpected error during Playwright installation: {e}", tag="ERROR")
+        logger.warning(
+            f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
+        )
+    
+    # Install Patchright browsers for undetected browser support
+    logger.info("Installing Patchright browsers for undetected mode...", tag="INIT")
+    try:
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                "patchright",
+                "install",
+                "--with-deps",
+                "--force",
+                "chromium",
+            ]
+        )
+        logger.success(
+            "Patchright installation completed successfully.", tag="COMPLETE"
+        )
+    except subprocess.CalledProcessError:
+        logger.warning(
+            f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."
+        )
+    except Exception:
+        logger.warning(
+            f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."
+        )
+
+
+def run_migration():
+    """Initialize database during installation"""
+    try:
+        logger.info("Starting database initialization...", tag="INIT")
+        from crawl4ai.async_database import async_db_manager
+
+        asyncio.run(async_db_manager.initialize())
+        logger.success(
+            "Database initialization completed successfully.", tag="COMPLETE"
+        )
+    except ImportError:
+        logger.warning("Database module not found. Will initialize on first use.")
+    except Exception as e:
+        logger.warning(f"Database initialization failed: {e}")
+        logger.warning("Database will be initialized on first use")
+
+
+async def run_doctor():
+    """Test if Crawl4AI is working properly"""
+    logger.info("Running Crawl4AI health check...", tag="INIT")
+    try:
+        from .async_webcrawler import (
+            AsyncWebCrawler,
+            BrowserConfig,
+            CrawlerRunConfig,
+            CacheMode,
+        )
+
+        browser_config = BrowserConfig(
+            headless=True,
+            browser_type="chromium",
+            ignore_https_errors=True,
+            light_mode=True,
+            viewport_width=1280,
+            viewport_height=720,
+        )
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            screenshot=True,
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            logger.info("Testing crawling capabilities...", tag="TEST")
+            result = await crawler.arun(url="https://crawl4ai.com", config=run_config)
+
+            if result and result.markdown:
+                logger.success("✅ Crawling test passed!", tag="COMPLETE")
+                return True
+            else:
+                raise Exception("Failed to get content")
+
+    except Exception as e:
+        logger.error(f"❌ Test failed: {e}", tag="ERROR")
+        return False
+
+
+def doctor():
+    """Entry point for the doctor command"""
+    import asyncio
+
+    asyncio.run(run_doctor())
+    sys.exit(0)
diff --git a/crawl4ai/js_snippet/__init__.py b/crawl4ai/js_snippet/__init__.py
new file mode 100644
index 0000000..e51f79d
--- /dev/null
+++ b/crawl4ai/js_snippet/__init__.py
@@ -0,0 +1,18 @@
+import os
+
+
+# Create a function get name of a js script, then load from the CURRENT folder of this script and return its content as string, make sure its error free
+def load_js_script(script_name):
+    # Get the path of the current script
+    current_script_path = os.path.dirname(os.path.realpath(__file__))
+    # Get the path of the script to load
+    script_path = os.path.join(current_script_path, script_name + ".js")
+    # Check if the script exists
+    if not os.path.exists(script_path):
+        raise ValueError(
+            f"Script {script_name} not found in the folder {current_script_path}"
+        )
+    # Load the content of the script
+    with open(script_path, "r") as f:
+        script_content = f.read()
+    return script_content
diff --git a/crawl4ai/js_snippet/flatten_shadow_dom.js b/crawl4ai/js_snippet/flatten_shadow_dom.js
new file mode 100644
index 0000000..e13f3f3
--- /dev/null
+++ b/crawl4ai/js_snippet/flatten_shadow_dom.js
@@ -0,0 +1,104 @@
+/**
+ * Flatten all open shadow DOM trees into the light DOM so that
+ * page.content() / outerHTML can serialize the full composed view.
+ *
+ * Uses manual recursive serialization with proper slot resolution.
+ * Resolves slots via the live DOM API (assignedNodes), skips only
+ * shadow-scoped styles, and produces clean HTML with no regex hacks.
+ *
+ * Returns the full HTML string including shadow content.
+ */
+(() => {
+    const VOID = new Set([
+        'area','base','br','col','embed','hr','img','input',
+        'link','meta','param','source','track','wbr'
+    ]);
+
+    // Serialize a DOM node. When it has a shadow root, switch to
+    // shadow-aware serialization that resolves  elements.
+    const serialize = (node) => {
+        if (node.nodeType === Node.TEXT_NODE) return node.textContent;
+        if (node.nodeType === Node.COMMENT_NODE) return '';
+        if (node.nodeType !== Node.ELEMENT_NODE) return '';
+
+        const tag = node.tagName.toLowerCase();
+        const attrs = serializeAttrs(node);
+        let inner = '';
+
+        if (node.shadowRoot) {
+            inner = serializeShadowRoot(node);
+        } else {
+            for (const child of node.childNodes) {
+                inner += serialize(child);
+            }
+        }
+
+        if (VOID.has(tag)) return `<${tag}${attrs}>`;
+        return `<${tag}${attrs}>${inner}`;
+    };
+
+    // Serialize a shadow root's children, resolving slots against
+    // the host's light DOM children.
+    const serializeShadowRoot = (host) => {
+        let result = '';
+        for (const child of host.shadowRoot.childNodes) {
+            result += serializeShadowChild(child, host);
+        }
+        return result;
+    };
+
+    // Serialize a node that lives inside a shadow root.
+    // 
+
+```
+
+
+## File: docs/md_v2/core/browser-crawler-config.md
+
+```md
+# Browser, Crawler & LLM Configuration (Quick Overview)
+
+Crawl4AI’s flexibility stems from two key classes:
+
+1. **`BrowserConfig`** – Dictates **how** the browser is launched and behaves (e.g., headless or visible, proxy, user agent).  
+2. **`CrawlerRunConfig`** – Dictates **how** each **crawl** operates (e.g., caching, extraction, timeouts, JavaScript code to run, etc.).  
+3. **`LLMConfig`** - Dictates **how** LLM providers are configured. (model, api token, base url, temperature etc.)
+
+In most examples, you create **one** `BrowserConfig` for the entire crawler session, then pass a **fresh** or re-used `CrawlerRunConfig` whenever you call `arun()`. This tutorial shows the most commonly used parameters. If you need advanced or rarely used fields, see the [Configuration Parameters](../api/parameters.md).
+
+---
+
+## 1. BrowserConfig Essentials
+
+```python
+class BrowserConfig:
+    def __init__(
+        browser_type="chromium",
+        headless=True,
+        proxy_config=None,
+        viewport_width=1080,
+        viewport_height=600,
+        verbose=True,
+        use_persistent_context=False,
+        user_data_dir=None,
+        cookies=None,
+        headers=None,
+        user_agent=None,
+        text_mode=False,
+        light_mode=False,
+        extra_args=None,
+        # ... other advanced parameters omitted here
+    ):
+        ...
+```
+
+### Key Fields to Note
+
+
+
+1. **`browser_type`**  
+- Options: `"chromium"`, `"firefox"`, or `"webkit"`.  
+- Defaults to `"chromium"`.  
+- If you need a different engine, specify it here.
+
+2. **`headless`**  
+   - `True`: Runs the browser in headless mode (invisible browser).  
+   - `False`: Runs the browser in visible mode, which helps with debugging.
+
+3. **`proxy_config`**  
+   - A dictionary with fields like:  
+```json
+{
+    "server": "http://proxy.example.com:8080", 
+    "username": "...", 
+    "password": "..."
+}
+```
+   - Leave as `None` if a proxy is not required.
+
+4. **`viewport_width` & `viewport_height`**:  
+   - The initial window size.  
+   - Some sites behave differently with smaller or bigger viewports.
+
+5. **`verbose`**:  
+   - If `True`, prints extra logs.  
+   - Handy for debugging.
+
+6. **`use_persistent_context`**:  
+   - If `True`, uses a **persistent** browser profile, storing cookies/local storage across runs.  
+   - Typically also set `user_data_dir` to point to a folder.
+
+7. **`cookies`** & **`headers`**:  
+   - If you want to start with specific cookies or add universal HTTP headers, set them here.  
+   - E.g. `cookies=[{"name": "session", "value": "abc123", "domain": "example.com"}]`.
+
+8. **`user_agent`**:  
+   - Custom User-Agent string. If `None`, a default is used.  
+   - You can also set `user_agent_mode="random"` for randomization (if you want to fight bot detection).
+
+9. **`text_mode`** & **`light_mode`**:  
+   - `text_mode=True` disables images, possibly speeding up text-only crawls.  
+   - `light_mode=True` turns off certain background features for performance.  
+
+10. **`extra_args`**:  
+    - Additional flags for the underlying browser.  
+    - E.g. `["--disable-extensions"]`.
+
+### Helper Methods
+
+Both configuration classes provide a `clone()` method to create modified copies:
+
+```python
+# Create a base browser config
+base_browser = BrowserConfig(
+    browser_type="chromium",
+    headless=True,
+    text_mode=True
+)
+
+# Create a visible browser config for debugging
+debug_browser = base_browser.clone(
+    headless=False,
+    verbose=True
+)
+```
+
+**Minimal Example**:
+
+```python
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+browser_conf = BrowserConfig(
+    browser_type="firefox",
+    headless=False,
+    text_mode=True
+)
+
+async with AsyncWebCrawler(config=browser_conf) as crawler:
+    result = await crawler.arun("https://example.com")
+    print(result.markdown[:300])
+```
+
+---
+
+## 2. CrawlerRunConfig Essentials
+
+```python
+class CrawlerRunConfig:
+    def __init__(
+        word_count_threshold=200,
+        extraction_strategy=None,
+        markdown_generator=None,
+        cache_mode=None,
+        js_code=None,
+        wait_for=None,
+        screenshot=False,
+        pdf=False,
+        capture_mhtml=False,
+        enable_rate_limiting=False,
+        rate_limit_config=None,
+        memory_threshold_percent=70.0,
+        check_interval=1.0,
+        max_session_permit=20,
+        display_mode=None,
+        verbose=True,
+        stream=False,  # Enable streaming for arun_many()
+        # ... other advanced parameters omitted
+    ):
+        ...
+```
+
+### Key Fields to Note
+
+1. **`word_count_threshold`**:  
+   - The minimum word count before a block is considered.  
+   - If your site has lots of short paragraphs or items, you can lower it.
+
+2. **`extraction_strategy`**:  
+   - Where you plug in JSON-based extraction (CSS, LLM, etc.).  
+   - If `None`, no structured extraction is done (only raw/cleaned HTML + markdown).
+
+3. **`markdown_generator`**:  
+   - E.g., `DefaultMarkdownGenerator(...)`, controlling how HTML→Markdown conversion is done.  
+   - If `None`, a default approach is used.
+
+4. **`cache_mode`**:  
+   - Controls caching behavior (`ENABLED`, `BYPASS`, `DISABLED`, etc.).  
+   - If `None`, defaults to some level of caching or you can specify `CacheMode.ENABLED`.
+
+5. **`js_code`**:  
+   - A string or list of JS strings to execute.  
+   - Great for “Load More” buttons or user interactions.  
+
+6. **`wait_for`**:  
+   - A CSS or JS expression to wait for before extracting content.  
+   - Common usage: `wait_for="css:.main-loaded"` or `wait_for="js:() => window.loaded === true"`.
+
+7. **`screenshot`**, **`pdf`**, & **`capture_mhtml`**:  
+   - If `True`, captures a screenshot, PDF, or MHTML snapshot after the page is fully loaded.  
+   - The results go to `result.screenshot` (base64), `result.pdf` (bytes), or `result.mhtml` (string).
+8. **`verbose`**:  
+   - Logs additional runtime details.  
+   - Overlaps with the browser’s verbosity if also set to `True` in `BrowserConfig`.
+
+9. **`enable_rate_limiting`**:  
+   - If `True`, enables rate limiting for batch processing.  
+   - Requires `rate_limit_config` to be set.
+
+10. **`memory_threshold_percent`**:  
+    - The memory threshold (as a percentage) to monitor.  
+    - If exceeded, the crawler will pause or slow down.
+
+11. **`check_interval`**:  
+    - The interval (in seconds) to check system resources.  
+    - Affects how often memory and CPU usage are monitored.
+
+12. **`max_session_permit`**:  
+    - The maximum number of concurrent crawl sessions.  
+    - Helps prevent overwhelming the system.
+
+13. **`display_mode`**:  
+    - The display mode for progress information (`DETAILED`, `BRIEF`, etc.).  
+    - Affects how much information is printed during the crawl.
+
+### Helper Methods
+
+The `clone()` method is particularly useful for creating variations of your crawler configuration:
+
+```python
+# Create a base configuration
+base_config = CrawlerRunConfig(
+    cache_mode=CacheMode.ENABLED,
+    word_count_threshold=200,
+    wait_until="networkidle"
+)
+
+# Create variations for different use cases
+stream_config = base_config.clone(
+    stream=True,  # Enable streaming mode
+    cache_mode=CacheMode.BYPASS
+)
+
+debug_config = base_config.clone(
+    page_timeout=120000,  # Longer timeout for debugging
+    verbose=True
+)
+```
+
+The `clone()` method:
+- Creates a new instance with all the same settings
+- Updates only the specified parameters
+- Leaves the original configuration unchanged
+- Perfect for creating variations without repeating all parameters
+
+---
+
+
+
+
+
+## 3. LLMConfig Essentials
+
+### Key fields to note
+
+1. **`provider`**:  
+- Which LLM provider to use. 
+- Possible values are `"ollama/llama3","groq/llama3-70b-8192","groq/llama3-8b-8192", "openai/gpt-4o-mini" ,"openai/gpt-4o","openai/o1-mini","openai/o1-preview","openai/o3-mini","openai/o3-mini-high","anthropic/claude-3-haiku-20240307","anthropic/claude-3-opus-20240229","anthropic/claude-3-sonnet-20240229","anthropic/claude-3-5-sonnet-20240620","gemini/gemini-pro","gemini/gemini-1.5-pro","gemini/gemini-2.0-flash","gemini/gemini-2.0-flash-exp","gemini/gemini-2.0-flash-lite-preview-02-05","deepseek/deepseek-chat"`
    *(default: `"openai/gpt-4o-mini"`)*
+
+2. **`api_token`**:  
+    - Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`"GEMINI_API_KEY"` will be read from environment variables  
+    - API token of LLM provider 
     eg: `api_token = "gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv"`
+    - Environment variable - use with prefix "env:" 
     eg:`api_token = "env: GROQ_API_KEY"`            
+
+3. **`base_url`**:  
+   - If your provider has a custom endpoint
+
+```python
+llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
+```
+
+## 4. Putting It All Together
+
+In a typical scenario, you define **one** `BrowserConfig` for your crawler session, then create **one or more** `CrawlerRunConfig` & `LLMConfig` depending on each call’s needs:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
+from crawl4ai import JsonCssExtractionStrategy
+
+async def main():
+    # 1) Browser config: headless, bigger viewport, no proxy
+    browser_conf = BrowserConfig(
+        headless=True,
+        viewport_width=1280,
+        viewport_height=720
+    )
+
+    # 2) Example extraction strategy
+    schema = {
+        "name": "Articles",
+        "baseSelector": "div.article",
+        "fields": [
+            {"name": "title", "selector": "h2", "type": "text"},
+            {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"}
+        ]
+    }
+    extraction = JsonCssExtractionStrategy(schema)
+
+    # 3) Example LLM content filtering
+
+    gemini_config = LLMConfig(
+        provider="gemini/gemini-1.5-pro" 
+        api_token = "env:GEMINI_API_TOKEN"
+    )
+
+    # Initialize LLM filter with specific instruction
+    filter = LLMContentFilter(
+        llm_config=gemini_config,  # or your preferred provider
+        instruction="""
+        Focus on extracting the core educational content.
+        Include:
+        - Key concepts and explanations
+        - Important code examples
+        - Essential technical details
+        Exclude:
+        - Navigation elements
+        - Sidebars
+        - Footer content
+        Format the output as clean markdown with proper code blocks and headers.
+        """,
+        chunk_token_threshold=500,  # Adjust based on your needs
+        verbose=True
+    )
+
+    md_generator = DefaultMarkdownGenerator(
+    content_filter=filter,
+    options={"ignore_links": True})
+
+    # 4) Crawler run config: skip cache, use extraction
+    run_conf = CrawlerRunConfig(
+        markdown_generator=md_generator,
+        extraction_strategy=extraction,
+        cache_mode=CacheMode.BYPASS,
+    )
+
+    async with AsyncWebCrawler(config=browser_conf) as crawler:
+        # 4) Execute the crawl
+        result = await crawler.arun(url="https://example.com/news", config=run_conf)
+
+        if result.success:
+            print("Extracted content:", result.extracted_content)
+        else:
+            print("Error:", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+## 5. Next Steps
+
+For a **detailed list** of available parameters (including advanced ones), see:
+
+- [BrowserConfig, CrawlerRunConfig & LLMConfig Reference](../api/parameters.md)  
+
+You can explore topics like:
+
+- **Custom Hooks & Auth** (Inject JavaScript or handle login forms).  
+- **Session Management** (Re-use pages, preserve state across multiple calls).  
+- **Magic Mode** or **Identity-based Crawling** (Fight bot detection by simulating user behavior).  
+- **Advanced Caching** (Fine-tune read/write cache modes).  
+
+---
+
+## 6. Conclusion
+
+**BrowserConfig**, **CrawlerRunConfig** and **LLMConfig** give you straightforward ways to define:
+
+- **Which** browser to launch, how it should run, and any proxy or user agent needs.  
+- **How** each crawl should behave—caching, timeouts, JavaScript code, extraction strategies, etc.
+- **Which** LLM provider to use, api token, temperature and base url for custom endpoints
+
+Use them together for **clear, maintainable** code, and when you need more specialized behavior, check out the advanced parameters in the [reference docs](../api/parameters.md). Happy crawling!
+```
+
+
+## File: docs/md_v2/core/cache-modes.md
+
+```md
+# Crawl4AI Cache System and Migration Guide
+
+## Overview
+Starting from version 0.5.0, Crawl4AI introduces a new caching system that replaces the old boolean flags with a more intuitive `CacheMode` enum. This change simplifies cache control and makes the behavior more predictable.
+
+## Old vs New Approach
+
+### Old Way (Deprecated)
+The old system used multiple boolean flags:
+- `bypass_cache`: Skip cache entirely
+- `disable_cache`: Disable all caching
+- `no_cache_read`: Don't read from cache
+- `no_cache_write`: Don't write to cache
+
+### New Way (Recommended)
+The new system uses a single `CacheMode` enum:
+- `CacheMode.ENABLED`: Normal caching (read/write)
+- `CacheMode.DISABLED`: No caching at all
+- `CacheMode.READ_ONLY`: Only read from cache
+- `CacheMode.WRITE_ONLY`: Only write to cache
+- `CacheMode.BYPASS`: Skip cache for this operation
+
+## Migration Example
+
+### Old Code (Deprecated)
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def use_proxy():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            bypass_cache=True  # Old way
+        )
+        print(len(result.markdown))
+
+async def main():
+    await use_proxy()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### New Code (Recommended)
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def use_proxy():
+    # Use CacheMode in CrawlerRunConfig
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)  
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            config=config  # Pass the configuration object
+        )
+        print(len(result.markdown))
+
+async def main():
+    await use_proxy()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Common Migration Patterns
+
+| Old Flag              | New Mode                       |
+|-----------------------|---------------------------------|
+| `bypass_cache=True`   | `cache_mode=CacheMode.BYPASS`  |
+| `disable_cache=True`  | `cache_mode=CacheMode.DISABLED`|
+| `no_cache_read=True`  | `cache_mode=CacheMode.WRITE_ONLY` |
+| `no_cache_write=True` | `cache_mode=CacheMode.READ_ONLY` |
+```
+
+
+## File: docs/md_v2/core/cli.md
+
+```md
+# Crawl4AI CLI Guide
+
+## Table of Contents
+- [Installation](#installation)
+- [Basic Usage](#basic-usage)
+- [Configuration](#configuration)
+  - [Browser Configuration](#browser-configuration)
+  - [Crawler Configuration](#crawler-configuration)
+  - [Extraction Configuration](#extraction-configuration)
+  - [Content Filtering](#content-filtering)
+- [Advanced Features](#advanced-features)
+  - [LLM Q&A](#llm-qa)
+  - [Structured Data Extraction](#structured-data-extraction)
+  - [Content Filtering](#content-filtering-1)
+- [Output Formats](#output-formats)
+- [Examples](#examples)
+- [Configuration Reference](#configuration-reference)
+- [Best Practices & Tips](#best-practices--tips)
+
+## Basic Usage
+
+The Crawl4AI CLI (`crwl`) provides a simple interface to the Crawl4AI library:
+
+```bash
+# Basic crawling
+crwl https://example.com
+
+# Get markdown output
+crwl https://example.com -o markdown
+
+# Verbose JSON output with cache bypass
+crwl https://example.com -o json -v --bypass-cache
+
+# See usage examples
+crwl --example
+```
+
+## Quick Example of Advanced Usage
+
+If you clone the repository and run the following command, you will receive the content of the page in JSON format according to a JSON-CSS schema:
+
+```bash
+crwl "https://www.infoq.com/ai-ml-data-eng/" -e docs/examples/cli/extract_css.yml -s docs/examples/cli/css_schema.json -o json;
+```
+
+## Configuration
+
+### Browser Configuration
+
+Browser settings can be configured via YAML file or command line parameters:
+
+```yaml
+# browser.yml
+headless: true
+viewport_width: 1280
+user_agent_mode: "random"
+verbose: true
+ignore_https_errors: true
+```
+
+```bash
+# Using config file
+crwl https://example.com -B browser.yml
+
+# Using direct parameters
+crwl https://example.com -b "headless=true,viewport_width=1280,user_agent_mode=random"
+```
+
+### Crawler Configuration
+
+Control crawling behavior:
+
+```yaml
+# crawler.yml
+cache_mode: "bypass"
+wait_until: "networkidle"
+page_timeout: 30000
+delay_before_return_html: 0.5
+word_count_threshold: 100
+scan_full_page: true
+scroll_delay: 0.3
+process_iframes: false
+remove_overlay_elements: true
+magic: true
+verbose: true
+```
+
+```bash
+# Using config file
+crwl https://example.com -C crawler.yml
+
+# Using direct parameters
+crwl https://example.com -c "css_selector=#main,delay_before_return_html=2,scan_full_page=true"
+```
+
+### Extraction Configuration
+
+Two types of extraction are supported:
+
+1. CSS/XPath-based extraction:
+```yaml
+# extract_css.yml
+type: "json-css"
+params:
+  verbose: true
+```
+
+```json
+// css_schema.json
+{
+  "name": "ArticleExtractor",
+  "baseSelector": ".article",
+  "fields": [
+    {
+      "name": "title",
+      "selector": "h1.title",
+      "type": "text"
+    },
+    {
+      "name": "link",
+      "selector": "a.read-more",
+      "type": "attribute",
+      "attribute": "href"
+    }
+  ]
+}
+```
+
+2. LLM-based extraction:
+```yaml
+# extract_llm.yml
+type: "llm"
+provider: "openai/gpt-4"
+instruction: "Extract all articles with their titles and links"
+api_token: "your-token"
+params:
+  temperature: 0.3
+  max_tokens: 1000
+```
+
+```json
+// llm_schema.json
+{
+  "title": "Article",
+  "type": "object",
+  "properties": {
+    "title": {
+      "type": "string",
+      "description": "The title of the article"
+    },
+    "link": {
+      "type": "string",
+      "description": "URL to the full article"
+    }
+  }
+}
+```
+
+## Advanced Features
+
+### LLM Q&A
+
+Ask questions about crawled content:
+
+```bash
+# Simple question
+crwl https://example.com -q "What is the main topic discussed?"
+
+# View content then ask questions
+crwl https://example.com -o markdown  # See content first
+crwl https://example.com -q "Summarize the key points"
+crwl https://example.com -q "What are the conclusions?"
+
+# Combined with advanced crawling
+crwl https://example.com \
+    -B browser.yml \
+    -c "css_selector=article,scan_full_page=true" \
+    -q "What are the pros and cons mentioned?"
+```
+
+First-time setup:
+- Prompts for LLM provider and API token
+- Saves configuration in `~/.crawl4ai/global.yml`
+- Supports various providers (openai/gpt-4, anthropic/claude-3-sonnet, etc.)
+- For case of `ollama` you do not need to provide API token.
+- See [LiteLLM Providers](https://docs.litellm.ai/docs/providers) for full list
+
+### Structured Data Extraction
+
+Extract structured data using CSS selectors:
+
+```bash
+crwl https://example.com \
+    -e extract_css.yml \
+    -s css_schema.json \
+    -o json
+```
+
+Or using LLM-based extraction:
+
+```bash
+crwl https://example.com \
+    -e extract_llm.yml \
+    -s llm_schema.json \
+    -o json
+```
+
+### Content Filtering
+
+Filter content for relevance:
+
+```yaml
+# filter_bm25.yml
+type: "bm25"
+query: "target content"
+threshold: 1.0
+
+# filter_pruning.yml
+type: "pruning"
+query: "focus topic"
+threshold: 0.48
+```
+
+```bash
+crwl https://example.com -f filter_bm25.yml -o markdown-fit
+```
+
+## Output Formats
+
+- `all` - Full crawl result including metadata
+- `json` - Extracted structured data (when using extraction)
+- `markdown` / `md` - Raw markdown output
+- `markdown-fit` / `md-fit` - Filtered markdown for better readability
+
+## Complete Examples
+
+1. Basic Extraction:
+```bash
+crwl https://example.com \
+    -B browser.yml \
+    -C crawler.yml \
+    -o json
+```
+
+2. Structured Data Extraction:
+```bash
+crwl https://example.com \
+    -e extract_css.yml \
+    -s css_schema.json \
+    -o json \
+    -v
+```
+
+3. LLM Extraction with Filtering:
+```bash
+crwl https://example.com \
+    -B browser.yml \
+    -e extract_llm.yml \
+    -s llm_schema.json \
+    -f filter_bm25.yml \
+    -o json
+```
+
+4. Interactive Q&A:
+```bash
+# First crawl and view
+crwl https://example.com -o markdown
+
+# Then ask questions
+crwl https://example.com -q "What are the main points?"
+crwl https://example.com -q "Summarize the conclusions"
+```
+
+## Best Practices & Tips
+
+1. **Configuration Management**:
+   - Keep common configurations in YAML files
+   - Use CLI parameters for quick overrides
+   - Store sensitive data (API tokens) in `~/.crawl4ai/global.yml`
+
+2. **Performance Optimization**:
+   - Use `--bypass-cache` for fresh content
+   - Enable `scan_full_page` for infinite scroll pages
+   - Adjust `delay_before_return_html` for dynamic content
+
+3. **Content Extraction**:
+   - Use CSS extraction for structured content
+   - Use LLM extraction for unstructured content
+   - Combine with filters for focused results
+
+4. **Q&A Workflow**:
+   - View content first with `-o markdown`
+   - Ask specific questions
+   - Use broader context with appropriate selectors
+
+## Recap
+
+The Crawl4AI CLI provides:
+- Flexible configuration via files and parameters
+- Multiple extraction strategies (CSS, XPath, LLM)
+- Content filtering and optimization
+- Interactive Q&A capabilities
+- Various output formats
+
+
+```
+
+
+## File: docs/md_v2/core/content-selection.md
+
+```md
+# Content Selection
+
+Crawl4AI provides multiple ways to **select**, **filter**, and **refine** the content from your crawls. Whether you need to target a specific CSS region, exclude entire tags, filter out external links, or remove certain domains and images, **`CrawlerRunConfig`** offers a wide range of parameters.
+
+Below, we show how to configure these parameters and combine them for precise control.
+
+---
+
+## 1. CSS-Based Selection
+
+There are two ways to select content from a page: using `css_selector` or the more flexible `target_elements`.
+
+### 1.1 Using `css_selector`
+
+A straightforward way to **limit** your crawl results to a certain region of the page is **`css_selector`** in **`CrawlerRunConfig`**:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    config = CrawlerRunConfig(
+        # e.g., first 30 items from Hacker News
+        css_selector=".athing:nth-child(-n+30)"  
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://news.ycombinator.com/newest", 
+            config=config
+        )
+        print("Partial HTML length:", len(result.cleaned_html))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Result**: Only elements matching that selector remain in `result.cleaned_html`.
+
+### 1.2 Using `target_elements`
+
+The `target_elements` parameter provides more flexibility by allowing you to target **multiple elements** for content extraction while preserving the entire page context for other features:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    config = CrawlerRunConfig(
+        # Target article body and sidebar, but not other content
+        target_elements=["article.main-content", "aside.sidebar"]
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com/blog-post", 
+            config=config
+        )
+        print("Markdown focused on target elements")
+        print("Links from entire page still available:", len(result.links.get("internal", [])))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key difference**: With `target_elements`, the markdown generation and structural data extraction focus on those elements, but other page elements (like links, images, and tables) are still extracted from the entire page. This gives you fine-grained control over what appears in your markdown content while preserving full page context for link analysis and media collection.
+
+---
+
+## 2. Content Filtering & Exclusions
+
+### 2.1 Basic Overview
+
+```python
+config = CrawlerRunConfig(
+    # Content thresholds
+    word_count_threshold=10,        # Minimum words per block
+
+    # Tag exclusions
+    excluded_tags=['form', 'header', 'footer', 'nav'],
+
+    # Link filtering
+    exclude_external_links=True,    
+    exclude_social_media_links=True,
+    # Block entire domains
+    exclude_domains=["adtrackers.com", "spammynews.org"],    
+    exclude_social_media_domains=["facebook.com", "twitter.com"],
+
+    # Media filtering
+    exclude_external_images=True
+)
+```
+
+**Explanation**:
+
+- **`word_count_threshold`**: Ignores text blocks under X words. Helps skip trivial blocks like short nav or disclaimers.  
+- **`excluded_tags`**: Removes entire tags (`
    `, `
    `, `
 + + + + `; + tbody.appendChild(row); + } + + this.tableRowsLoaded += batch; + console.log(`📄 Loaded ${batch} more rows. Total: ${this.tableRowsLoaded}`); + } + + // Load initial data + loadInitialData() { + // Load initial products + this.loadMoreProducts(); + + // Load initial table rows + this.loadMoreTableRows(); + } + + // Load content when navigating to sections + loadSectionContent(sectionId) { + switch(sectionId) { + case 'catalog': + // Ensure products are loaded in catalog + if (this.productsLoaded === 0) { + this.loadMoreProducts(); + } + break; + case 'data-tables': + // Ensure table rows are loaded + if (this.tableRowsLoaded === 0) { + this.loadMoreTableRows(); + } + break; + case 'forms': + // Forms are already set up + break; + case 'tabs': + // Tabs content is static + break; + } + } + + // Inspector Mode + setupInspector() { + const inspectorBtn = document.getElementById('inspector-btn'); + + // Create tooltip element + this.tooltip = document.createElement('div'); + this.tooltip.className = 'inspector-tooltip'; + this.tooltip.style.cssText = ` + position: fixed; + background: rgba(0, 0, 0, 0.9); + color: white; + padding: 8px 12px; + border-radius: 4px; + font-size: 12px; + font-family: monospace; + pointer-events: none; + z-index: 10000; + display: none; + max-width: 300px; + `; + document.body.appendChild(this.tooltip); + + inspectorBtn.addEventListener('click', () => { + this.toggleInspector(); + }); + + // Add mouse event listeners + document.addEventListener('mousemove', this.handleMouseMove.bind(this)); + document.addEventListener('mouseout', this.handleMouseOut.bind(this)); + } + + toggleInspector() { + this.inspectorMode = !this.inspectorMode; + const inspectorBtn = document.getElementById('inspector-btn'); + + if (this.inspectorMode) { + inspectorBtn.classList.add('active'); + inspectorBtn.style.background = '#0fbbaa'; + document.body.style.cursor = 'crosshair'; + } else { + inspectorBtn.classList.remove('active'); + inspectorBtn.style.background = ''; + document.body.style.cursor = ''; + this.tooltip.style.display = 'none'; + this.removeHighlight(); + } + } + + handleMouseMove(e) { + if (!this.inspectorMode) return; + + const element = e.target; + if (element === this.tooltip) return; + + // Highlight element + this.highlightElement(element); + + // Show tooltip with element info + const info = this.getElementInfo(element); + this.tooltip.innerHTML = info; + this.tooltip.style.display = 'block'; + + // Position tooltip + const x = e.clientX + 15; + const y = e.clientY + 15; + + // Adjust position if tooltip would go off screen + const rect = this.tooltip.getBoundingClientRect(); + const adjustedX = x + rect.width > window.innerWidth ? x - rect.width - 30 : x; + const adjustedY = y + rect.height > window.innerHeight ? y - rect.height - 30 : y; + + this.tooltip.style.left = adjustedX + 'px'; + this.tooltip.style.top = adjustedY + 'px'; + } + + handleMouseOut(e) { + if (!this.inspectorMode) return; + if (e.target === document.body) { + this.removeHighlight(); + this.tooltip.style.display = 'none'; + } + } + + highlightElement(element) { + this.removeHighlight(); + element.style.outline = '2px solid #0fbbaa'; + element.style.outlineOffset = '1px'; + element.setAttribute('data-inspector-highlighted', 'true'); + } + + removeHighlight() { + const highlighted = document.querySelector('[data-inspector-highlighted]'); + if (highlighted) { + highlighted.style.outline = ''; + highlighted.style.outlineOffset = ''; + highlighted.removeAttribute('data-inspector-highlighted'); + } + } + + getElementInfo(element) { + const tagName = element.tagName.toLowerCase(); + const id = element.id ? `#${element.id}` : ''; + const classes = element.className ? + `.${element.className.split(' ').filter(c => c).join('.')}` : ''; + + let selector = tagName; + if (id) { + selector = id; + } else if (classes) { + selector = `${tagName}${classes}`; + } + + // Build info HTML + let info = `${selector}`; + + // Add additional attributes + const attrs = []; + if (element.name) attrs.push(`name="${element.name}"`); + if (element.type) attrs.push(`type="${element.type}"`); + if (element.href) attrs.push(`href="${element.href}"`); + if (element.value && element.tagName === 'INPUT') attrs.push(`value="${element.value}"`); + + if (attrs.length > 0) { + info += `
${attrs.join(' ')}`; + } + + return info; + } +} + +// Initialize app when DOM is ready +document.addEventListener('DOMContentLoaded', () => { + window.playgroundApp = new PlaygroundApp(); + console.log('🎮 Playground app initialized!'); +}); \ No newline at end of file diff --git a/docs/examples/c4a_script/tutorial/playground/index.html b/docs/examples/c4a_script/tutorial/playground/index.html new file mode 100644 index 0000000..6ae8efd --- /dev/null +++ b/docs/examples/c4a_script/tutorial/playground/index.html @@ -0,0 +1,328 @@ + + + + + + C4A-Script Playground + + + + + + + + + + +
+ +
+ + + +
+
+ + +
+ +
+

Welcome to C4A-Script Playground

+

This is an interactive demo for testing C4A-Script commands. Each section contains different challenges for web automation.

+ + + +
+
+

🔐 Authentication

+

Test login forms and user sessions

+
+
+

📜 Dynamic Content

+

Infinite scroll and pagination

+
+
+

📝 Forms

+

Complex form interactions

+
+
+

📊 Data Tables

+

Sortable and filterable data

+
+
+
+ + + + + +
+

Product Catalog

+ +
+ + +
+ + +
+ + + +
+
+ +
+ + +
+ +
+ + + +
+
+
+ + +
+

Form Examples

+ + +
+

Contact Form

+
+
+ + +
+
+ + +
+
+ + +
+ +
+ + +
+ +
+ +
+ + +
+

Multi-step Survey

+
+
+
+
+ +
+

Step 1: Basic Information

+
+ + +
+
+ + +
+ +
+ + + + + + + + +
+
+ + +
+

Tabs Demo

+
+
+ + + +
+
+
+

Product Description

+

This is a detailed description of the product...

+
+

Lorem ipsum dolor sit amet, consectetur adipiscing elit...

+ + +
+
+ +
User ${id}user${id}@example.com${new Date().toLocaleDateString()}
+ + + +
ModelXYZ-2000
Weight2.5 kg
Dimensions30 x 20 x 10 cm
+ + + + + + +
+

Data Tables

+
+ + +
+ + + + + + + + + + + + +
Name ↕Email ↕Date ↕Actions
+ +
+ + + + + \ No newline at end of file diff --git a/docs/examples/c4a_script/tutorial/playground/styles.css b/docs/examples/c4a_script/tutorial/playground/styles.css new file mode 100644 index 0000000..bfb2c07 --- /dev/null +++ b/docs/examples/c4a_script/tutorial/playground/styles.css @@ -0,0 +1,627 @@ +/* Playground Styles - Modern Web App Theme */ +:root { + --primary-color: #0fbbaa; + --secondary-color: #3f3f44; + --background-color: #ffffff; + --text-color: #333333; + --border-color: #e0e0e0; + --error-color: #ff3c74; + --success-color: #0fbbaa; + --warning-color: #ffa500; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + padding: 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + font-size: 16px; + line-height: 1.6; + color: var(--text-color); + background-color: var(--background-color); +} + +/* Cookie Banner */ +.cookie-banner { + position: fixed; + bottom: 0; + left: 0; + right: 0; + background-color: #2c3e50; + color: white; + padding: 1rem; + z-index: 1000; + box-shadow: 0 -2px 10px rgba(0,0,0,0.1); +} + +.cookie-content { + max-width: 1200px; + margin: 0 auto; + display: flex; + align-items: center; + justify-content: space-between; + flex-wrap: wrap; + gap: 1rem; +} + +.cookie-actions { + display: flex; + gap: 0.5rem; +} + +/* Header */ +.site-header { + background-color: #fff; + border-bottom: 1px solid var(--border-color); + padding: 1rem 2rem; + position: sticky; + top: 0; + z-index: 100; + display: flex; + justify-content: space-between; + align-items: center; +} + +.nav-menu { + display: flex; + gap: 2rem; + align-items: center; +} + +.nav-link { + text-decoration: none; + color: var(--text-color); + font-weight: 500; + transition: color 0.2s; +} + +.nav-link:hover, +.nav-link.active { + color: var(--primary-color); +} + +/* Dropdown */ +.dropdown { + position: relative; +} + +.dropdown-content { + display: none; + position: absolute; + background-color: white; + min-width: 160px; + box-shadow: 0 8px 16px rgba(0,0,0,0.1); + z-index: 1; + border-radius: 4px; + top: 100%; + margin-top: 0.5rem; +} + +.dropdown:hover .dropdown-content { + display: block; +} + +.dropdown-content a { + color: var(--text-color); + padding: 0.75rem 1rem; + text-decoration: none; + display: block; +} + +.dropdown-content a:hover { + background-color: #f5f5f5; +} + +/* Auth Section */ +.auth-section { + display: flex; + align-items: center; + gap: 1rem; +} + +.user-info { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.user-avatar { + font-size: 1.5rem; +} + +/* Main Content */ +.main-content { + padding: 2rem; + max-width: 1200px; + margin: 0 auto; +} + +.section { + display: none; +} + +.section.active { + display: block; +} + +/* Buttons */ +.btn { + background-color: var(--primary-color); + color: white; + border: none; + padding: 0.5rem 1rem; + border-radius: 4px; + cursor: pointer; + font-size: 1rem; + font-weight: 500; + transition: all 0.2s; +} + +.btn:hover { + background-color: #0aa599; + transform: translateY(-1px); +} + +.btn-sm { + padding: 0.25rem 0.75rem; + font-size: 0.875rem; +} + +.btn-secondary { + background-color: var(--secondary-color); +} + +.btn-secondary:hover { + background-color: #333; +} + +.btn-primary { + background-color: var(--primary-color); +} + +/* Feature Grid */ +.feature-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1.5rem; + margin-top: 2rem; +} + +.feature-card { + background-color: #f8f9fa; + padding: 1.5rem; + border-radius: 8px; + text-align: center; + transition: transform 0.2s; +} + +.feature-card:hover { + transform: translateY(-4px); + box-shadow: 0 4px 12px rgba(0,0,0,0.1); +} + +.feature-card h3 { + margin-top: 0; +} + +/* Modal */ +.modal { + position: fixed; + z-index: 1000; + left: 0; + top: 0; + width: 100%; + height: 100%; + background-color: rgba(0,0,0,0.5); + display: flex; + align-items: center; + justify-content: center; +} + +.modal-content { + background-color: white; + padding: 2rem; + border-radius: 8px; + max-width: 500px; + width: 90%; + position: relative; + animation: modalFadeIn 0.3s; +} + +@keyframes modalFadeIn { + from { opacity: 0; transform: translateY(-20px); } + to { opacity: 1; transform: translateY(0); } +} + +.close { + position: absolute; + right: 1rem; + top: 1rem; + font-size: 1.5rem; + cursor: pointer; + color: #999; +} + +.close:hover { + color: #333; +} + +/* Forms */ +.form-group { + margin-bottom: 1rem; +} + +.form-group label { + display: block; + margin-bottom: 0.5rem; + font-weight: 500; +} + +.input { + width: 100%; + padding: 0.5rem; + border: 1px solid var(--border-color); + border-radius: 4px; + font-size: 1rem; +} + +.input:focus { + outline: none; + border-color: var(--primary-color); +} + +.checkbox-label { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.form-message { + margin-top: 1rem; + padding: 0.75rem; + border-radius: 4px; + display: none; +} + +.form-message.error { + background-color: #ffe6e6; + color: var(--error-color); + display: block; +} + +.form-message.success { + background-color: #e6fff6; + color: var(--success-color); + display: block; +} + +/* Product Catalog */ +.view-toggle { + margin-bottom: 1rem; +} + +.catalog-layout { + display: grid; + grid-template-columns: 250px 1fr; + gap: 2rem; +} + +.filters-sidebar { + background-color: #f8f9fa; + padding: 1rem; + border-radius: 8px; +} + +.filter-group { + margin-bottom: 1.5rem; +} + +.collapsible { + cursor: pointer; + display: flex; + justify-content: space-between; + align-items: center; +} + +.filter-content { + margin-top: 0.5rem; +} + +.filter-content label { + display: block; + margin-bottom: 0.5rem; +} + +/* Product Grid */ +.product-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 1.5rem; +} + +.product-card { + background-color: white; + border: 1px solid var(--border-color); + border-radius: 8px; + padding: 1rem; + text-align: center; + transition: transform 0.2s; +} + +.product-card:hover { + transform: translateY(-4px); + box-shadow: 0 4px 12px rgba(0,0,0,0.1); +} + +.product-image { + width: 100%; + height: 150px; + background-color: #f0f0f0; + margin-bottom: 1rem; + display: flex; + align-items: center; + justify-content: center; + font-size: 3rem; +} + +.product-name { + font-weight: 600; + margin-bottom: 0.5rem; +} + +.product-price { + color: var(--primary-color); + font-size: 1.2rem; + font-weight: 700; +} + +/* Loading Indicator */ +.loading-indicator { + text-align: center; + padding: 2rem; +} + +.spinner { + border: 3px solid #f3f3f3; + border-top: 3px solid var(--primary-color); + border-radius: 50%; + width: 40px; + height: 40px; + animation: spin 1s linear infinite; + margin: 0 auto; +} + +@keyframes spin { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } +} + +/* Pagination */ +.pagination { + display: flex; + gap: 0.5rem; + justify-content: center; + margin-top: 2rem; +} + +.page-btn { + padding: 0.5rem 1rem; + border: 1px solid var(--border-color); + background-color: white; + cursor: pointer; + border-radius: 4px; +} + +.page-btn:hover, +.page-btn.active { + background-color: var(--primary-color); + color: white; +} + +/* Multi-step Form */ +.progress-bar { + width: 100%; + height: 8px; + background-color: #e0e0e0; + border-radius: 4px; + margin-bottom: 2rem; +} + +.progress-fill { + height: 100%; + background-color: var(--primary-color); + border-radius: 4px; + transition: width 0.3s; +} + +.form-step { + display: none; +} + +.form-step.active { + display: block; +} + +/* Tabs */ +.tabs-container { + margin-top: 2rem; +} + +.tabs-header { + display: flex; + border-bottom: 2px solid var(--border-color); +} + +.tab-btn { + background: none; + border: none; + padding: 1rem 2rem; + cursor: pointer; + font-size: 1rem; + font-weight: 500; + color: var(--text-color); + position: relative; +} + +.tab-btn:hover { + color: var(--primary-color); +} + +.tab-btn.active { + color: var(--primary-color); +} + +.tab-btn.active::after { + content: ''; + position: absolute; + bottom: -2px; + left: 0; + right: 0; + height: 2px; + background-color: var(--primary-color); +} + +.tabs-content { + padding: 2rem 0; +} + +.tab-pane { + display: none; +} + +.tab-pane.active { + display: block; +} + +/* Expandable Text */ +.expandable-text { + margin-top: 1rem; +} + +.text-preview { + margin-bottom: 0.5rem; +} + +.show-more { + margin-top: 0.5rem; +} + +/* Comments Section */ +.comments-section { + margin-top: 1rem; +} + +.comment { + background-color: #f8f9fa; + padding: 1rem; + border-radius: 4px; + margin-bottom: 1rem; +} + +.comment-author { + font-weight: 600; + margin-bottom: 0.5rem; +} + +/* Data Table */ +.table-controls { + display: flex; + gap: 1rem; + margin-bottom: 1rem; +} + +.search-input { + flex: 1; + max-width: 300px; +} + +.data-table { + width: 100%; + border-collapse: collapse; + background-color: white; +} + +.data-table th, +.data-table td { + padding: 0.75rem; + text-align: left; + border-bottom: 1px solid var(--border-color); +} + +.data-table th { + background-color: #f8f9fa; + font-weight: 600; +} + +.sortable { + cursor: pointer; +} + +.sortable:hover { + color: var(--primary-color); +} + +/* Form Cards */ +.form-card { + background-color: white; + border: 1px solid var(--border-color); + border-radius: 8px; + padding: 2rem; + margin-bottom: 2rem; +} + +.form-card h2 { + margin-top: 0; +} + +/* Success Message */ +.success-message { + background-color: #e6fff6; + color: var(--success-color); + padding: 1rem; + border-radius: 4px; + text-align: center; + font-weight: 500; +} + +/* Load More Button */ +.load-more, +.load-more-rows { + display: block; + margin: 2rem auto; +} + +/* Responsive */ +@media (max-width: 768px) { + .catalog-layout { + grid-template-columns: 1fr; + } + + .feature-grid { + grid-template-columns: 1fr; + } + + .nav-menu { + flex-wrap: wrap; + gap: 1rem; + } + + .cookie-content { + flex-direction: column; + text-align: center; + } +} + +/* Inspector Mode */ +#inspector-btn.active { + background: var(--primary-color) !important; + color: var(--bg-primary) !important; +} + +.inspector-tooltip { + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3); + border: 1px solid rgba(255, 255, 255, 0.1); +} diff --git a/docs/examples/c4a_script/tutorial/requirements.txt b/docs/examples/c4a_script/tutorial/requirements.txt new file mode 100644 index 0000000..2a6bcb3 --- /dev/null +++ b/docs/examples/c4a_script/tutorial/requirements.txt @@ -0,0 +1,2 @@ +flask>=2.3.0 +flask-cors>=4.0.0 \ No newline at end of file diff --git a/docs/examples/c4a_script/tutorial/server.py b/docs/examples/c4a_script/tutorial/server.py new file mode 100644 index 0000000..2537e4c --- /dev/null +++ b/docs/examples/c4a_script/tutorial/server.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +""" +C4A-Script Tutorial Server +Serves the tutorial app and provides C4A compilation API +""" + +import sys +import os +from pathlib import Path +from flask import Flask, render_template_string, request, jsonify, send_from_directory +from flask_cors import CORS + +# Add parent directories to path to import crawl4ai +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent)) + +try: + from crawl4ai.script import compile as c4a_compile + C4A_AVAILABLE = True +except ImportError: + print("⚠️ C4A compiler not available. Using mock compiler.") + C4A_AVAILABLE = False + +app = Flask(__name__) +CORS(app) + +# Serve static files +@app.route('/') +def index(): + return send_from_directory('.', 'index.html') + +@app.route('/assets/') +def serve_assets(path): + return send_from_directory('assets', path) + +@app.route('/playground/') +def playground(): + return send_from_directory('playground', 'index.html') + +@app.route('/playground/') +def serve_playground(path): + return send_from_directory('playground', path) + +# API endpoint for C4A compilation +@app.route('/api/compile', methods=['POST']) +def compile_endpoint(): + try: + data = request.get_json() + script = data.get('script', '') + + if not script: + return jsonify({ + 'success': False, + 'error': { + 'line': 1, + 'column': 1, + 'message': 'No script provided', + 'suggestion': 'Write some C4A commands' + } + }) + + if C4A_AVAILABLE: + # Use real C4A compiler + result = c4a_compile(script) + + if result.success: + return jsonify({ + 'success': True, + 'jsCode': result.js_code, + 'metadata': { + 'lineCount': len(result.js_code), + 'sourceLines': len(script.split('\n')) + } + }) + else: + error = result.first_error + return jsonify({ + 'success': False, + 'error': { + 'line': error.line, + 'column': error.column, + 'message': error.message, + 'suggestion': error.suggestions[0].message if error.suggestions else None, + 'code': error.code, + 'sourceLine': error.source_line + } + }) + else: + # Use mock compiler for demo + result = mock_compile(script) + return jsonify(result) + + except Exception as e: + return jsonify({ + 'success': False, + 'error': { + 'line': 1, + 'column': 1, + 'message': f'Server error: {str(e)}', + 'suggestion': 'Check server logs' + } + }), 500 + +def mock_compile(script): + """Simple mock compiler for demo when C4A is not available""" + lines = [line for line in script.split('\n') if line.strip() and not line.strip().startswith('#')] + js_code = [] + + for i, line in enumerate(lines): + line = line.strip() + + try: + if line.startswith('GO '): + url = line[3:].strip() + # Handle relative URLs + if not url.startswith(('http://', 'https://')): + url = '/' + url.lstrip('/') + js_code.append(f"await page.goto('{url}');") + + elif line.startswith('WAIT '): + parts = line[5:].strip().split(' ') + if parts[0].startswith('`'): + selector = parts[0].strip('`') + timeout = parts[1] if len(parts) > 1 else '5' + js_code.append(f"await page.waitForSelector('{selector}', {{ timeout: {timeout}000 }});") + else: + seconds = parts[0] + js_code.append(f"await page.waitForTimeout({seconds}000);") + + elif line.startswith('CLICK '): + selector = line[6:].strip().strip('`') + js_code.append(f"await page.click('{selector}');") + + elif line.startswith('TYPE '): + text = line[5:].strip().strip('"') + js_code.append(f"await page.keyboard.type('{text}');") + + elif line.startswith('SCROLL '): + parts = line[7:].strip().split(' ') + direction = parts[0] + amount = parts[1] if len(parts) > 1 else '500' + if direction == 'DOWN': + js_code.append(f"await page.evaluate(() => window.scrollBy(0, {amount}));") + elif direction == 'UP': + js_code.append(f"await page.evaluate(() => window.scrollBy(0, -{amount}));") + + elif line.startswith('IF '): + if 'THEN' not in line: + return { + 'success': False, + 'error': { + 'line': i + 1, + 'column': len(line), + 'message': "Missing 'THEN' keyword after IF condition", + 'suggestion': "Add 'THEN' after the condition", + 'sourceLine': line + } + } + + condition = line[3:line.index('THEN')].strip() + action = line[line.index('THEN') + 4:].strip() + + if 'EXISTS' in condition: + selector_match = condition.split('`') + if len(selector_match) >= 2: + selector = selector_match[1] + action_selector = action.split('`')[1] if '`' in action else '' + js_code.append( + f"if (await page.$$('{selector}').length > 0) {{ " + f"await page.click('{action_selector}'); }}" + ) + + elif line.startswith('PRESS '): + key = line[6:].strip() + js_code.append(f"await page.keyboard.press('{key}');") + + else: + # Unknown command + return { + 'success': False, + 'error': { + 'line': i + 1, + 'column': 1, + 'message': f"Unknown command: {line.split()[0]}", + 'suggestion': "Check command syntax", + 'sourceLine': line + } + } + + except Exception as e: + return { + 'success': False, + 'error': { + 'line': i + 1, + 'column': 1, + 'message': f"Failed to parse: {str(e)}", + 'suggestion': "Check syntax", + 'sourceLine': line + } + } + + return { + 'success': True, + 'jsCode': js_code, + 'metadata': { + 'lineCount': len(js_code), + 'sourceLines': len(lines) + } + } + +# Example scripts endpoint +@app.route('/api/examples') +def get_examples(): + examples = [ + { + 'id': 'cookie-banner', + 'name': 'Handle Cookie Banner', + 'description': 'Accept cookies and close newsletter popup', + 'script': '''# Handle cookie banner and newsletter +GO http://127.0.0.1:8000/playground/ +WAIT `body` 2 +IF (EXISTS `.cookie-banner`) THEN CLICK `.accept` +IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`''' + }, + { + 'id': 'login', + 'name': 'Login Flow', + 'description': 'Complete login with credentials', + 'script': '''# Login to the site +CLICK `#login-btn` +WAIT `.login-form` 2 +CLICK `#email` +TYPE "demo@example.com" +CLICK `#password` +TYPE "demo123" +IF (EXISTS `#remember-me`) THEN CLICK `#remember-me` +CLICK `button[type="submit"]` +WAIT `.welcome-message` 5''' + }, + { + 'id': 'infinite-scroll', + 'name': 'Infinite Scroll', + 'description': 'Load products with scrolling', + 'script': '''# Navigate to catalog and scroll +CLICK `#catalog-link` +WAIT `.product-grid` 3 + +# Scroll multiple times to load products +SCROLL DOWN 1000 +WAIT 1 +SCROLL DOWN 1000 +WAIT 1 +SCROLL DOWN 1000''' + }, + { + 'id': 'form-wizard', + 'name': 'Multi-step Form', + 'description': 'Complete a multi-step survey', + 'script': '''# Navigate to forms +CLICK `a[href="#forms"]` +WAIT `#survey-form` 2 + +# Step 1: Basic info +CLICK `#full-name` +TYPE "John Doe" +CLICK `#survey-email` +TYPE "john@example.com" +CLICK `.next-step` +WAIT 1 + +# Step 2: Preferences +CLICK `#interests` +CLICK `option[value="tech"]` +CLICK `option[value="music"]` +CLICK `.next-step` +WAIT 1 + +# Step 3: Submit +CLICK `#submit-survey` +WAIT `.success-message` 5''' + } + ] + + return jsonify(examples) + +if __name__ == '__main__': + port = int(os.environ.get('PORT', 8000)) + print(f""" +╔══════════════════════════════════════════════════════════╗ +║ C4A-Script Interactive Tutorial Server ║ +╠══════════════════════════════════════════════════════════╣ +║ ║ +║ Server running at: http://localhost:{port:<6} ║ +║ ║ +║ Features: ║ +║ • C4A-Script compilation API ║ +║ • Interactive playground ║ +║ • Real-time execution visualization ║ +║ ║ +║ C4A Compiler: {'✓ Available' if C4A_AVAILABLE else '✗ Using mock compiler':<30} ║ +║ ║ +╚══════════════════════════════════════════════════════════╝ + """) + + app.run(host='0.0.0.0', port=port, debug=True) \ No newline at end of file diff --git a/docs/examples/c4a_script/tutorial/test_blockly.html b/docs/examples/c4a_script/tutorial/test_blockly.html new file mode 100644 index 0000000..0bf9f06 --- /dev/null +++ b/docs/examples/c4a_script/tutorial/test_blockly.html @@ -0,0 +1,69 @@ + + + + + + Blockly Test + + + +

C4A-Script Blockly Test

+
+
+

Generated C4A-Script:

+ 

+    

+    
+    
+    
+    
+
+
\ No newline at end of file
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_aws_waf.py b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_aws_waf.py
new file mode 100644
index 0000000..351c048
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_aws_waf.py
@@ -0,0 +1,62 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/awsWaf/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"              # your api key of capsolver
+site_url = "https://nft.porsche.com/onboarding@6"  # page url of your target site
+cookie_domain = ".nft.porsche.com"                 # the domain name to which you want to apply the cookie
+captcha_type = "AntiAwsWafTaskProxyLess"           # type of your target captcha
+capsolver.api_key = api_key
+
+
+async def main():
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # get aws waf cookie using capsolver sdk
+        solution = capsolver.solve({
+            "type": captcha_type,
+            "websiteURL": site_url,
+        })
+        cookie = solution["cookie"]
+        print("aws waf cookie:", cookie)
+
+        js_code = """
+            document.cookie = \'aws-waf-token=""" + cookie + """;domain=""" + cookie_domain + """;path=/\';
+            location.reload();
+        """
+
+        wait_condition = """() => {
+            return document.title === \'Join Porsche’s journey into Web3\';
+        }"""
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test",
+            js_code=js_code,
+            js_only=True,
+            wait_for=f"js:{wait_condition}"
+        )
+
+        result_next = await crawler.arun(
+            url=site_url,
+            config=run_config,
+        )
+        print(result_next.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_challenge.py b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_challenge.py
new file mode 100644
index 0000000..39ef3e7
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_challenge.py
@@ -0,0 +1,60 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/cloudflare_challenge/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"          # your api key of capsolver
+site_url = "https://gitlab.com/users/sign_in"  # page url of your target site
+captcha_type = "AntiCloudflareTask"            # type of your target captcha
+# your http proxy to solve cloudflare challenge
+proxy_server = "proxy.example.com:8080"
+proxy_username = "myuser"
+proxy_password = "mypass"
+capsolver.api_key = api_key
+
+
+async def main():
+    # get challenge cookie using capsolver sdk
+    solution = capsolver.solve({
+        "type": captcha_type,
+        "websiteURL": site_url,
+        "proxy": f"{proxy_server}:{proxy_username}:{proxy_password}",
+    })
+    cookies = solution["cookies"]
+    user_agent = solution["userAgent"]
+    print("challenge cookies:", cookies)
+
+    cookies_list = []
+    for name, value in cookies.items():
+        cookies_list.append({
+            "name": name,
+            "value": value,
+            "url": site_url,
+        })
+
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+        user_agent=user_agent,
+        cookies=cookies_list,
+        proxy_config={
+            "server": f"http://{proxy_server}",
+            "username": proxy_username,
+            "password": proxy_password,
+        },
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+        print(result.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_turnstile.py b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_turnstile.py
new file mode 100644
index 0000000..b160306
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_turnstile.py
@@ -0,0 +1,64 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/cloudflare_turnstile/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"                       # your api key of capsolver
+site_key = "0x4AAAAAAAGlwMzq_9z6S9Mh"                       # site key of your target site
+site_url = "https://clifford.io/demo/cloudflare-turnstile"  # page url of your target site
+captcha_type = "AntiTurnstileTaskProxyLess"                 # type of your target captcha
+capsolver.api_key = api_key
+
+
+async def main():
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # get turnstile token using capsolver sdk
+        solution = capsolver.solve({
+            "type": captcha_type,
+            "websiteURL": site_url,
+            "websiteKey": site_key,
+        })
+        token = solution["token"]
+        print("turnstile token:", token)
+
+        js_code = """
+            document.querySelector(\'input[name="cf-turnstile-response"]\').value = \'"""+token+"""\';
+            document.querySelector(\'button[type="submit"]\').click();
+        """
+
+        wait_condition = """() => {
+            const items = document.querySelectorAll(\'h1\');
+            return items.length === 0;
+        }"""
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test",
+            js_code=js_code,
+            js_only=True,
+            wait_for=f"js:{wait_condition}"
+        )
+
+        result_next = await crawler.arun(
+            url=site_url,
+            config=run_config,
+        )
+        print(result_next.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v2.py b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v2.py
new file mode 100644
index 0000000..c9302c4
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v2.py
@@ -0,0 +1,67 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/ReCaptchaV2/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"                                      # your api key of capsolver
+site_key = "6LfW6wATAAAAAHLqO2pb8bDBahxlMxNdo9g947u9"                      # site key of your target site
+site_url = "https://recaptcha-demo.appspot.com/recaptcha-v2-checkbox.php"  # page url of your target site
+captcha_type = "ReCaptchaV2TaskProxyLess"                                  # type of your target captcha
+capsolver.api_key = api_key
+
+
+async def main():
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # get recaptcha token using capsolver sdk
+        solution = capsolver.solve({
+            "type": captcha_type,
+            "websiteURL": site_url,
+            "websiteKey": site_key,
+        })
+        token = solution["gRecaptchaResponse"]
+        print("recaptcha token:", token)
+
+        js_code = """
+            const textarea = document.getElementById(\'g-recaptcha-response\');
+            if (textarea) {
+                textarea.value = \"""" + token + """\";
+                document.querySelector(\'button.form-field[type="submit"]\').click();
+            }
+        """
+
+        wait_condition = """() => {
+            const items = document.querySelectorAll(\'h2\');
+            return items.length > 1;
+        }"""
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test",
+            js_code=js_code,
+            js_only=True,
+            wait_for=f"js:{wait_condition}"
+        )
+
+        result_next = await crawler.arun(
+            url=site_url,
+            config=run_config,
+        )
+        print(result_next.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v3.py b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v3.py
new file mode 100644
index 0000000..401f0c8
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v3.py
@@ -0,0 +1,75 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/ReCaptchaV3/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"                                            # your api key of capsolver
+site_key = "6LdKlZEpAAAAAAOQjzC2v_d36tWxCl6dWsozdSy9"                            # site key of your target site
+site_url = "https://recaptcha-demo.appspot.com/recaptcha-v3-request-scores.php"  # page url of your target site
+page_action = "examples/v3scores"                                                # page action of your target site
+captcha_type = "ReCaptchaV3TaskProxyLess"                                        # type of your target captcha
+capsolver.api_key = api_key
+
+
+async def main():
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+    )
+
+    # get recaptcha token using capsolver sdk
+    solution = capsolver.solve({
+        "type": captcha_type,
+        "websiteURL": site_url,
+        "websiteKey": site_key,
+        "pageAction": page_action,
+    })
+    token = solution["gRecaptchaResponse"]
+    print("recaptcha token:", token)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        js_code = """
+            const originalFetch = window.fetch;
+
+            window.fetch = function(...args) {
+              if (typeof args[0] === 'string' && args[0].includes('/recaptcha-v3-verify.php')) {
+                const url = new URL(args[0], window.location.origin);
+                url.searchParams.set('action', '""" + token + """');
+                args[0] = url.toString();
+                document.querySelector('.token').innerHTML = "fetch('/recaptcha-v3-verify.php?action=examples/v3scores&token=""" + token + """')";
+                console.log('Fetch URL hooked:', args[0]);
+              }
+              return originalFetch.apply(this, args);
+            };
+        """
+
+        wait_condition = """() => {
+            return document.querySelector('.step3:not(.hidden)');
+        }"""
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test",
+            js_code=js_code,
+            js_only=True,
+            wait_for=f"js:{wait_condition}"
+        )
+
+        result_next = await crawler.arun(
+            url=site_url,
+            config=run_config,
+        )
+        print(result_next.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_aws_waf.py b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_aws_waf.py
new file mode 100644
index 0000000..d123846
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_aws_waf.py
@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://nft.porsche.com/onboarding@6",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_challenge.py b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_challenge.py
new file mode 100644
index 0000000..3f0e967
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_challenge.py
@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://gitlab.com/users/sign_in",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_turnstile.py b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_turnstile.py
new file mode 100644
index 0000000..ca074f5
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_turnstile.py
@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://clifford.io/demo/cloudflare-turnstile",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v2.py b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v2.py
new file mode 100644
index 0000000..bdcd0f9
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v2.py
@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://recaptcha-demo.appspot.com/recaptcha-v2-checkbox.php",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v3.py b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v3.py
new file mode 100644
index 0000000..899b83b
--- /dev/null
+++ b/docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v3.py
@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://recaptcha-demo.appspot.com/recaptcha-v3-request-scores.php",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/chainlit.md b/docs/examples/chainlit.md
new file mode 100644
index 0000000..3b34b02
--- /dev/null
+++ b/docs/examples/chainlit.md
@@ -0,0 +1,3 @@
+# Welcome to Crawl4AI! 🚀🤖
+
+Hi there, Developer! 👋 Here is an example of a research pipeline, where you can share a URL in your conversation with any LLM, and then the context of crawled pages will be used as the context.
\ No newline at end of file
diff --git a/docs/examples/cli/browser.yml b/docs/examples/cli/browser.yml
new file mode 100644
index 0000000..dd6caf6
--- /dev/null
+++ b/docs/examples/cli/browser.yml
@@ -0,0 +1,13 @@
+browser_type: "chromium"
+headless: true
+viewport_width: 1280
+viewport_height: 800
+user_agent_mode: "random"
+verbose: true
+text_mode: false
+light_mode: false
+ignore_https_errors: true
+java_script_enabled: true
+extra_args:
+  - "--disable-gpu"
+  - "--no-sandbox"
\ No newline at end of file
diff --git a/docs/examples/cli/crawler.yml b/docs/examples/cli/crawler.yml
new file mode 100644
index 0000000..61bd667
--- /dev/null
+++ b/docs/examples/cli/crawler.yml
@@ -0,0 +1,13 @@
+cache_mode: "bypass"
+wait_until: "networkidle"
+page_timeout: 30000
+delay_before_return_html: 0.5
+word_count_threshold: 100
+scan_full_page: true
+scroll_delay: 0.3
+process_iframes: false
+remove_overlay_elements: true
+magic: true
+verbose: true
+exclude_external_links: true
+exclude_social_media_links: true
\ No newline at end of file
diff --git a/docs/examples/cli/css_schema.json b/docs/examples/cli/css_schema.json
new file mode 100644
index 0000000..935efeb
--- /dev/null
+++ b/docs/examples/cli/css_schema.json
@@ -0,0 +1,27 @@
+{
+  "name": "ArticleExtractor",
+  "baseSelector": ".cards[data-tax=news] .card__data",
+  "fields": [
+    {
+      "name": "title",
+      "selector": "h4.card__title",
+      "type": "text"
+    },
+    {
+      "name": "link",
+      "selector": "h4.card__title a", 
+      "type": "attribute",
+      "attribute": "href"
+    },
+    {
+      "name": "details",
+      "selector": ".card__details",
+      "type": "text"
+    },
+    {
+      "name": "topics",
+      "selector": ".card__topics.topics",
+      "type": "text"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/docs/examples/cli/extract.yml b/docs/examples/cli/extract.yml
new file mode 100644
index 0000000..be22dd5
--- /dev/null
+++ b/docs/examples/cli/extract.yml
@@ -0,0 +1,11 @@
+type: "llm"
+provider: "openai/gpt-4o-mini"
+api_token: "env:OPENAI_API_KEY"
+instruction: "Extract all articles with their titles, authors, publication dates and main topics in a structured format"
+params:
+  chunk_token_threshold: 4096
+  overlap_rate: 0.1
+  word_token_rate: 0.75
+  temperature: 0.3
+  max_tokens: 1000
+  verbose: true
\ No newline at end of file
diff --git a/docs/examples/cli/extract_css.yml b/docs/examples/cli/extract_css.yml
new file mode 100644
index 0000000..a4004a3
--- /dev/null
+++ b/docs/examples/cli/extract_css.yml
@@ -0,0 +1,3 @@
+type: "json-css"
+params:
+  verbose: true 
\ No newline at end of file
diff --git a/docs/examples/cli/llm_schema.json b/docs/examples/cli/llm_schema.json
new file mode 100644
index 0000000..a6969cc
--- /dev/null
+++ b/docs/examples/cli/llm_schema.json
@@ -0,0 +1,26 @@
+{
+  "title": "NewsArticle",
+  "type": "object",
+  "properties": {
+    "title": {
+      "type": "string",
+      "description": "The title/headline of the news article"
+    },
+    "link": {
+      "type": "string",
+      "description": "The URL or link to the full article"
+    },
+    "details": {
+      "type": "string", 
+      "description": "Brief summary or details about the article content"
+    },
+    "topics": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "description": "List of topics or categories associated with the article"
+    }
+  },
+  "required": ["title", "details"]
+}
\ No newline at end of file
diff --git a/docs/examples/cloud_browser/scrapeless_browser.py b/docs/examples/cloud_browser/scrapeless_browser.py
new file mode 100644
index 0000000..4981c81
--- /dev/null
+++ b/docs/examples/cloud_browser/scrapeless_browser.py
@@ -0,0 +1,61 @@
+import json
+import asyncio
+from urllib.parse import quote, urlencode
+from crawl4ai import CrawlerRunConfig, BrowserConfig, AsyncWebCrawler
+
+# Scrapeless provides a free anti-detection fingerprint browser client and cloud browsers:
+# https://www.scrapeless.com/en/blog/scrapeless-nstbrowser-strategic-integration
+
+async def main():
+    # customize browser fingerprint
+    fingerprint = {
+        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.1.2.3 Safari/537.36",
+        "platform": "Windows",
+        "screen": {
+            "width": 1280, "height": 1024
+        },
+        "localization": {
+            "languages": ["zh-HK", "en-US", "en"], "timezone": "Asia/Hong_Kong",
+        }
+    }
+
+    fingerprint_json = json.dumps(fingerprint)
+    encoded_fingerprint = quote(fingerprint_json)
+
+    scrapeless_params = {
+        "token": "your token",
+        "sessionTTL": 1000,
+        "sessionName": "Demo",
+        "fingerprint": encoded_fingerprint,
+        # Sets the target country/region for the proxy, sending requests via an IP address from that region. You can specify a country code (e.g., US for the United States, GB for the United Kingdom, ANY for any country). See country codes for all supported options.
+        # "proxyCountry": "ANY",
+        # create profile on scrapeless
+        # "profileId": "your profileId",
+        # For more usage details, please refer to https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started
+    }
+    query_string = urlencode(scrapeless_params)
+    scrapeless_connection_url = f"wss://browser.scrapeless.com/api/v2/browser?{query_string}"
+    async with AsyncWebCrawler(
+        config=BrowserConfig(
+            headless=False,
+            browser_mode="cdp",
+            cdp_url=scrapeless_connection_url,
+        )
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.scrapeless.com/en",
+            config=CrawlerRunConfig(
+                wait_for="css:.content",
+                scan_full_page=True,
+            ),
+        )
+        print("-" * 20)
+        print(f'Status Code: {result.status_code}')
+        print("-" * 20)
+        print(f'Title: {result.metadata["title"]}')
+        print(f'Description: {result.metadata["description"]}')
+        print("-" * 20)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+    
\ No newline at end of file
diff --git a/docs/examples/crawlai_vs_firecrawl.py b/docs/examples/crawlai_vs_firecrawl.py
new file mode 100644
index 0000000..ff89acb
--- /dev/null
+++ b/docs/examples/crawlai_vs_firecrawl.py
@@ -0,0 +1,70 @@
+import os, time
+
+# append the path to the root of the project
+import sys
+import asyncio
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from firecrawl import FirecrawlApp
+from crawl4ai import AsyncWebCrawler
+
+__data__ = os.path.join(os.path.dirname(__file__), "..", "..") + "/.data"
+
+
+async def compare():
+    app = FirecrawlApp(api_key=os.environ["FIRECRAWL_API_KEY"])
+
+    # Tet Firecrawl with a simple crawl
+    start = time.time()
+    scrape_status = app.scrape_url(
+        "https://www.nbcnews.com/business", params={"formats": ["markdown", "html"]}
+    )
+    end = time.time()
+    print(f"Time taken: {end - start} seconds")
+    print(len(scrape_status["markdown"]))
+    # save the markdown content with provider name
+    with open(f"{__data__}/firecrawl_simple.md", "w") as f:
+        f.write(scrape_status["markdown"])
+    # Count how many "cldnry.s-nbcnews.com" are in the markdown
+    print(scrape_status["markdown"].count("cldnry.s-nbcnews.com"))
+
+    async with AsyncWebCrawler() as crawler:
+        start = time.time()
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            # js_code=["const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"],
+            word_count_threshold=0,
+            cache_mode=CacheMode.BYPASS,
+            verbose=False,
+        )
+        end = time.time()
+        print(f"Time taken: {end - start} seconds")
+        print(len(result.markdown))
+        # save the markdown content with provider name
+        with open(f"{__data__}/crawl4ai_simple.md", "w") as f:
+            f.write(result.markdown)
+        # count how many "cldnry.s-nbcnews.com" are in the markdown
+        print(result.markdown.count("cldnry.s-nbcnews.com"))
+
+        start = time.time()
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            js_code=[
+                "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+            ],
+            word_count_threshold=0,
+            cache_mode=CacheMode.BYPASS,
+            verbose=False,
+        )
+        end = time.time()
+        print(f"Time taken: {end - start} seconds")
+        print(len(result.markdown))
+        # save the markdown content with provider name
+        with open(f"{__data__}/crawl4ai_js.md", "w") as f:
+            f.write(result.markdown)
+        # count how many "cldnry.s-nbcnews.com" are in the markdown
+        print(result.markdown.count("cldnry.s-nbcnews.com"))
+
+
+if __name__ == "__main__":
+    asyncio.run(compare())
diff --git a/docs/examples/crawler_monitor_example.py b/docs/examples/crawler_monitor_example.py
new file mode 100644
index 0000000..85d80ae
--- /dev/null
+++ b/docs/examples/crawler_monitor_example.py
@@ -0,0 +1,209 @@
+"""
+CrawlerMonitor Example
+
+This example demonstrates how to use the CrawlerMonitor component 
+to visualize and track web crawler operations in real-time.
+"""
+
+import time
+import uuid
+import random
+import threading
+from crawl4ai.components.crawler_monitor import CrawlerMonitor
+from crawl4ai.models import CrawlStatus
+
+def simulate_webcrawler_operations(monitor, num_tasks=20):
+    """
+    Simulates a web crawler's operations with multiple tasks and different states.
+    
+    Args:
+        monitor: The CrawlerMonitor instance
+        num_tasks: Number of tasks to simulate
+    """
+    print(f"Starting simulation with {num_tasks} tasks...")
+    
+    # Create and register all tasks first
+    task_ids = []
+    for i in range(num_tasks):
+        task_id = str(uuid.uuid4())
+        url = f"https://example.com/page{i}"
+        monitor.add_task(task_id, url)
+        task_ids.append((task_id, url))
+        
+        # Small delay between task creation
+        time.sleep(0.2)
+    
+    # Process tasks with a variety of different behaviors
+    threads = []
+    for i, (task_id, url) in enumerate(task_ids):
+        # Create a thread for each task
+        thread = threading.Thread(
+            target=process_task,
+            args=(monitor, task_id, url, i)
+        )
+        thread.daemon = True
+        threads.append(thread)
+    
+    # Start threads in batches to simulate concurrent processing
+    batch_size = 4  # Process 4 tasks at a time
+    for i in range(0, len(threads), batch_size):
+        batch = threads[i:i+batch_size]
+        for thread in batch:
+            thread.start()
+            time.sleep(0.5)  # Stagger thread start times
+        
+        # Wait a bit before starting next batch
+        time.sleep(random.uniform(1.0, 3.0))
+        
+        # Update queue statistics
+        update_queue_stats(monitor)
+        
+        # Simulate memory pressure changes
+        active_threads = [t for t in threads if t.is_alive()]
+        if len(active_threads) > 8:
+            monitor.update_memory_status("CRITICAL")
+        elif len(active_threads) > 4:
+            monitor.update_memory_status("PRESSURE")
+        else:
+            monitor.update_memory_status("NORMAL")
+    
+    # Wait for all threads to complete
+    for thread in threads:
+        thread.join()
+    
+    # Final updates
+    update_queue_stats(monitor)
+    monitor.update_memory_status("NORMAL")
+    
+    print("Simulation completed!")
+    
+def process_task(monitor, task_id, url, index):
+    """Simulate processing of a single task."""
+    # Tasks start in queued state (already added)
+    
+    # Simulate waiting in queue
+    wait_time = random.uniform(0.5, 3.0)
+    time.sleep(wait_time)
+    
+    # Start processing - move to IN_PROGRESS
+    monitor.update_task(
+        task_id=task_id,
+        status=CrawlStatus.IN_PROGRESS,
+        start_time=time.time(),
+        wait_time=wait_time
+    )
+    
+    # Simulate task processing with memory usage changes
+    total_process_time = random.uniform(2.0, 10.0)
+    step_time = total_process_time / 5  # Update in 5 steps
+    
+    for step in range(5):
+        # Simulate increasing then decreasing memory usage
+        if step < 3:  # First 3 steps - increasing
+            memory_usage = random.uniform(5.0, 20.0) * (step + 1)
+        else:  # Last 2 steps - decreasing
+            memory_usage = random.uniform(5.0, 20.0) * (5 - step)
+            
+        # Update peak memory if this is higher
+        peak = max(memory_usage, monitor.get_task_stats(task_id).get("peak_memory", 0))
+        
+        monitor.update_task(
+            task_id=task_id,
+            memory_usage=memory_usage,
+            peak_memory=peak
+        )
+        
+        time.sleep(step_time)
+    
+    # Determine final state - 80% success, 20% failure
+    if index % 5 == 0:  # Every 5th task fails
+        monitor.update_task(
+            task_id=task_id,
+            status=CrawlStatus.FAILED,
+            end_time=time.time(),
+            memory_usage=0.0,
+            error_message="Connection timeout"
+        )
+    else:
+        monitor.update_task(
+            task_id=task_id,
+            status=CrawlStatus.COMPLETED,
+            end_time=time.time(),
+            memory_usage=0.0
+        )
+
+def update_queue_stats(monitor):
+    """Update queue statistics based on current tasks."""
+    task_stats = monitor.get_all_task_stats()
+    
+    # Count queued tasks
+    queued_tasks = [
+        stats for stats in task_stats.values() 
+        if stats["status"] == CrawlStatus.QUEUED.name
+    ]
+    
+    total_queued = len(queued_tasks)
+    
+    if total_queued > 0:
+        current_time = time.time()
+        # Calculate wait times
+        wait_times = [
+            current_time - stats.get("enqueue_time", current_time)
+            for stats in queued_tasks
+        ]
+        highest_wait_time = max(wait_times) if wait_times else 0.0
+        avg_wait_time = sum(wait_times) / len(wait_times) if wait_times else 0.0
+    else:
+        highest_wait_time = 0.0
+        avg_wait_time = 0.0
+    
+    # Update monitor
+    monitor.update_queue_statistics(
+        total_queued=total_queued,
+        highest_wait_time=highest_wait_time,
+        avg_wait_time=avg_wait_time
+    )
+
+def main():
+    # Initialize the monitor
+    monitor = CrawlerMonitor(
+        urls_total=20,  # Total URLs to process
+        refresh_rate=0.5,  # Update UI twice per second
+        enable_ui=True,    # Enable terminal UI
+        max_width=120     # Set maximum width to 120 characters
+    )
+    
+    # Start the monitor
+    monitor.start()
+    
+    try:
+        # Run simulation
+        simulate_webcrawler_operations(monitor)
+        
+        # Keep monitor running a bit to see final state
+        print("Waiting to view final state...")
+        time.sleep(5)
+        
+    except KeyboardInterrupt:
+        print("\nExample interrupted by user")
+    finally:
+        # Stop the monitor
+        monitor.stop()
+        print("Example completed!")
+        
+        # Print some statistics
+        summary = monitor.get_summary()
+        print("\nCrawler Statistics Summary:")
+        print(f"Total URLs: {summary['urls_total']}")
+        print(f"Completed: {summary['urls_completed']}")
+        print(f"Completion percentage: {summary['completion_percentage']:.1f}%")
+        print(f"Peak memory usage: {summary['peak_memory_percent']:.1f}%")
+        
+        # Print task status counts
+        status_counts = summary['status_counts']
+        print("\nTask Status Counts:")
+        for status, count in status_counts.items():
+            print(f"  {status}: {count}")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/docs/examples/crypto_analysis_example.py b/docs/examples/crypto_analysis_example.py
new file mode 100644
index 0000000..c5537a9
--- /dev/null
+++ b/docs/examples/crypto_analysis_example.py
@@ -0,0 +1,445 @@
+"""
+Crawl4AI Crypto Trading Analysis Demo
+Author: Unclecode
+Date: 2024-03-15
+
+This script demonstrates advanced crypto market analysis using:
+1. Web scraping of real-time CoinMarketCap data
+2. Smart table extraction with layout detection
+3. Hedge fund-grade financial metrics
+4. Interactive visualizations for trading signals
+
+Key Features:
+- Volume Anomaly Detection: Finds unusual trading activity
+- Liquidity Power Score: Identifies easily tradable assets
+- Volatility-Weighted Momentum: Surface sustainable trends
+- Smart Money Signals: Algorithmic buy/hold recommendations
+"""
+
+import asyncio
+import pandas as pd
+import numpy as np
+import re
+import plotly.express as px
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    LXMLWebScrapingStrategy,
+)
+from crawl4ai import CrawlResult
+from typing import List
+
+__current_dir__ = __file__.rsplit("/", 1)[0]
+
+class CryptoAlphaGenerator:
+    """
+    Advanced crypto analysis engine that transforms raw web data into:
+    - Volume anomaly flags
+    - Liquidity scores
+    - Momentum-risk ratios
+    - Machine learning-inspired trading signals
+
+    Methods:
+    analyze_tables(): Process raw tables into trading insights
+    create_visuals(): Generate institutional-grade visualizations
+    generate_insights(): Create plain English trading recommendations
+    """
+
+    def clean_data(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Convert crypto market data to machine-readable format.
+        Handles currency symbols, units (B=Billions), and percentage values.
+        """
+        # Make a copy to avoid SettingWithCopyWarning
+        df = df.copy()
+        
+        # Clean Price column (handle currency symbols)
+        df["Price"] = df["Price"].astype(str).str.replace("[^\d.]", "", regex=True).astype(float)
+        
+        # Handle Market Cap and Volume, considering both Billions and Trillions
+        def convert_large_numbers(value):
+            if pd.isna(value):
+                return float('nan')
+            value = str(value)
+            multiplier = 1
+            if 'B' in value:
+                multiplier = 1e9
+            elif 'T' in value:
+                multiplier = 1e12
+            # Handle cases where the value might already be numeric
+            cleaned_value = re.sub(r"[^\d.]", "", value)
+            return float(cleaned_value) * multiplier if cleaned_value else float('nan')
+        
+        df["Market Cap"] = df["Market Cap"].apply(convert_large_numbers)
+        df["Volume(24h)"] = df["Volume(24h)"].apply(convert_large_numbers)
+        
+        # Convert percentages to decimal values
+        for col in ["1h %", "24h %", "7d %"]:
+            if col in df.columns:
+                # First ensure it's string, then clean
+                df[col] = (
+                    df[col].astype(str)
+                    .str.replace("%", "")
+                    .str.replace(",", ".")
+                    .replace("nan", np.nan)
+                )
+                df[col] = pd.to_numeric(df[col], errors='coerce') / 100
+        
+        return df
+
+    def calculate_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Compute advanced trading metrics used by quantitative funds:
+
+        1. Volume/Market Cap Ratio - Measures liquidity efficiency
+           (High ratio = Underestimated attention, and small-cap = higher growth potential)
+
+        2. Volatility Score - Risk-adjusted momentum potential - Shows how stable is the trend
+           (STD of 1h/24h/7d returns)
+
+        3. Momentum Score - Weighted average of returns - Shows how strong is the trend
+           (1h:30% + 24h:50% + 7d:20%)
+
+        4. Volume Anomaly - 3σ deviation detection
+           (Flags potential insider activity) - Unusual trading activity – Flags coins with volume spikes (potential insider buying or news).
+        """
+        # Liquidity Metrics
+        df["Volume/Market Cap Ratio"] = df["Volume(24h)"] / df["Market Cap"]
+
+        # Risk Metrics
+        df["Volatility Score"] = df[["1h %", "24h %", "7d %"]].std(axis=1)
+
+        # Momentum Metrics
+        df["Momentum Score"] = df["1h %"] * 0.3 + df["24h %"] * 0.5 + df["7d %"] * 0.2
+
+        # Anomaly Detection
+        median_vol = df["Volume(24h)"].median()
+        df["Volume Anomaly"] = df["Volume(24h)"] > 3 * median_vol
+
+        # Value Flags
+        # Undervalued Flag - Low market cap and high momentum
+        # (High growth potential and low attention)
+        df["Undervalued Flag"] = (df["Market Cap"] < 1e9) & (
+            df["Momentum Score"] > 0.05
+        )
+        # Liquid Giant Flag - High volume/market cap ratio and large market cap
+        # (High liquidity and large market cap = institutional interest)
+        df["Liquid Giant"] = (df["Volume/Market Cap Ratio"] > 0.15) & (
+            df["Market Cap"] > 1e9
+        )
+
+        return df
+
+    def generate_insights_simple(self, df: pd.DataFrame) -> str:
+        """
+        Generates an ultra-actionable crypto trading report with:
+        - Risk-tiered opportunities (High/Medium/Low)
+        - Concrete examples for each trade type
+        - Entry/exit strategies spelled out
+        - Visual cues for quick scanning
+        """
+        report = [
+            "🚀 **CRYPTO TRADING CHEAT SHEET** 🚀",
+            "*Based on quantitative signals + hedge fund tactics*",
+            "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+        ]
+
+        # 1. HIGH-RISK: Undervalued Small-Caps (Momentum Plays)
+        high_risk = df[df["Undervalued Flag"]].sort_values("Momentum Score", ascending=False)
+        if not high_risk.empty:
+            example_coin = high_risk.iloc[0]
+            report.extend([
+                "\n🔥 **HIGH-RISK: Rocket Fuel Small-Caps**",
+                f"*Example Trade:* {example_coin['Name']} (Price: ${example_coin['Price']:.6f})",
+                "📊 *Why?* Tiny market cap (<$1B) but STRONG momentum (+{:.0f}% last week)".format(example_coin['7d %']*100),
+                "🎯 *Strategy:*",
+                "1. Wait for 5-10% dip from recent high (${:.6f} → Buy under ${:.6f})".format(
+                    example_coin['Price'] / (1 - example_coin['24h %']),  # Approx recent high
+                    example_coin['Price'] * 0.95
+                ),
+                "2. Set stop-loss at -10% (${:.6f})".format(example_coin['Price'] * 0.90),
+                "3. Take profit at +20% (${:.6f})".format(example_coin['Price'] * 1.20),
+                "⚠️ *Risk Warning:* These can drop 30% fast! Never bet more than 5% of your portfolio.",
+                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+            ])
+
+        # 2. MEDIUM-RISK: Liquid Giants (Swing Trades)
+        medium_risk = df[df["Liquid Giant"]].sort_values("Volume/Market Cap Ratio", ascending=False)
+        if not medium_risk.empty:
+            example_coin = medium_risk.iloc[0]
+            report.extend([
+                "\n💎 **MEDIUM-RISK: Liquid Giants (Safe Swing Trades)**",
+                f"*Example Trade:* {example_coin['Name']} (Market Cap: ${example_coin['Market Cap']/1e9:.1f}B)",
+                "📊 *Why?* Huge volume (${:.1f}M/day) makes it easy to enter/exit".format(example_coin['Volume(24h)']/1e6),
+                "🎯 *Strategy:*",
+                "1. Buy when 24h volume > 15% of market cap (Current: {:.0f}%)".format(example_coin['Volume/Market Cap Ratio']*100),
+                "2. Hold 1-4 weeks (Big coins trend longer)",
+                "3. Exit when momentum drops below 5% (Current: {:.0f}%)".format(example_coin['Momentum Score']*100),
+                "📉 *Pro Tip:* Watch Bitcoin's trend - if BTC drops 5%, these usually follow.",
+                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+            ])
+
+        # 3. LOW-RISK: Stable Momentum (DCA Targets)
+        low_risk = df[
+            (df["Momentum Score"] > 0.05) & 
+            (df["Volatility Score"] < 0.03)
+        ].sort_values("Market Cap", ascending=False)
+        if not low_risk.empty:
+            example_coin = low_risk.iloc[0]
+            report.extend([
+                "\n🛡️ **LOW-RISK: Steady Climbers (DCA & Forget)**",
+                f"*Example Trade:* {example_coin['Name']} (Volatility: {example_coin['Volatility Score']:.2f}/5)",
+                "📊 *Why?* Rises steadily (+{:.0f}%/week) with LOW drama".format(example_coin['7d %']*100),
+                "🎯 *Strategy:*",
+                "1. Buy small amounts every Tuesday/Friday (DCA)",
+                "2. Hold for 3+ months (Compound gains work best here)",
+                "3. Sell 10% at every +25% milestone",
+                "💰 *Best For:* Long-term investors who hate sleepless nights",
+                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+            ])
+
+        # Volume Spike Alerts
+        anomalies = df[df["Volume Anomaly"]].sort_values("Volume(24h)", ascending=False)
+        if not anomalies.empty:
+            example_coin = anomalies.iloc[0]
+            report.extend([
+                "\n🚨 **Volume Spike Alert (Possible News/Whale Action)**",
+                f"*Coin:* {example_coin['Name']} (Volume: ${example_coin['Volume(24h)']/1e6:.1f}M, usual: ${example_coin['Volume(24h)']/3/1e6:.1f}M)",
+                "🔍 *Check:* Twitter/CoinGecko for news before trading",
+                "⚡ *If no news:* Could be insider buying - watch price action:",
+                "- Break above today's high → Buy with tight stop-loss",
+                "- Fade back down → Avoid (may be a fakeout)"
+            ])
+
+        # Pro Tip Footer
+        report.append("\n✨ *Pro Tip:* Bookmark this report & check back in 24h to see if signals held up.")
+
+        return "\n".join(report)
+
+    def generate_insights(self, df: pd.DataFrame) -> str:
+        """
+        Generates a tactical trading report with:
+        - Top 3 trades per risk level (High/Medium/Low)
+        - Auto-calculated entry/exit prices
+        - BTC chart toggle tip
+        """
+        # Filter top candidates for each risk level
+        high_risk = (
+            df[df["Undervalued Flag"]]
+            .sort_values("Momentum Score", ascending=False)
+            .head(3)
+        )
+        medium_risk = (
+            df[df["Liquid Giant"]]
+            .sort_values("Volume/Market Cap Ratio", ascending=False)
+            .head(3)
+        )
+        low_risk = (
+            df[(df["Momentum Score"] > 0.05) & (df["Volatility Score"] < 0.03)]
+            .sort_values("Momentum Score", ascending=False)
+            .head(3)
+        )
+
+        report = ["# 🎯 Crypto Trading Tactical Report (Top 3 Per Risk Tier)"]
+        
+        # 1. High-Risk Trades (Small-Cap Momentum)
+        if not high_risk.empty:
+            report.append("\n## 🔥 HIGH RISK: Small-Cap Rockets (5-50% Potential)")
+            for i, coin in high_risk.iterrows():
+                current_price = coin["Price"]
+                entry = current_price * 0.95  # -5% dip
+                stop_loss = current_price * 0.90  # -10%
+                take_profit = current_price * 1.20  # +20%
+                
+                report.append(
+                    f"\n### {coin['Name']} (Momentum: {coin['Momentum Score']:.1%})"
+                    f"\n- **Current Price:** ${current_price:.4f}"
+                    f"\n- **Entry:** < ${entry:.4f} (Wait for pullback)"
+                    f"\n- **Stop-Loss:** ${stop_loss:.4f} (-10%)"
+                    f"\n- **Target:** ${take_profit:.4f} (+20%)"
+                    f"\n- **Risk/Reward:** 1:2"
+                    f"\n- **Watch:** Volume spikes above {coin['Volume(24h)']/1e6:.1f}M"
+                )
+
+        # 2. Medium-Risk Trades (Liquid Giants)
+        if not medium_risk.empty:
+            report.append("\n## 💎 MEDIUM RISK: Liquid Swing Trades (10-30% Potential)")
+            for i, coin in medium_risk.iterrows():
+                current_price = coin["Price"]
+                entry = current_price * 0.98  # -2% dip
+                stop_loss = current_price * 0.94  # -6%
+                take_profit = current_price * 1.15  # +15%
+                
+                report.append(
+                    f"\n### {coin['Name']} (Liquidity Score: {coin['Volume/Market Cap Ratio']:.1%})"
+                    f"\n- **Current Price:** ${current_price:.2f}"
+                    f"\n- **Entry:** < ${entry:.2f} (Buy slight dips)"
+                    f"\n- **Stop-Loss:** ${stop_loss:.2f} (-6%)"
+                    f"\n- **Target:** ${take_profit:.2f} (+15%)"
+                    f"\n- **Hold Time:** 1-3 weeks"
+                    f"\n- **Key Metric:** Volume/Cap > 15%"
+                )
+
+        # 3. Low-Risk Trades (Stable Momentum)
+        if not low_risk.empty:
+            report.append("\n## 🛡️ LOW RISK: Steady Gainers (5-15% Potential)")
+            for i, coin in low_risk.iterrows():
+                current_price = coin["Price"]
+                entry = current_price * 0.99  # -1% dip
+                stop_loss = current_price * 0.97  # -3%
+                take_profit = current_price * 1.10  # +10%
+                
+                report.append(
+                    f"\n### {coin['Name']} (Stability Score: {1/coin['Volatility Score']:.1f}x)"
+                    f"\n- **Current Price:** ${current_price:.2f}"
+                    f"\n- **Entry:** < ${entry:.2f} (Safe zone)"
+                    f"\n- **Stop-Loss:** ${stop_loss:.2f} (-3%)"
+                    f"\n- **Target:** ${take_profit:.2f} (+10%)"
+                    f"\n- **DCA Suggestion:** 3 buys over 72 hours"
+                )
+
+        # Volume Anomaly Alert
+        anomalies = df[df["Volume Anomaly"]].sort_values("Volume(24h)", ascending=False).head(2)
+        if not anomalies.empty:
+            report.append("\n⚠️ **Volume Spike Alerts**")
+            for i, coin in anomalies.iterrows():
+                report.append(
+                    f"- {coin['Name']}: Volume {coin['Volume(24h)']/1e6:.1f}M "
+                    f"(3x normal) | Price moved: {coin['24h %']:.1%}"
+                )
+
+        # Pro Tip
+        report.append(
+            "\n📊 **Chart Hack:** Hide BTC in visuals:\n"
+            "```python\n"
+            "# For 3D Map:\n"
+            "fig.update_traces(visible=False, selector={'name':'Bitcoin'})\n"
+            "# For Treemap:\n"
+            "df = df[df['Name'] != 'Bitcoin']\n"
+            "```"
+        )
+
+        return "\n".join(report)
+
+    def create_visuals(self, df: pd.DataFrame) -> dict:
+        """Enhanced visuals with BTC toggle support"""
+        # 3D Market Map (with BTC toggle hint)
+        fig1 = px.scatter_3d(
+            df,
+            x="Market Cap",
+            y="Volume/Market Cap Ratio",
+            z="Momentum Score",
+            color="Name",  # Color by name to allow toggling
+            hover_name="Name",
+            title="Market Map (Toggle BTC in legend to focus on alts)",
+            log_x=True
+        )
+        fig1.update_traces(
+            marker=dict(size=df["Volatility Score"]*100 + 5)  # Dynamic sizing
+        )
+        
+        # Liquidity Tree (exclude BTC if too dominant)
+        if df[df["Name"] == "BitcoinBTC"]["Market Cap"].values[0] > df["Market Cap"].median() * 10:
+            df = df[df["Name"] != "BitcoinBTC"]
+        
+        fig2 = px.treemap(
+            df,
+            path=["Name"],
+            values="Market Cap",
+            color="Volume/Market Cap Ratio",
+            title="Liquidity Tree (BTC auto-removed if dominant)"
+        )
+        
+        return {"market_map": fig1, "liquidity_tree": fig2}
+
+async def main():
+    """
+    Main execution flow:
+    1. Configure headless browser for scraping
+    2. Extract live crypto market data
+    3. Clean and analyze using hedge fund models
+    4. Generate visualizations and insights
+    5. Output professional trading report
+    """
+    # Configure browser with anti-detection features
+    browser_config = BrowserConfig(
+        headless=False,
+    )
+
+    # Initialize crawler with smart table detection
+    crawler = AsyncWebCrawler(config=browser_config)
+    await crawler.start()
+
+    try:
+        # Set up scraping parameters
+        crawl_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            table_score_threshold=8,  # Strict table detection
+            keep_data_attributes=True,
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            scan_full_page=True,
+            scroll_delay=0.2,
+        )
+
+        # Execute market data extraction
+        results: List[CrawlResult] = await crawler.arun(
+            url="https://coinmarketcap.com/?page=1", config=crawl_config
+        )
+
+        # Process results
+        raw_df = pd.DataFrame()
+        for result in results:
+            # Use the new tables field, falling back to media["tables"] for backward compatibility
+            tables = result.tables if hasattr(result, "tables") and result.tables else result.media.get("tables", [])
+            if result.success and tables:
+                # Extract primary market table
+                # DataFrame
+                raw_df = pd.DataFrame(
+                    tables[0]["rows"],
+                    columns=tables[0]["headers"],
+                )
+                break
+
+
+        # This is for debugging only
+        # ////// Remove this in production from here..
+        # Save raw data for debugging
+        raw_df.to_csv(f"{__current_dir__}/tmp/raw_crypto_data.csv", index=False)
+        print("🔍 Raw data saved to 'raw_crypto_data.csv'")
+
+        # Read from file for debugging
+        raw_df = pd.read_csv(f"{__current_dir__}/tmp/raw_crypto_data.csv")
+        # ////// ..to here
+
+        # Select top 20
+        raw_df = raw_df.head(50)
+        # Remove "Buy" from name
+        raw_df["Name"] = raw_df["Name"].str.replace("Buy", "")
+
+        # Initialize analysis engine
+        analyzer = CryptoAlphaGenerator()
+        clean_df = analyzer.clean_data(raw_df)
+        analyzed_df = analyzer.calculate_metrics(clean_df)
+
+        # Generate outputs
+        visuals = analyzer.create_visuals(analyzed_df)
+        insights = analyzer.generate_insights(analyzed_df)
+
+        # Save visualizations
+        visuals["market_map"].write_html(f"{__current_dir__}/tmp/market_map.html")
+        visuals["liquidity_tree"].write_html(f"{__current_dir__}/tmp/liquidity_tree.html")
+
+        # Display results
+        print("🔑 Key Trading Insights:")
+        print(insights)
+        print("\n📊 Open 'market_map.html' for interactive analysis")
+        print("\n📊 Open 'liquidity_tree.html' for interactive analysis")
+
+    finally:
+        await crawler.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/deep_crawl_cancellation.py b/docs/examples/deep_crawl_cancellation.py
new file mode 100644
index 0000000..c9cf06b
--- /dev/null
+++ b/docs/examples/deep_crawl_cancellation.py
@@ -0,0 +1,427 @@
+"""
+Deep Crawl Cancellation Example
+
+This example demonstrates how to implement cancellable deep crawls in Crawl4AI.
+Useful for cloud platforms, job management systems, or any scenario where you
+need to stop a running crawl mid-execution and retrieve partial results.
+
+Features demonstrated:
+1. Callback-based cancellation (check external source like Redis)
+2. Direct cancellation via cancel() method
+3. Checking cancellation status
+4. State tracking with cancelled flag
+5. Strategy reuse after cancellation
+
+Requirements:
+    pip install crawl4ai redis
+"""
+
+import asyncio
+import json
+from typing import Dict, Any, List
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.deep_crawling import (
+    BFSDeepCrawlStrategy,
+    DFSDeepCrawlStrategy,
+    BestFirstCrawlingStrategy,
+)
+
+
+# =============================================================================
+# Example 1: Basic Cancellation with Callback
+# =============================================================================
+
+async def example_callback_cancellation():
+    """
+    Cancel a crawl after reaching a certain number of pages.
+    This simulates checking an external cancellation source.
+
+    Note: We use DFS here because it processes one URL at a time,
+    giving precise cancellation control. BFS processes URLs in batches
+    (levels), so cancellation happens at level boundaries.
+    """
+    print("\n" + "="*60)
+    print("Example 1: Callback-based Cancellation (DFS)")
+    print("="*60)
+
+    pages_crawled = 0
+    max_before_cancel = 5
+
+    # This callback is checked before each URL is processed
+    async def should_cancel():
+        # In production, you might check Redis, a database, or an API:
+        # job = await redis.get(f"job:{job_id}")
+        # return job.get("status") == "cancelled"
+        return pages_crawled >= max_before_cancel
+
+    # Track progress via state changes
+    async def on_state_change(state: Dict[str, Any]):
+        nonlocal pages_crawled
+        pages_crawled = state.get("pages_crawled", 0)
+        cancelled = state.get("cancelled", False)
+        print(f"  Progress: {pages_crawled} pages | Cancelled: {cancelled}")
+
+    # Use DFS for precise per-URL cancellation control
+    strategy = DFSDeepCrawlStrategy(
+        max_depth=3,
+        max_pages=100,  # Would crawl up to 100, but we'll cancel at 5
+        should_cancel=should_cancel,
+        on_state_change=on_state_change,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    print(f"Starting crawl (will cancel after {max_before_cancel} pages)...")
+
+    async with AsyncWebCrawler() as crawler:
+        results = await crawler.arun(
+            "https://docs.crawl4ai.com",
+            config=config
+        )
+
+    print(f"\nResults:")
+    print(f"  - Crawled {len(results)} pages")
+    print(f"  - Strategy cancelled: {strategy.cancelled}")
+    print(f"  - Pages crawled counter: {strategy._pages_crawled}")
+
+    return results
+
+
+# =============================================================================
+# Example 2: Direct Cancellation via cancel() Method
+# =============================================================================
+
+async def example_direct_cancellation():
+    """
+    Cancel a crawl directly using the cancel() method.
+    This is useful when you have direct access to the strategy instance.
+    """
+    print("\n" + "="*60)
+    print("Example 2: Direct Cancellation via cancel()")
+    print("="*60)
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=3,
+        max_pages=100,
+    )
+
+    # Cancel after 3 seconds
+    async def cancel_after_delay():
+        await asyncio.sleep(3)
+        print("  Calling strategy.cancel()...")
+        strategy.cancel()
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    print("Starting crawl (will cancel after 3 seconds)...")
+
+    async with AsyncWebCrawler() as crawler:
+        # Start cancellation timer in background
+        cancel_task = asyncio.create_task(cancel_after_delay())
+
+        try:
+            results = await crawler.arun(
+                "https://docs.crawl4ai.com",
+                config=config
+            )
+        finally:
+            cancel_task.cancel()
+            try:
+                await cancel_task
+            except asyncio.CancelledError:
+                pass
+
+    print(f"\nResults:")
+    print(f"  - Crawled {len(results)} pages")
+    print(f"  - Strategy cancelled: {strategy.cancelled}")
+
+    return results
+
+
+# =============================================================================
+# Example 3: Streaming Mode with Cancellation
+# =============================================================================
+
+async def example_streaming_cancellation():
+    """
+    Cancel a streaming crawl and process partial results as they arrive.
+    """
+    print("\n" + "="*60)
+    print("Example 3: Streaming Mode with Cancellation")
+    print("="*60)
+
+    results_count = 0
+    cancel_after = 3
+
+    async def should_cancel():
+        return results_count >= cancel_after
+
+    strategy = DFSDeepCrawlStrategy(
+        max_depth=5,
+        max_pages=50,
+        should_cancel=should_cancel,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        stream=True,  # Enable streaming
+        verbose=False,
+    )
+
+    print(f"Starting streaming crawl (will cancel after {cancel_after} results)...")
+
+    results = []
+    async with AsyncWebCrawler() as crawler:
+        async for result in await crawler.arun(
+            "https://docs.crawl4ai.com",
+            config=config
+        ):
+            results_count += 1
+            results.append(result)
+            print(f"  Received result {results_count}: {result.url[:60]}...")
+
+    print(f"\nResults:")
+    print(f"  - Received {len(results)} results")
+    print(f"  - Strategy cancelled: {strategy.cancelled}")
+
+    return results
+
+
+# =============================================================================
+# Example 4: Strategy Reuse After Cancellation
+# =============================================================================
+
+async def example_strategy_reuse():
+    """
+    Demonstrate that a strategy can be reused after cancellation.
+    The cancel flag is automatically reset on the next crawl.
+    """
+    print("\n" + "="*60)
+    print("Example 4: Strategy Reuse After Cancellation")
+    print("="*60)
+
+    crawl_number = 0
+
+    async def cancel_first_crawl_only():
+        # Only cancel during the first crawl
+        return crawl_number == 1
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=1,
+        max_pages=10,
+        should_cancel=cancel_first_crawl_only,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        # First crawl - will be cancelled immediately
+        crawl_number = 1
+        print("First crawl (will be cancelled)...")
+        results1 = await crawler.arun(
+            "https://docs.crawl4ai.com",
+            config=config
+        )
+        print(f"  - Results: {len(results1)}, Cancelled: {strategy.cancelled}")
+
+        # Second crawl - should work normally
+        crawl_number = 2
+        print("\nSecond crawl (should complete normally)...")
+        results2 = await crawler.arun(
+            "https://docs.crawl4ai.com",
+            config=config
+        )
+        print(f"  - Results: {len(results2)}, Cancelled: {strategy.cancelled}")
+
+    print(f"\nSummary:")
+    print(f"  - First crawl: {len(results1)} results (cancelled)")
+    print(f"  - Second crawl: {len(results2)} results (completed)")
+
+
+# =============================================================================
+# Example 5: Best-First Strategy with Cancellation
+# =============================================================================
+
+async def example_best_first_cancellation():
+    """
+    Cancel a Best-First crawl that prioritizes URLs by relevance score.
+
+    Note: Best-First processes URLs in batches (default 10), so cancellation
+    happens at batch boundaries. You may see more results than the cancel
+    threshold before the crawl stops.
+    """
+    print("\n" + "="*60)
+    print("Example 5: Best-First Strategy with Cancellation")
+    print("="*60)
+
+    from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
+
+    pages_crawled = 0
+    cancel_threshold = 5
+
+    async def should_cancel():
+        return pages_crawled >= cancel_threshold
+
+    async def track_progress(state: Dict[str, Any]):
+        nonlocal pages_crawled
+        pages_crawled = state.get("pages_crawled", 0)
+        print(f"  Pages: {pages_crawled}, Cancelled: {state.get('cancelled', False)}")
+
+    scorer = KeywordRelevanceScorer(
+        keywords=["api", "example", "tutorial"],
+        weight=0.8
+    )
+
+    strategy = BestFirstCrawlingStrategy(
+        max_depth=2,
+        max_pages=50,
+        url_scorer=scorer,
+        should_cancel=should_cancel,
+        on_state_change=track_progress,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        stream=True,
+        verbose=False,
+    )
+
+    print(f"Starting Best-First crawl (will cancel after {cancel_threshold} pages)...")
+    print("  (Note: Best-First processes in batches, so may crawl slightly more)")
+
+    results = []
+    async with AsyncWebCrawler() as crawler:
+        async for result in await crawler.arun(
+            "https://docs.crawl4ai.com",
+            config=config
+        ):
+            results.append(result)
+            score = result.metadata.get("score", 0)
+            print(f"  Result: {result.url[:50]}... (score: {score:.2f})")
+
+    print(f"\nResults:")
+    print(f"  - Crawled {len(results)} high-priority pages")
+    print(f"  - Strategy cancelled: {strategy.cancelled}")
+
+
+# =============================================================================
+# Example 6: Production Pattern - Redis-Based Cancellation (Simulated)
+# =============================================================================
+
+async def example_production_pattern():
+    """
+    Simulate a production pattern where cancellation is checked from Redis.
+    This pattern is suitable for cloud platforms with job management.
+    """
+    print("\n" + "="*60)
+    print("Example 6: Production Pattern (Simulated Redis)")
+    print("="*60)
+
+    # Simulate Redis storage
+    redis_storage: Dict[str, str] = {}
+
+    job_id = "crawl-job-12345"
+
+    # Simulate Redis operations
+    async def redis_get(key: str) -> str:
+        return redis_storage.get(key)
+
+    async def redis_set(key: str, value: str):
+        redis_storage[key] = value
+
+    # Initialize job status
+    await redis_set(f"{job_id}:status", "running")
+
+    # Cancellation check
+    async def check_cancelled():
+        status = await redis_get(f"{job_id}:status")
+        return status == "cancelled"
+
+    # Progress tracking
+    async def save_progress(state: Dict[str, Any]):
+        await redis_set(f"{job_id}:state", json.dumps(state))
+        await redis_set(f"{job_id}:pages", str(state["pages_crawled"]))
+        print(f"  Saved progress: {state['pages_crawled']} pages")
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=2,
+        max_pages=20,
+        should_cancel=check_cancelled,
+        on_state_change=save_progress,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    # Simulate external cancellation after 2 seconds
+    async def external_cancel():
+        await asyncio.sleep(2)
+        print("\n  [External] Setting job status to 'cancelled'...")
+        await redis_set(f"{job_id}:status", "cancelled")
+
+    print("Starting crawl with simulated Redis job management...")
+
+    async with AsyncWebCrawler() as crawler:
+        cancel_task = asyncio.create_task(external_cancel())
+
+        try:
+            results = await crawler.arun(
+                "https://docs.crawl4ai.com",
+                config=config
+            )
+        finally:
+            cancel_task.cancel()
+            try:
+                await cancel_task
+            except asyncio.CancelledError:
+                pass
+
+    # Final status
+    final_status = "cancelled" if strategy.cancelled else "completed"
+    await redis_set(f"{job_id}:status", final_status)
+
+    print(f"\nJob completed:")
+    print(f"  - Final status: {final_status}")
+    print(f"  - Pages crawled: {await redis_get(f'{job_id}:pages')}")
+    print(f"  - Results returned: {len(results)}")
+
+    # Show final state
+    final_state = json.loads(await redis_get(f"{job_id}:state"))
+    print(f"  - State saved: {len(final_state.get('visited', []))} URLs visited")
+
+
+# =============================================================================
+# Main
+# =============================================================================
+
+async def main():
+    """Run all examples."""
+    print("="*60)
+    print("Deep Crawl Cancellation Examples")
+    print("="*60)
+
+    await example_callback_cancellation()
+    await example_direct_cancellation()
+    await example_streaming_cancellation()
+    await example_strategy_reuse()
+    await example_best_first_cancellation()
+    await example_production_pattern()
+
+    print("\n" + "="*60)
+    print("All examples completed!")
+    print("="*60)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/deep_crawl_crash_recovery.py b/docs/examples/deep_crawl_crash_recovery.py
new file mode 100644
index 0000000..bc5a42e
--- /dev/null
+++ b/docs/examples/deep_crawl_crash_recovery.py
@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+"""
+Deep Crawl Crash Recovery Example
+
+This example demonstrates how to implement crash recovery for long-running
+deep crawls. The feature is useful for:
+
+- Cloud deployments with spot/preemptible instances
+- Long-running crawls that may be interrupted
+- Distributed crawling with state coordination
+
+Key concepts:
+- `on_state_change`: Callback fired after each URL is processed
+- `resume_state`: Pass saved state to continue from a checkpoint
+- `export_state()`: Get the last captured state manually
+
+Works with all strategies: BFSDeepCrawlStrategy, DFSDeepCrawlStrategy,
+BestFirstCrawlingStrategy
+"""
+
+import asyncio
+import json
+import os
+from pathlib import Path
+from typing import Dict, Any, List
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+
+# File to store crawl state (in production, use Redis/database)
+STATE_FILE = Path("crawl_state.json")
+
+
+async def save_state_to_file(state: Dict[str, Any]) -> None:
+    """
+    Callback to save state after each URL is processed.
+
+    In production, you might save to:
+    - Redis: await redis.set("crawl_state", json.dumps(state))
+    - Database: await db.execute("UPDATE crawls SET state = ?", json.dumps(state))
+    - S3: await s3.put_object(Bucket="crawls", Key="state.json", Body=json.dumps(state))
+    """
+    with open(STATE_FILE, "w") as f:
+        json.dump(state, f, indent=2)
+    print(f"  [State saved] Pages: {state['pages_crawled']}, Pending: {len(state['pending'])}")
+
+
+def load_state_from_file() -> Dict[str, Any] | None:
+    """Load previously saved state, if it exists."""
+    if STATE_FILE.exists():
+        with open(STATE_FILE, "r") as f:
+            return json.load(f)
+    return None
+
+
+async def example_basic_state_persistence():
+    """
+    Example 1: Basic state persistence with file storage.
+
+    The on_state_change callback is called after each URL is processed,
+    allowing you to save progress in real-time.
+    """
+    print("\n" + "=" * 60)
+    print("Example 1: Basic State Persistence")
+    print("=" * 60)
+
+    # Clean up any previous state
+    if STATE_FILE.exists():
+        STATE_FILE.unlink()
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=2,
+        max_pages=5,
+        on_state_change=save_state_to_file,  # Save after each URL
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    print("\nStarting crawl with state persistence...")
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        results = await crawler.arun("https://books.toscrape.com", config=config)
+
+    # Show final state
+    if STATE_FILE.exists():
+        with open(STATE_FILE, "r") as f:
+            final_state = json.load(f)
+
+        print(f"\nFinal state saved to {STATE_FILE}:")
+        print(f"  - Strategy: {final_state['strategy_type']}")
+        print(f"  - Pages crawled: {final_state['pages_crawled']}")
+        print(f"  - URLs visited: {len(final_state['visited'])}")
+        print(f"  - URLs pending: {len(final_state['pending'])}")
+
+    print(f"\nCrawled {len(results)} pages total")
+
+
+async def example_crash_and_resume():
+    """
+    Example 2: Simulate a crash and resume from checkpoint.
+
+    This demonstrates the full crash recovery workflow:
+    1. Start crawling with state persistence
+    2. "Crash" after N pages
+    3. Resume from saved state
+    4. Verify no duplicate work
+    """
+    print("\n" + "=" * 60)
+    print("Example 2: Crash and Resume")
+    print("=" * 60)
+
+    # Clean up any previous state
+    if STATE_FILE.exists():
+        STATE_FILE.unlink()
+
+    crash_after = 3
+    crawled_urls_phase1: List[str] = []
+
+    async def save_and_maybe_crash(state: Dict[str, Any]) -> None:
+        """Save state, then simulate crash after N pages."""
+        # Always save state first
+        await save_state_to_file(state)
+        crawled_urls_phase1.clear()
+        crawled_urls_phase1.extend(state["visited"])
+
+        # Simulate crash after reaching threshold
+        if state["pages_crawled"] >= crash_after:
+            raise Exception("Simulated crash! (This is intentional)")
+
+    # Phase 1: Start crawl that will "crash"
+    print(f"\n--- Phase 1: Crawl until 'crash' after {crash_after} pages ---")
+
+    strategy1 = BFSDeepCrawlStrategy(
+        max_depth=2,
+        max_pages=10,
+        on_state_change=save_and_maybe_crash,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy1,
+        verbose=False,
+    )
+
+    try:
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            await crawler.arun("https://books.toscrape.com", config=config)
+    except Exception as e:
+        print(f"\n  Crash occurred: {e}")
+        print(f"  URLs crawled before crash: {len(crawled_urls_phase1)}")
+
+    # Phase 2: Resume from checkpoint
+    print("\n--- Phase 2: Resume from checkpoint ---")
+
+    saved_state = load_state_from_file()
+    if not saved_state:
+        print("  ERROR: No saved state found!")
+        return
+
+    print(f"  Loaded state: {saved_state['pages_crawled']} pages, {len(saved_state['pending'])} pending")
+
+    crawled_urls_phase2: List[str] = []
+
+    async def track_resumed_crawl(state: Dict[str, Any]) -> None:
+        """Track new URLs crawled in phase 2."""
+        await save_state_to_file(state)
+        new_urls = set(state["visited"]) - set(saved_state["visited"])
+        for url in new_urls:
+            if url not in crawled_urls_phase2:
+                crawled_urls_phase2.append(url)
+
+    strategy2 = BFSDeepCrawlStrategy(
+        max_depth=2,
+        max_pages=10,
+        resume_state=saved_state,  # Resume from checkpoint!
+        on_state_change=track_resumed_crawl,
+    )
+
+    config2 = CrawlerRunConfig(
+        deep_crawl_strategy=strategy2,
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        results = await crawler.arun("https://books.toscrape.com", config=config2)
+
+    # Verify no duplicates
+    already_crawled = set(saved_state["visited"])
+    duplicates = set(crawled_urls_phase2) & already_crawled
+
+    print(f"\n--- Results ---")
+    print(f"  Phase 1 URLs: {len(crawled_urls_phase1)}")
+    print(f"  Phase 2 new URLs: {len(crawled_urls_phase2)}")
+    print(f"  Duplicate crawls: {len(duplicates)} (should be 0)")
+    print(f"  Total results: {len(results)}")
+
+    if len(duplicates) == 0:
+        print("\n  SUCCESS: No duplicate work after resume!")
+    else:
+        print(f"\n  WARNING: Found duplicates: {duplicates}")
+
+
+async def example_export_state():
+    """
+    Example 3: Manual state export using export_state().
+
+    If you don't need real-time persistence, you can export
+    the state manually after the crawl completes.
+    """
+    print("\n" + "=" * 60)
+    print("Example 3: Manual State Export")
+    print("=" * 60)
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=1,
+        max_pages=3,
+        # No callback - state is still tracked internally
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    print("\nCrawling without callback...")
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        results = await crawler.arun("https://books.toscrape.com", config=config)
+
+    # Export state after crawl completes
+    # Note: This only works if on_state_change was set during crawl
+    # For this example, we'd need to set on_state_change to get state
+    print(f"\nCrawled {len(results)} pages")
+    print("(For manual export, set on_state_change to capture state)")
+
+
+async def example_state_structure():
+    """
+    Example 4: Understanding the state structure.
+
+    Shows the complete state dictionary that gets saved.
+    """
+    print("\n" + "=" * 60)
+    print("Example 4: State Structure")
+    print("=" * 60)
+
+    captured_state = None
+
+    async def capture_state(state: Dict[str, Any]) -> None:
+        nonlocal captured_state
+        captured_state = state
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=1,
+        max_pages=2,
+        on_state_change=capture_state,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        await crawler.arun("https://books.toscrape.com", config=config)
+
+    if captured_state:
+        print("\nState structure:")
+        print(json.dumps(captured_state, indent=2, default=str)[:1000] + "...")
+
+        print("\n\nKey fields:")
+        print(f"  strategy_type: '{captured_state['strategy_type']}'")
+        print(f"  visited: List of {len(captured_state['visited'])} URLs")
+        print(f"  pending: List of {len(captured_state['pending'])} queued items")
+        print(f"  depths: Dict mapping URL -> depth level")
+        print(f"  pages_crawled: {captured_state['pages_crawled']}")
+
+
+async def main():
+    """Run all examples."""
+    print("=" * 60)
+    print("Deep Crawl Crash Recovery Examples")
+    print("=" * 60)
+
+    await example_basic_state_persistence()
+    await example_crash_and_resume()
+    await example_state_structure()
+
+    # # Cleanup
+    # if STATE_FILE.exists():
+    #     STATE_FILE.unlink()
+    #     print(f"\n[Cleaned up {STATE_FILE}]")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/deepcrawl_example.py b/docs/examples/deepcrawl_example.py
new file mode 100644
index 0000000..741c003
--- /dev/null
+++ b/docs/examples/deepcrawl_example.py
@@ -0,0 +1,498 @@
+import asyncio
+import time
+
+from crawl4ai import CrawlerRunConfig, AsyncWebCrawler, CacheMode
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+from crawl4ai.deep_crawling.filters import (
+    FilterChain,
+    URLPatternFilter,
+    DomainFilter,
+    ContentTypeFilter,
+    ContentRelevanceFilter,
+    SEOFilter,
+)
+from crawl4ai.deep_crawling.scorers import (
+    KeywordRelevanceScorer,
+)
+
+
+# 1️⃣ Basic Deep Crawl Setup
+async def basic_deep_crawl():
+    """
+    PART 1: Basic Deep Crawl setup - Demonstrates a simple two-level deep crawl.
+
+    This function shows:
+    - How to set up BFSDeepCrawlStrategy (Breadth-First Search)
+    - Setting depth and domain parameters
+    - Processing the results to show the hierarchy
+    """
+    print("\n===== BASIC DEEP CRAWL SETUP =====")
+
+    # Configure a 2-level deep crawl using Breadth-First Search strategy
+    # max_depth=2 means: initial page (depth 0) + 2 more levels
+    # include_external=False means: only follow links within the same domain
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=2, include_external=False),
+        scraping_strategy=LXMLWebScrapingStrategy(),
+        verbose=True,  # Show progress during crawling
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        start_time = time.perf_counter()
+        results = await crawler.arun(url="https://docs.crawl4ai.com", config=config)
+
+        # Group results by depth to visualize the crawl tree
+        pages_by_depth = {}
+        for result in results:
+            depth = result.metadata.get("depth", 0)
+            if depth not in pages_by_depth:
+                pages_by_depth[depth] = []
+            pages_by_depth[depth].append(result.url)
+
+        print(f"✅ Crawled {len(results)} pages total")
+
+        # Display crawl structure by depth
+        for depth, urls in sorted(pages_by_depth.items()):
+            print(f"\nDepth {depth}: {len(urls)} pages")
+            # Show first 3 URLs for each depth as examples
+            for url in urls[:3]:
+                print(f"  → {url}")
+            if len(urls) > 3:
+                print(f"  ... and {len(urls) - 3} more")
+
+        print(
+            f"\n✅ Performance: {len(results)} pages in {time.perf_counter() - start_time:.2f} seconds"
+        )
+
+# 2️⃣ Stream vs. Non-Stream Execution
+async def stream_vs_nonstream():
+    """
+    PART 2: Demonstrates the difference between stream and non-stream execution.
+
+    Non-stream: Waits for all results before processing
+    Stream: Processes results as they become available
+    """
+    print("\n===== STREAM VS. NON-STREAM EXECUTION =====")
+
+    # Common configuration for both examples
+    base_config = CrawlerRunConfig(
+        deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=1, include_external=False),
+        scraping_strategy=LXMLWebScrapingStrategy(),
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        # NON-STREAMING MODE
+        print("\n📊 NON-STREAMING MODE:")
+        print("  In this mode, all results are collected before being returned.")
+
+        non_stream_config = base_config.clone()
+        non_stream_config.stream = False
+
+        start_time = time.perf_counter()
+        results = await crawler.arun(
+            url="https://docs.crawl4ai.com", config=non_stream_config
+        )
+
+        print(f"  ✅ Received all {len(results)} results at once")
+        print(f"  ✅ Total duration: {time.perf_counter() - start_time:.2f} seconds")
+
+        # STREAMING MODE
+        print("\n📊 STREAMING MODE:")
+        print("  In this mode, results are processed as they become available.")
+
+        stream_config = base_config.clone()
+        stream_config.stream = True
+
+        start_time = time.perf_counter()
+        result_count = 0
+        first_result_time = None
+
+        async for result in await crawler.arun(
+            url="https://docs.crawl4ai.com", config=stream_config
+        ):
+            result_count += 1
+            if result_count == 1:
+                first_result_time = time.perf_counter() - start_time
+                print(
+                    f"  ✅ First result received after {first_result_time:.2f} seconds: {result.url}"
+                )
+            elif result_count % 5 == 0:  # Show every 5th result for brevity
+                print(f"  → Result #{result_count}: {result.url}")
+
+        print(f"  ✅ Total: {result_count} results")
+        print(f"  ✅ First result: {first_result_time:.2f} seconds")
+        print(f"  ✅ All results: {time.perf_counter() - start_time:.2f} seconds")
+        print("\n🔍 Key Takeaway: Streaming allows processing results immediately")
+
+# 3️⃣ Introduce Filters & Scorers
+async def filters_and_scorers():
+    """
+    PART 3: Demonstrates the use of filters and scorers for more targeted crawling.
+
+    This function progressively adds:
+    1. A single URL pattern filter
+    2. Multiple filters in a chain
+    3. Scorers for prioritizing pages
+    """
+    print("\n===== FILTERS AND SCORERS =====")
+
+    async with AsyncWebCrawler() as crawler:
+        # SINGLE FILTER EXAMPLE
+        print("\n📊 EXAMPLE 1: SINGLE URL PATTERN FILTER")
+        print("  Only crawl pages containing 'core' in the URL")
+
+        # Create a filter that only allows URLs with 'guide' in them
+        url_filter = URLPatternFilter(patterns=["*core*"])
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=BFSDeepCrawlStrategy(
+                max_depth=1,
+                include_external=False,
+                filter_chain=FilterChain([url_filter]),  # Single filter
+            ),
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            cache_mode=CacheMode.BYPASS,
+            verbose=True,
+        )
+
+        results = await crawler.arun(url="https://docs.crawl4ai.com", config=config)
+
+        print(f"  ✅ Crawled {len(results)} pages matching '*core*'")
+        for result in results[:3]:  # Show first 3 results
+            print(f"  → {result.url}")
+        if len(results) > 3:
+            print(f"  ... and {len(results) - 3} more")
+
+        # MULTIPLE FILTERS EXAMPLE
+        print("\n📊 EXAMPLE 2: MULTIPLE FILTERS IN A CHAIN")
+        print("  Only crawl pages that:")
+        print("  1. Contain '2024' in the URL")
+        print("  2. Are from 'techcrunch.com'")
+        print("  3. Are of text/html or application/javascript content type")
+
+        # Create a chain of filters
+        filter_chain = FilterChain(
+            [
+                URLPatternFilter(patterns=["*2024*"]),
+                DomainFilter(
+                    allowed_domains=["techcrunch.com"],
+                    blocked_domains=["guce.techcrunch.com", "oidc.techcrunch.com"],
+                ),
+                ContentTypeFilter(
+                    allowed_types=["text/html", "application/javascript"]
+                ),
+            ]
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=BFSDeepCrawlStrategy(
+                max_depth=1, include_external=False, filter_chain=filter_chain
+            ),
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            verbose=True,
+        )
+
+        results = await crawler.arun(url="https://techcrunch.com", config=config)
+
+        print(f"  ✅ Crawled {len(results)} pages after applying all filters")
+        for result in results[:3]:
+            print(f"  → {result.url}")
+        if len(results) > 3:
+            print(f"  ... and {len(results) - 3} more")
+
+        # SCORERS EXAMPLE
+        print("\n📊 EXAMPLE 3: USING A KEYWORD RELEVANCE SCORER")
+        print(
+            "Score pages based on relevance to keywords: 'crawl', 'example', 'async', 'configuration','javascript','css'"
+        )
+
+        # Create a keyword relevance scorer
+        keyword_scorer = KeywordRelevanceScorer(
+            keywords=["crawl", "example", "async", "configuration","javascript","css"], weight=1
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=BestFirstCrawlingStrategy(  
+                max_depth=1, include_external=False, url_scorer=keyword_scorer
+            ),
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            cache_mode=CacheMode.BYPASS,
+            verbose=True,
+            stream=True,
+        )
+
+        results = []
+        async for result in await crawler.arun(
+            url="https://docs.crawl4ai.com", config=config
+        ):
+            results.append(result)
+            score = result.metadata.get("score")
+            print(f"  → Score: {score:.2f} | {result.url}")
+
+        print(f"  ✅ Crawler prioritized {len(results)} pages by relevance score")
+        print("  🔍 Note: BestFirstCrawlingStrategy visits highest-scoring pages first")
+
+# 4️⃣ Advanced Filters
+async def advanced_filters():
+    """
+    PART 4: Demonstrates advanced filtering techniques for specialized crawling.
+
+    This function covers:
+    - SEO filters
+    - Text relevancy filtering
+    - Combining advanced filters
+    """
+    print("\n===== ADVANCED FILTERS =====")
+
+    async with AsyncWebCrawler() as crawler:
+        # SEO FILTER EXAMPLE
+        print("\n📊 EXAMPLE 1: SEO FILTERS")
+        print(
+            "Quantitative SEO quality assessment filter based searching keywords in the head section"
+        )
+
+        seo_filter = SEOFilter(
+            threshold=0.5, keywords=["dynamic", "interaction", "javascript"]
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=BFSDeepCrawlStrategy(
+                max_depth=1, filter_chain=FilterChain([seo_filter])
+            ),
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            verbose=True,
+            cache_mode=CacheMode.BYPASS,
+        )
+
+        results = await crawler.arun(url="https://docs.crawl4ai.com", config=config)
+
+        print(f"  ✅ Found {len(results)} pages with relevant keywords")
+        for result in results:
+            print(f"  → {result.url}")
+
+        # ADVANCED TEXT RELEVANCY FILTER
+        print("\n📊 EXAMPLE 2: ADVANCED TEXT RELEVANCY FILTER")
+
+        # More sophisticated content relevance filter
+        relevance_filter = ContentRelevanceFilter(
+            query="Interact with the web using your authentic digital identity",
+            threshold=0.7,
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=BFSDeepCrawlStrategy(
+                max_depth=1, filter_chain=FilterChain([relevance_filter])
+            ),
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            verbose=True,
+            cache_mode=CacheMode.BYPASS,
+        )
+
+        results = await crawler.arun(url="https://docs.crawl4ai.com", config=config)
+
+        print(f"  ✅ Found {len(results)} pages")
+        for result in results:
+            relevance_score = result.metadata.get("relevance_score", 0)
+            print(f"  → Score: {relevance_score:.2f} | {result.url}")
+
+# 5️⃣ Max Pages and Score Thresholds
+async def max_pages_and_thresholds():
+    """
+    PART 5: Demonstrates using max_pages and score_threshold parameters with different strategies.
+    
+    This function shows:
+    - How to limit the number of pages crawled
+    - How to set score thresholds for more targeted crawling
+    - Comparing BFS, DFS, and Best-First strategies with these parameters
+    """
+    print("\n===== MAX PAGES AND SCORE THRESHOLDS =====")
+    
+    from crawl4ai.deep_crawling import DFSDeepCrawlStrategy
+    
+    async with AsyncWebCrawler() as crawler:
+        # Define a common keyword scorer for all examples
+        keyword_scorer = KeywordRelevanceScorer(
+            keywords=["browser", "crawler", "web", "automation"], 
+            weight=1.0
+        )
+        
+        # EXAMPLE 1: BFS WITH MAX PAGES
+        print("\n📊 EXAMPLE 1: BFS STRATEGY WITH MAX PAGES LIMIT")
+        print("  Limit the crawler to a maximum of 5 pages")
+        
+        bfs_config = CrawlerRunConfig(
+            deep_crawl_strategy=BFSDeepCrawlStrategy(
+                max_depth=2, 
+                include_external=False,
+                url_scorer=keyword_scorer,
+                max_pages=5  # Only crawl 5 pages
+            ),
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            verbose=True,
+            cache_mode=CacheMode.BYPASS,
+        )
+        
+        results = await crawler.arun(url="https://docs.crawl4ai.com", config=bfs_config)
+        
+        print(f"  ✅ Crawled exactly {len(results)} pages as specified by max_pages")
+        for result in results:
+            depth = result.metadata.get("depth", 0)
+            print(f"  → Depth: {depth} | {result.url}")
+            
+        # EXAMPLE 2: DFS WITH SCORE THRESHOLD
+        print("\n📊 EXAMPLE 2: DFS STRATEGY WITH SCORE THRESHOLD")
+        print("  Only crawl pages with a relevance score above 0.5")
+        
+        dfs_config = CrawlerRunConfig(
+            deep_crawl_strategy=DFSDeepCrawlStrategy(
+                max_depth=2,
+                include_external=False, 
+                url_scorer=keyword_scorer,
+                score_threshold=0.7,  # Only process URLs with scores above 0.5
+                max_pages=10
+            ),
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            verbose=True,
+            cache_mode=CacheMode.BYPASS,
+        )
+        
+        results = await crawler.arun(url="https://docs.crawl4ai.com", config=dfs_config)
+        
+        print(f"  ✅ Crawled {len(results)} pages with scores above threshold")
+        for result in results:
+            score = result.metadata.get("score", 0)
+            depth = result.metadata.get("depth", 0)
+            print(f"  → Depth: {depth} | Score: {score:.2f} | {result.url}")
+            
+        # EXAMPLE 3: BEST-FIRST WITH BOTH CONSTRAINTS
+        print("\n📊 EXAMPLE 3: BEST-FIRST STRATEGY WITH BOTH CONSTRAINTS")
+        print("  Limit to 7 pages with scores above 0.3, prioritizing highest scores")
+        
+        bf_config = CrawlerRunConfig(
+            deep_crawl_strategy=BestFirstCrawlingStrategy(
+                max_depth=2,
+                include_external=False,
+                url_scorer=keyword_scorer,
+                max_pages=7,          # Limit to 7 pages total
+            ),
+            scraping_strategy=LXMLWebScrapingStrategy(),
+            verbose=True,
+            cache_mode=CacheMode.BYPASS,
+            stream=True,
+        )
+        
+        results = []
+        async for result in await crawler.arun(url="https://docs.crawl4ai.com", config=bf_config):
+            results.append(result)
+            score = result.metadata.get("score", 0)
+            depth = result.metadata.get("depth", 0)
+            print(f"  → Depth: {depth} | Score: {score:.2f} | {result.url}")
+            
+        print(f"  ✅ Crawled {len(results)} high-value pages with scores above 0.3")
+        if results:
+            avg_score = sum(r.metadata.get('score', 0) for r in results) / len(results)
+            print(f"  ✅ Average score: {avg_score:.2f}")
+            print("  🔍 Note: BestFirstCrawlingStrategy visited highest-scoring pages first")
+
+# 6️⃣ Wrap-Up and Key Takeaways
+async def wrap_up():
+    """
+    PART 6: Wrap-Up and Key Takeaways
+
+    Summarize the key concepts learned in this tutorial.
+    """
+    print("\n===== COMPLETE CRAWLER EXAMPLE =====")
+    print("Combining filters, scorers, and streaming for an optimized crawl")
+
+    # Create a sophisticated filter chain
+    filter_chain = FilterChain(
+        [
+            DomainFilter(
+                allowed_domains=["docs.crawl4ai.com"],
+                blocked_domains=["old.docs.crawl4ai.com"],
+            ),
+            URLPatternFilter(patterns=["*core*", "*advanced*", "*blog*"]),
+            ContentTypeFilter(allowed_types=["text/html"]),
+        ]
+    )
+
+    # Create a composite scorer that combines multiple scoring strategies
+    keyword_scorer = KeywordRelevanceScorer(
+        keywords=["crawl", "example", "async", "configuration"], weight=0.7
+    )
+    # Set up the configuration
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=BestFirstCrawlingStrategy(
+            max_depth=1,
+            include_external=False,
+            filter_chain=filter_chain,
+            url_scorer=keyword_scorer,
+        ),
+        scraping_strategy=LXMLWebScrapingStrategy(),
+        stream=True,
+        verbose=True,
+    )
+
+    # Execute the crawl
+    results = []
+    start_time = time.perf_counter()
+
+    async with AsyncWebCrawler() as crawler:
+        async for result in await crawler.arun(
+            url="https://docs.crawl4ai.com", config=config
+        ):
+            results.append(result)
+            score = result.metadata.get("score", 0)
+            depth = result.metadata.get("depth", 0)
+            print(f"→ Depth: {depth} | Score: {score:.2f} | {result.url}")
+
+    duration = time.perf_counter() - start_time
+
+    # Summarize the results
+    print(f"\n✅ Crawled {len(results)} high-value pages in {duration:.2f} seconds")
+    print(
+        f"✅ Average score: {sum(r.metadata.get('score', 0) for r in results) / len(results):.2f}"
+    )
+
+    # Group by depth
+    depth_counts = {}
+    for result in results:
+        depth = result.metadata.get("depth", 0)
+        depth_counts[depth] = depth_counts.get(depth, 0) + 1
+
+    print("\n📊 Pages crawled by depth:")
+    for depth, count in sorted(depth_counts.items()):
+        print(f"  Depth {depth}: {count} pages")
+
+
+async def run_tutorial():
+    """
+    Executes all tutorial sections in sequence.
+    """
+    print("\n🚀 CRAWL4AI DEEP CRAWLING TUTORIAL 🚀")
+    print("======================================")
+    print("This tutorial will walk you through deep crawling techniques,")
+    print("from basic to advanced, using the Crawl4AI library.")
+
+    # Define sections - uncomment to run specific parts during development
+    tutorial_sections = [
+        basic_deep_crawl,
+        stream_vs_nonstream,
+        filters_and_scorers,
+        max_pages_and_thresholds, 
+        advanced_filters,
+        wrap_up,
+    ]
+
+    for section in tutorial_sections:
+        await section()
+
+    print("\n🎉 TUTORIAL COMPLETE! 🎉")
+    print("You now have a comprehensive understanding of deep crawling with Crawl4AI.")
+    print("For more information, check out https://docs.crawl4ai.com")
+
+# Execute the tutorial when run directly
+if __name__ == "__main__":
+    asyncio.run(run_tutorial())
\ No newline at end of file
diff --git a/docs/examples/demo_multi_config_clean.py b/docs/examples/demo_multi_config_clean.py
new file mode 100644
index 0000000..09df71c
--- /dev/null
+++ b/docs/examples/demo_multi_config_clean.py
@@ -0,0 +1,303 @@
+"""
+🎯 Multi-Config URL Matching Demo
+=================================
+Learn how to use different crawler configurations for different URL patterns
+in a single crawl batch with Crawl4AI's multi-config feature.
+
+Part 1: Understanding URL Matching (Pattern Testing)
+Part 2: Practical Example with Real Crawling
+"""
+
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler, 
+    CrawlerRunConfig,
+    MatchMode
+)
+from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+
+def print_section(title):
+    """Print a formatted section header"""
+    print(f"\n{'=' * 60}")
+    print(f"{title}")
+    print(f"{'=' * 60}\n")
+
+
+def test_url_matching(config, test_urls, config_name):
+    """Test URL matching for a config and show results"""
+    print(f"Config: {config_name}")
+    print(f"Matcher: {config.url_matcher}")
+    if hasattr(config, 'match_mode'):
+        print(f"Mode: {config.match_mode.value}")
+    print("-" * 40)
+    
+    for url in test_urls:
+        matches = config.is_match(url)
+        symbol = "✓" if matches else "✗"
+        print(f"{symbol} {url}")
+    print()
+
+
+# ==============================================================================
+# PART 1: Understanding URL Matching
+# ==============================================================================
+
+def demo_part1_pattern_matching():
+    """Part 1: Learn how URL matching works without crawling"""
+    
+    print_section("PART 1: Understanding URL Matching")
+    print("Let's explore different ways to match URLs with configs.\n")
+    
+    # Test URLs we'll use throughout
+    test_urls = [
+        "https://example.com/report.pdf",
+        "https://example.com/data.json",
+        "https://example.com/blog/post-1",
+        "https://example.com/article/news",
+        "https://api.example.com/v1/users",
+        "https://example.com/about"
+    ]
+    
+    # 1.1 Simple String Pattern
+    print("1.1 Simple String Pattern Matching")
+    print("-" * 40)
+    
+    pdf_config = CrawlerRunConfig(
+        url_matcher="*.pdf"
+    )
+    
+    test_url_matching(pdf_config, test_urls, "PDF Config")
+    
+    
+    # 1.2 Multiple String Patterns
+    print("1.2 Multiple String Patterns (OR logic)")
+    print("-" * 40)
+    
+    blog_config = CrawlerRunConfig(
+        url_matcher=["*/blog/*", "*/article/*", "*/news/*"],
+        match_mode=MatchMode.OR  # This is default, shown for clarity
+    )
+    
+    test_url_matching(blog_config, test_urls, "Blog/Article Config")
+    
+    
+    # 1.3 Single Function Matcher
+    print("1.3 Function-based Matching")
+    print("-" * 40)
+    
+    api_config = CrawlerRunConfig(
+        url_matcher=lambda url: 'api' in url or url.endswith('.json')
+    )
+    
+    test_url_matching(api_config, test_urls, "API Config")
+    
+    
+    # 1.4 List of Functions
+    print("1.4 Multiple Functions with AND Logic")
+    print("-" * 40)
+    
+    # Must be HTTPS AND contain 'api' AND have version number
+    secure_api_config = CrawlerRunConfig(
+        url_matcher=[
+            lambda url: url.startswith('https://'),
+            lambda url: 'api' in url,
+            lambda url: '/v' in url  # Version indicator
+        ],
+        match_mode=MatchMode.AND
+    )
+    
+    test_url_matching(secure_api_config, test_urls, "Secure API Config")
+    
+    
+    # 1.5 Mixed: String and Function Together
+    print("1.5 Mixed Patterns: String + Function")
+    print("-" * 40)
+    
+    # Match JSON files OR any API endpoint
+    json_or_api_config = CrawlerRunConfig(
+        url_matcher=[
+            "*.json",  # String pattern
+            lambda url: 'api' in url  # Function
+        ],
+        match_mode=MatchMode.OR
+    )
+    
+    test_url_matching(json_or_api_config, test_urls, "JSON or API Config")
+    
+    
+    # 1.6 Complex: Multiple Strings + Multiple Functions
+    print("1.6 Complex Matcher: Mixed Types with AND Logic")
+    print("-" * 40)
+    
+    # Must be: HTTPS AND (.com domain) AND (blog OR article) AND NOT a PDF
+    complex_config = CrawlerRunConfig(
+        url_matcher=[
+            lambda url: url.startswith('https://'),  # Function: HTTPS check
+            "*.com/*",  # String: .com domain
+            lambda url: any(pattern in url for pattern in ['/blog/', '/article/']),  # Function: Blog OR article
+            lambda url: not url.endswith('.pdf')  # Function: Not PDF
+        ],
+        match_mode=MatchMode.AND
+    )
+    
+    test_url_matching(complex_config, test_urls, "Complex Mixed Config")
+    
+    print("\n✅ Key Takeaway: First matching config wins when passed to arun_many()!")
+
+
+# ==============================================================================
+# PART 2: Practical Multi-URL Crawling
+# ==============================================================================
+
+async def demo_part2_practical_crawling():
+    """Part 2: Real-world example with different content types"""
+    
+    print_section("PART 2: Practical Multi-URL Crawling")
+    print("Now let's see multi-config in action with real URLs.\n")
+    
+    # Create specialized configs for different content types
+    configs = [
+        # Config 1: PDF documents - only match files ending with .pdf
+        CrawlerRunConfig(
+            url_matcher="*.pdf",
+            scraping_strategy=PDFContentScrapingStrategy()
+        ),
+        
+        # Config 2: Blog/article pages with content filtering
+        CrawlerRunConfig(
+            url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(threshold=0.48)
+            )
+        ),
+        
+        # Config 3: Dynamic pages requiring JavaScript
+        CrawlerRunConfig(
+            url_matcher=lambda url: 'github.com' in url,
+            js_code="window.scrollTo(0, 500);"  # Scroll to load content
+        ),
+        
+        # Config 4: Mixed matcher - API endpoints (string OR function)
+        CrawlerRunConfig(
+            url_matcher=[
+                "*.json",  # String pattern for JSON files
+                lambda url: 'api' in url or 'httpbin.org' in url  # Function for API endpoints
+            ],
+            match_mode=MatchMode.OR,
+        ),
+        
+        # Config 5: Complex matcher - Secure documentation sites
+        CrawlerRunConfig(
+            url_matcher=[
+                lambda url: url.startswith('https://'),  # Must be HTTPS
+                "*.org/*",  # String: .org domain
+                lambda url: any(doc in url for doc in ['docs', 'documentation', 'reference']),  # Has docs
+                lambda url: not url.endswith(('.pdf', '.json'))  # Not PDF or JSON
+            ],
+            match_mode=MatchMode.AND,
+            # wait_for="css:.content, css:article"  # Wait for content to load
+        ),
+        
+        # Default config for everything else
+        # CrawlerRunConfig()  # No url_matcher means it matches everything (use it as fallback)
+    ]
+    
+    # URLs to crawl - each will use a different config
+    urls = [
+        "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",  # → PDF config
+        "https://blog.python.org/",  # → Blog config with content filter
+        "https://github.com/microsoft/playwright",  # → JS config
+        "https://httpbin.org/json",  # → Mixed matcher config (API)
+        "https://docs.python.org/3/reference/",  # → Complex matcher config
+        "https://www.w3schools.com/",  # → Default config, if you uncomment the default config line above, if not you will see `Error: No matching configuration`
+    ]
+    
+    print("URLs to crawl:")
+    for i, url in enumerate(urls, 1):
+        print(f"{i}. {url}")
+    
+    print("\nCrawling with appropriate config for each URL...\n")
+    
+    async with AsyncWebCrawler() as crawler:
+        results = await crawler.arun_many(
+            urls=urls,
+            config=configs
+        )
+        
+        # Display results
+        print("Results:")
+        print("-" * 60)
+        
+        for result in results:
+            if result.success:
+                # Determine which config was used
+                config_type = "Default"
+                if result.url.endswith('.pdf'):
+                    config_type = "PDF Strategy"
+                elif any(pattern in result.url for pattern in ['blog', 'python.org']) and 'docs' not in result.url:
+                    config_type = "Blog + Content Filter"
+                elif 'github.com' in result.url:
+                    config_type = "JavaScript Enabled"
+                elif 'httpbin.org' in result.url or result.url.endswith('.json'):
+                    config_type = "Mixed Matcher (API)"
+                elif 'docs.python.org' in result.url:
+                    config_type = "Complex Matcher (Secure Docs)"
+                
+                print(f"\n✓ {result.url}")
+                print(f"  Config used: {config_type}")
+                print(f"  Content size: {len(result.markdown)} chars")
+                
+                # Show if we have fit_markdown (from content filter)
+                if hasattr(result.markdown, 'fit_markdown') and result.markdown.fit_markdown:
+                    print(f"  Fit markdown size: {len(result.markdown.fit_markdown)} chars")
+                    reduction = (1 - len(result.markdown.fit_markdown) / len(result.markdown)) * 100
+                    print(f"  Content reduced by: {reduction:.1f}%")
+                
+                # Show extracted data if using extraction strategy
+                if hasattr(result, 'extracted_content') and result.extracted_content:
+                    print(f"  Extracted data: {str(result.extracted_content)[:100]}...")
+            else:
+                print(f"\n✗ {result.url}")
+                print(f"  Error: {result.error_message}")
+    
+    print("\n" + "=" * 60)
+    print("✅ Multi-config crawling complete!")
+    print("\nBenefits demonstrated:")
+    print("- PDFs handled with specialized scraper")
+    print("- Blog content filtered for relevance") 
+    print("- JavaScript executed only where needed")
+    print("- Mixed matchers (string + function) for flexible matching")
+    print("- Complex matchers for precise URL targeting")
+    print("- Each URL got optimal configuration automatically!")
+
+
+async def main():
+    """Run both parts of the demo"""
+    
+    print("""
+🎯 Multi-Config URL Matching Demo
+=================================
+Learn how Crawl4AI can use different configurations
+for different URLs in a single batch.
+    """)
+    
+    # Part 1: Pattern matching
+    demo_part1_pattern_matching()
+    
+    print("\nPress Enter to continue to Part 2...")
+    try:
+        input()
+    except EOFError:
+        # Running in non-interactive mode, skip input
+        pass
+    
+    # Part 2: Practical crawling
+    await demo_part2_practical_crawling()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/dfs_crawl_demo.py b/docs/examples/dfs_crawl_demo.py
new file mode 100644
index 0000000..321c413
--- /dev/null
+++ b/docs/examples/dfs_crawl_demo.py
@@ -0,0 +1,39 @@
+"""
+Simple demonstration of the DFS deep crawler visiting multiple pages.
+
+Run with:  python docs/examples/dfs_crawl_demo.py
+"""
+import asyncio
+
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+from crawl4ai.cache_context import CacheMode
+from crawl4ai.deep_crawling.dfs_strategy import DFSDeepCrawlStrategy
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+
+async def main() -> None:
+    dfs_strategy = DFSDeepCrawlStrategy(
+        max_depth=3,
+        max_pages=50,
+        include_external=False,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=dfs_strategy,
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(),
+        stream=True,
+    )
+
+    seed_url = "https://docs.python.org/3/"  # Plenty of internal links
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        async for result in await crawler.arun(url=seed_url, config=config):
+            depth = result.metadata.get("depth")
+            status = "SUCCESS" if result.success else "FAILED"
+            print(f"[{status}] depth={depth} url={result.url}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/dispatcher_example.py b/docs/examples/dispatcher_example.py
new file mode 100644
index 0000000..8ac24d3
--- /dev/null
+++ b/docs/examples/dispatcher_example.py
@@ -0,0 +1,136 @@
+import asyncio
+import time
+from rich import print
+from rich.table import Table
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    MemoryAdaptiveDispatcher,
+    SemaphoreDispatcher,
+    RateLimiter,
+    CrawlerMonitor,
+    DisplayMode,
+    CacheMode,
+    LXMLWebScrapingStrategy,
+)
+
+
+async def memory_adaptive(urls, browser_config, run_config):
+    """Memory adaptive crawler with monitoring"""
+    start = time.perf_counter()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        dispatcher = MemoryAdaptiveDispatcher(
+            memory_threshold_percent=70.0,
+            max_session_permit=10,
+            monitor=CrawlerMonitor(
+                max_visible_rows=15, display_mode=DisplayMode.DETAILED
+            ),
+        )
+        results = await crawler.arun_many(
+            urls, config=run_config, dispatcher=dispatcher
+        )
+    duration = time.perf_counter() - start
+    return len(results), duration
+
+
+async def memory_adaptive_with_rate_limit(urls, browser_config, run_config):
+    """Memory adaptive crawler with rate limiting"""
+    start = time.perf_counter()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        dispatcher = MemoryAdaptiveDispatcher(
+            memory_threshold_percent=95.0,
+            max_session_permit=10,
+            rate_limiter=RateLimiter(
+                base_delay=(1.0, 2.0), max_delay=30.0, max_retries=2
+            ),
+            monitor=CrawlerMonitor(
+                max_visible_rows=15, display_mode=DisplayMode.DETAILED
+            ),
+        )
+        results = await crawler.arun_many(
+            urls, config=run_config, dispatcher=dispatcher
+        )
+    duration = time.perf_counter() - start
+    return len(results), duration
+
+
+async def semaphore(urls, browser_config, run_config):
+    """Basic semaphore crawler"""
+    start = time.perf_counter()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        dispatcher = SemaphoreDispatcher(
+            semaphore_count=5,
+            monitor=CrawlerMonitor(
+                max_visible_rows=15, display_mode=DisplayMode.DETAILED
+            ),
+        )
+        results = await crawler.arun_many(
+            urls, config=run_config, dispatcher=dispatcher
+        )
+    duration = time.perf_counter() - start
+    return len(results), duration
+
+
+async def semaphore_with_rate_limit(urls, browser_config, run_config):
+    """Semaphore crawler with rate limiting"""
+    start = time.perf_counter()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        dispatcher = SemaphoreDispatcher(
+            semaphore_count=5,
+            rate_limiter=RateLimiter(
+                base_delay=(1.0, 2.0), max_delay=30.0, max_retries=2
+            ),
+            monitor=CrawlerMonitor(
+                max_visible_rows=15, display_mode=DisplayMode.DETAILED
+            ),
+        )
+        results = await crawler.arun_many(
+            urls, config=run_config, dispatcher=dispatcher
+        )
+    duration = time.perf_counter() - start
+    return len(results), duration
+
+
+def create_performance_table(results):
+    """Creates a rich table showing performance results"""
+    table = Table(title="Crawler Strategy Performance Comparison")
+    table.add_column("Strategy", style="cyan")
+    table.add_column("URLs Crawled", justify="right", style="green")
+    table.add_column("Time (seconds)", justify="right", style="yellow")
+    table.add_column("URLs/second", justify="right", style="magenta")
+
+    sorted_results = sorted(results.items(), key=lambda x: x[1][1])
+
+    for strategy, (urls_crawled, duration) in sorted_results:
+        urls_per_second = urls_crawled / duration
+        table.add_row(
+            strategy, str(urls_crawled), f"{duration:.2f}", f"{urls_per_second:.2f}"
+        )
+
+    return table
+
+
+async def main():
+    urls = [f"https://example.com/page{i}" for i in range(1, 40)]
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, scraping_strategy=LXMLWebScrapingStrategy())
+
+    results = {
+        "Memory Adaptive": await memory_adaptive(urls, browser_config, run_config),
+        # "Memory Adaptive + Rate Limit": await memory_adaptive_with_rate_limit(
+        #     urls, browser_config, run_config
+        # ),
+        # "Semaphore": await semaphore(urls, browser_config, run_config),
+        # "Semaphore + Rate Limit": await semaphore_with_rate_limit(
+        #     urls, browser_config, run_config
+        # ),
+    }
+
+    table = create_performance_table(results)
+    print("\nPerformance Summary:")
+    print(table)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/docker/demo_docker_api.py b/docs/examples/docker/demo_docker_api.py
new file mode 100644
index 0000000..0a3d51a
--- /dev/null
+++ b/docs/examples/docker/demo_docker_api.py
@@ -0,0 +1,1317 @@
+import asyncio
+import httpx
+import json
+import os
+import time
+from typing import List, Dict, Any, AsyncGenerator, Optional
+import textwrap          # ← new: for pretty code literals
+import urllib.parse  # ← needed for URL-safe /llm calls
+from dotenv import load_dotenv
+from rich.console import Console
+from rich.syntax import Syntax
+from rich.panel import Panel
+from rich.table import Table
+
+# --- Setup & Configuration ---
+load_dotenv()  # Load environment variables from .env file
+
+console = Console()
+
+# --- Configuration ---
+BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:8020")
+BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:11235")
+# Target URLs
+SIMPLE_URL = "https://example.com"  # For demo purposes
+SIMPLE_URL = "https://httpbin.org/html"
+LINKS_URL = "https://httpbin.org/links/10/0"
+FORMS_URL = "https://httpbin.org/forms/post"  # For JS demo
+BOOKS_URL = "http://books.toscrape.com/"  # For CSS extraction
+PYTHON_URL = "https://python.org"  # For deeper crawl
+# Use the same sample site as deep crawl tests for consistency
+DEEP_CRAWL_BASE_URL = os.getenv(
+    "DEEP_CRAWL_TEST_SITE", "https://docs.crawl4ai.com/samples/deepcrawl/")
+DEEP_CRAWL_DOMAIN = "docs.crawl4ai.com"
+
+# --- Helper Functions ---
+
+
+async def check_server_health(client: httpx.AsyncClient):
+    """Check if the server is healthy before running tests."""
+    console.print("[bold cyan]Checking server health...[/]", end="")
+    try:
+        response = await client.get("/health", timeout=10.0)
+        response.raise_for_status()
+        health_data = response.json()
+        console.print(
+            f"[bold green] Server OK! Version: {health_data.get('version', 'N/A')}[/]")
+        return True
+    except (httpx.RequestError, httpx.HTTPStatusError) as e:
+        console.print(f"\n[bold red]Server health check FAILED:[/]")
+        console.print(f"Error: {e}")
+        console.print(f"Is the server running at {BASE_URL}?")
+        return False
+    except Exception as e:
+        console.print(
+            f"\n[bold red]An unexpected error occurred during health check:[/]")
+        console.print(e)
+        return False
+
+
+def print_payload(payload: Dict[str, Any]):
+    """Prints the JSON payload nicely with a dark theme."""
+    syntax = Syntax(
+        json.dumps(payload, indent=2),
+        "json",
+        theme="monokai",  # <--- Changed theme here
+        line_numbers=False,
+        word_wrap=True      # Added word wrap for potentially long payloads
+    )
+    console.print(Panel(syntax, title="Request Payload",
+                  border_style="blue", expand=False))
+
+
+def print_result_summary(results: List[Dict[str, Any]], title: str = "Crawl Results Summary", max_items: int = 3):
+    """Prints a concise summary of crawl results."""
+    if not results:
+        console.print(f"[yellow]{title}: No results received.[/]")
+        return
+
+    console.print(Panel(f"[bold]{title}[/]",
+                  border_style="green", expand=False))
+    count = 0
+    for result in results:
+        if count >= max_items:
+            console.print(
+                f"... (showing first {max_items} of {len(results)} results)")
+            break
+        count += 1
+        success_icon = "[green]✔[/]" if result.get('success') else "[red]✘[/]"
+        url = result.get('url', 'N/A')
+        status = result.get('status_code', 'N/A')
+        content_info = ""
+        if result.get('extracted_content'):
+            content_str = json.dumps(result['extracted_content'])
+            snippet = (
+                content_str[:70] + '...') if len(content_str) > 70 else content_str
+            content_info = f" | Extracted: [cyan]{snippet}[/]"
+        elif result.get('markdown'):
+            content_info = f" | Markdown: [cyan]Present[/]"
+        elif result.get('html'):
+            content_info = f" | HTML Size: [cyan]{len(result['html'])}[/]"
+
+        console.print(
+            f"{success_icon} URL: [link={url}]{url}[/link] (Status: {status}){content_info}")
+        if "metadata" in result and "depth" in result["metadata"]:
+            console.print(f"  Depth: {result['metadata']['depth']}")
+        if not result.get('success') and result.get('error_message'):
+            console.print(f"  [red]Error: {result['error_message']}[/]")
+
+
+async def make_request(client: httpx.AsyncClient, endpoint: str, payload: Dict[str, Any], title: str) -> Optional[List[Dict[str, Any]]]:
+    """Handles non-streaming POST requests."""
+    console.rule(f"[bold blue]{title}[/]", style="blue")
+    print_payload(payload)
+    console.print(f"Sending POST request to {client.base_url}{endpoint}...")
+    try:
+        start_time = time.time()
+        response = await client.post(endpoint, json=payload)
+        duration = time.time() - start_time
+        console.print(
+            f"Response Status: [bold {'green' if response.is_success else 'red'}]{response.status_code}[/] (took {duration:.2f}s)")
+        response.raise_for_status()
+        data = response.json()
+        if data.get("success"):
+            results = data.get("results", [])
+            print_result_summary(results, title=f"{title} Results")
+            return results
+        else:
+            console.print("[bold red]Request reported failure:[/]")
+            console.print(data)
+            return None
+    except httpx.HTTPStatusError as e:
+        console.print(f"[bold red]HTTP Error:[/]")
+        console.print(f"Status: {e.response.status_code}")
+        try:
+            console.print(Panel(Syntax(json.dumps(
+                e.response.json(), indent=2), "json", theme="default"), title="Error Response"))
+        except json.JSONDecodeError:
+            console.print(f"Response Body: {e.response.text}")
+    except httpx.RequestError as e:
+        console.print(f"[bold red]Request Error: {e}[/]")
+    except Exception as e:
+        console.print(f"[bold red]Unexpected Error: {e}[/]")
+    return None
+
+
+async def stream_request(client: httpx.AsyncClient, endpoint: str, payload: Dict[str, Any], title: str):
+    """Handles streaming POST requests."""
+    console.rule(f"[bold magenta]{title}[/]", style="magenta")
+    print_payload(payload)
+    console.print(
+        f"Sending POST stream request to {client.base_url}{endpoint}...")
+    all_results = []
+    initial_status_code = None  # Store initial status code
+
+    try:
+        start_time = time.time()
+        async with client.stream("POST", endpoint, json=payload) as response:
+            initial_status_code = response.status_code  # Capture initial status
+            duration = time.time() - start_time  # Time to first byte potentially
+            console.print(
+                f"Initial Response Status: [bold {'green' if response.is_success else 'red'}]{initial_status_code}[/] (first byte ~{duration:.2f}s)")
+            response.raise_for_status()  # Raise exception for bad *initial* status codes
+
+            console.print("[magenta]--- Streaming Results ---[/]")
+            completed = False
+            async for line in response.aiter_lines():
+                if line:
+                    try:
+                        data = json.loads(line)
+                        if data.get("status") == "completed":
+                            completed = True
+                            console.print(
+                                "[bold green]--- Stream Completed ---[/]")
+                            break
+                        elif data.get("url"):  # Looks like a result dictionary
+                            all_results.append(data)
+                            # Display summary info as it arrives
+                            success_icon = "[green]✔[/]" if data.get(
+                                'success') else "[red]✘[/]"
+                            url = data.get('url', 'N/A')
+                            # Display status code FROM THE RESULT DATA if available
+                            result_status = data.get('status_code', 'N/A')
+                            console.print(
+                                f"  {success_icon} Received: [link={url}]{url}[/link] (Status: {result_status})")
+                            if not data.get('success') and data.get('error_message'):
+                                console.print(
+                                    f"    [red]Error: {data['error_message']}[/]")
+                        else:
+                            console.print(
+                                f"  [yellow]Stream meta-data:[/yellow] {data}")
+                    except json.JSONDecodeError:
+                        console.print(
+                            f"  [red]Stream decode error for line:[/red] {line}")
+            if not completed:
+                console.print(
+                    "[bold yellow]Warning: Stream ended without 'completed' marker.[/]")
+
+    except httpx.HTTPStatusError as e:
+        # Use the captured initial status code if available, otherwise from the exception
+        status = initial_status_code if initial_status_code is not None else e.response.status_code
+        console.print(f"[bold red]HTTP Error (Initial Request):[/]")
+        console.print(f"Status: {status}")
+        try:
+            console.print(Panel(Syntax(json.dumps(
+                e.response.json(), indent=2), "json", theme="default"), title="Error Response"))
+        except json.JSONDecodeError:
+            console.print(f"Response Body: {e.response.text}")
+    except httpx.RequestError as e:
+        console.print(f"[bold red]Request Error: {e}[/]")
+    except Exception as e:
+        console.print(f"[bold red]Unexpected Error during streaming: {e}[/]")
+        # Print stack trace for unexpected errors
+        console.print_exception(show_locals=False)
+
+    # Call print_result_summary with the *collected* results AFTER the stream is done
+    print_result_summary(all_results, title=f"{title} Collected Results")
+
+
+def load_proxies_from_env() -> List[Dict]:
+    """
+    Load proxies from the PROXIES environment variable.
+    Expected format: IP:PORT:USER:PASS,IP:PORT,IP2:PORT2:USER2:PASS2,...
+    Returns a list of dictionaries suitable for the 'params' of ProxyConfig.
+    """
+    proxies_params_list = []
+    proxies_str = os.getenv("PROXIES", "")
+    if not proxies_str:
+        # console.print("[yellow]PROXIES environment variable not set or empty.[/]")
+        return proxies_params_list  # Return empty list if not set
+
+    try:
+        proxy_entries = proxies_str.split(",")
+        for entry in proxy_entries:
+            entry = entry.strip()
+            if not entry:
+                continue
+
+            parts = entry.split(":")
+            proxy_dict = {}
+
+            if len(parts) == 4:  # Format: IP:PORT:USER:PASS
+                ip, port, username, password = parts
+                proxy_dict = {
+                    "server": f"http://{ip}:{port}",  # Assuming http protocol
+                    "username": username,
+                    "password": password,
+                    # "ip": ip # 'ip' is not a standard ProxyConfig param, 'server' contains it
+                }
+            elif len(parts) == 2:  # Format: IP:PORT
+                ip, port = parts
+                proxy_dict = {
+                    "server": f"http://{ip}:{port}",
+                    # "ip": ip
+                }
+            else:
+                console.print(
+                    f"[yellow]Skipping invalid proxy string format:[/yellow] {entry}")
+                continue
+
+            proxies_params_list.append(proxy_dict)
+
+    except Exception as e:
+        console.print(
+            f"[red]Error loading proxies from environment:[/red] {e}")
+
+    if proxies_params_list:
+        console.print(
+            f"[cyan]Loaded {len(proxies_params_list)} proxies from environment.[/]")
+    # else:
+    #     console.print("[yellow]No valid proxies loaded from environment.[/]")
+
+    return proxies_params_list
+
+
+# --- Demo Functions ---
+
+# 1. Basic Crawling
+async def demo_basic_single_url(client: httpx.AsyncClient):
+    payload = {
+        "urls": [SIMPLE_URL],
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "cache_mode": "BYPASS"
+            }
+        }
+    }
+    result = await make_request(client, "/crawl", payload, "Demo 1a: Basic Single URL Crawl")
+    return result
+
+
+async def demo_basic_multi_url(client: httpx.AsyncClient):
+    payload = {
+        "urls": [SIMPLE_URL, LINKS_URL],
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {"type": "CrawlerRunConfig", "params": {"cache_mode": "BYPASS"}}
+    }
+    result = await make_request(client, "/crawl", payload, "Demo 1b: Basic Multi URL Crawl")
+    return result
+
+
+async def demo_streaming_multi_url(client: httpx.AsyncClient):
+    payload = {
+        # "urls": [SIMPLE_URL, LINKS_URL, FORMS_URL, SIMPLE_URL, LINKS_URL, FORMS_URL], # Add another URL
+        "urls": [
+            "https://example.com/page1",
+            "https://example.com/page2",
+            "https://example.com/page3",
+            "https://example.com/page4",
+            "https://example.com/page5"
+        ],  # Add another URL
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "stream": True,
+            }
+        }
+    }
+    result = await stream_request(client, "/crawl/stream", payload, "Demo 1c: Streaming Multi URL Crawl")
+    return result
+
+# 2. Markdown Generation & Content Filtering
+
+
+async def demo_markdown_default(client: httpx.AsyncClient):
+    payload = {
+        "urls": [SIMPLE_URL],
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "markdown_generator": {
+                    "type": "DefaultMarkdownGenerator",
+                    "params": {
+                        "content_source": "fit_html",
+                        "options": {
+                            "type": "dict",
+                            "value": {
+                                "ignore_links": True
+                            }
+                        }
+                    }
+                }  # Explicitly default
+            }
+        }
+    }
+    result = await make_request(client, "/crawl", payload, "Demo 2a: Default Markdown Generation")
+    return result
+
+
+async def demo_markdown_pruning(client: httpx.AsyncClient):
+    payload = {
+        "urls": [PYTHON_URL],  # Use a more complex page
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "cache_mode": "BYPASS",
+                "markdown_generator": {
+                    "type": "DefaultMarkdownGenerator",
+                    "params": {
+                        "content_filter": {
+                            "type": "PruningContentFilter",
+                            "params": {
+                                "threshold": 0.6,
+                                "threshold_type": "relative"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    result = await make_request(client, "/crawl", payload, "Demo 2b: Markdown with Pruning Filter")
+    return result
+
+
+async def demo_markdown_bm25(client: httpx.AsyncClient):
+    payload = {
+        "urls": [PYTHON_URL],
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "cache_mode": "BYPASS",
+                "markdown_generator": {
+                    "type": "DefaultMarkdownGenerator",
+                    "params": {
+                        "content_filter": {
+                            "type": "BM25ContentFilter",
+                            "params": {
+                                "user_query": "Python documentation language reference"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    result = await make_request(client, "/crawl", payload, "Demo 2c: Markdown with BM25 Filter")
+    return result
+
+# 3. Specific Parameters
+# Corrected Demo Function: demo_param_css_selector
+
+
+async def demo_param_css_selector(client: httpx.AsyncClient):
+    css_selector = ".main-content"  # Using the suggested correct selector
+    payload = {
+        "urls": [PYTHON_URL],
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "css_selector": css_selector  # Target specific div
+                # No extraction strategy is needed to demo this parameter's effect on input HTML
+            }
+        }
+    }
+    results = await make_request(client, "/crawl", payload, f"Demo 3a: Using css_selector ('{css_selector}')")
+
+    if results:
+        result = results[0]
+        if result['success'] and result.get('html'):
+            # Check if the returned HTML is likely constrained
+            # A simple check: does it contain expected content from within the selector,
+            # and does it LACK content known to be outside (like footer links)?
+            html_content = result['html']
+            # Text likely within .main-content somewhere
+            content_present = 'Python Software Foundation' in html_content
+            # Text likely in the footer, outside .main-content
+            footer_absent = 'Legal Statements' not in html_content
+
+            console.print(
+                f"  Content Check: Text inside '{css_selector}' likely present? {'[green]Yes[/]' if content_present else '[red]No[/]'}")
+            console.print(
+                f"  Content Check: Text outside '{css_selector}' (footer) likely absent? {'[green]Yes[/]' if footer_absent else '[red]No[/]'}")
+
+            if not content_present or not footer_absent:
+                console.print(
+                    f"  [yellow]Note:[/yellow] HTML filtering might not be precise or page structure changed. Result HTML length: {len(html_content)}")
+            else:
+                console.print(
+                    f"  [green]Verified:[/green] Returned HTML appears limited by css_selector. Result HTML length: {len(html_content)}")
+
+        elif result['success']:
+            console.print(
+                "[yellow]HTML content was empty in the successful result.[/]")
+        # Error message is handled by print_result_summary called by make_request
+
+
+async def demo_param_js_execution(client: httpx.AsyncClient):
+    payload = {
+        "urls": ["https://example.com"],  # Use a page with a form
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "cache_mode": "BYPASS",
+                # Simple JS to fill and maybe click (won't submit without more complex setup)
+                "js_code": """
+                    (() => {
+                        document.querySelector('h1').innerText = 'Crawl4AI Demo';
+                        return { filled_name: document.querySelector('h1').innerText };
+                    })();
+                """,
+                "delay_before_return_html": 0.5  # Give JS time to potentially run
+            }
+        }
+    }
+    results = await make_request(client, "/crawl", payload, "Demo 3b: Using js_code Parameter")
+    if results and results[0].get("js_execution_result"):
+        console.print("[cyan]JS Execution Result:[/]",
+                      results[0]["js_execution_result"])
+    elif results:
+        console.print("[yellow]JS Execution Result not found in response.[/]")
+
+
+async def demo_param_screenshot(client: httpx.AsyncClient):
+    payload = {
+        "urls": [SIMPLE_URL],
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {"cache_mode": "BYPASS", "screenshot": True}
+        }
+    }
+    results = await make_request(client, "/crawl", payload, "Demo 3c: Taking a Screenshot")
+    if results and results[0].get("screenshot"):
+        console.print(
+            f"[cyan]Screenshot data received (length):[/] {len(results[0]['screenshot'])}")
+    elif results:
+        console.print("[yellow]Screenshot data not found in response.[/]")
+
+
+async def demo_param_ssl_fetch(client: httpx.AsyncClient):
+    payload = {
+        "urls": [PYTHON_URL],  # Needs HTTPS
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {"cache_mode": "BYPASS", "fetch_ssl_certificate": True}
+        }
+    }
+    results = await make_request(client, "/crawl", payload, "Demo 3d: Fetching SSL Certificate")
+    if results and results[0].get("ssl_certificate"):
+        console.print("[cyan]SSL Certificate Info:[/]")
+        console.print(results[0]["ssl_certificate"])
+    elif results:
+        console.print("[yellow]SSL Certificate data not found in response.[/]")
+
+
+async def demo_param_proxy(client: httpx.AsyncClient):
+    proxy_params_list = load_proxies_from_env()  # Get the list of parameter dicts
+    if not proxy_params_list:
+        console.rule(
+            "[bold yellow]Demo 3e: Using Proxies (SKIPPED)[/]", style="yellow")
+        console.print("Set the PROXIES environment variable to run this demo.")
+        console.print("Format: IP:PORT:USR:PWD,IP:PORT,...")
+        return
+
+    payload = {
+        "urls": ["https://httpbin.org/ip"],  # URL that shows originating IP
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "cache_mode": "BYPASS",
+                "proxy_rotation_strategy": {
+                    "type": "RoundRobinProxyStrategy",
+                    "params": {
+                        "proxies": [
+                            # [
+                            # {
+                            # "type": "ProxyConfig",
+                            # "params": {
+                            # server:"...",
+                            # "username": "...",
+                            # "password": "..."
+                            # }
+                            # },
+                            # ...
+                            # ]
+
+                            # Filter out the 'ip' key when sending to server, as it's not part of ProxyConfig
+                            {"type": "ProxyConfig", "params": {
+                                k: v for k, v in p.items() if k != 'ip'}}
+                            for p in proxy_params_list
+                        ]
+                    }
+                }
+            }
+        }
+    }
+    results = await make_request(client, "/crawl", payload, "Demo 3e: Using Proxies")
+
+    # --- Verification Logic ---
+    if results and results[0].get("success"):
+        result = results[0]
+        try:
+            # httpbin.org/ip returns JSON within the HTML body's 
 tag
+            html_content = result.get('html', '')
+            # Basic extraction - find JSON within 
 tags or just the JSON itself
+            json_str = None
+            if ' 500 else md
+        console.print(Panel(snippet, title="Markdown snippet",
+                      border_style="cyan", expand=False))
+    except Exception as e:
+        console.print(f"[bold red]Error hitting /md:[/] {e}")
+
+# 8. LLM QA helper endpoint
+
+
+async def demo_llm_endpoint(client: httpx.AsyncClient):
+    """
+    Quick QA round-trip with /llm.
+    Asks a trivial question against SIMPLE_URL just to show wiring.
+    """
+    page_url = SIMPLE_URL
+    question = "What is the title of this page?"
+
+    console.rule("[bold magenta]Demo 7b: /llm Endpoint[/]", style="magenta")
+    enc = urllib.parse.quote_plus(page_url, safe="")
+    console.print(f"GET /llm/{enc}?q={question}")
+
+    try:
+        t0 = time.time()
+        resp = await client.get(f"/llm/{enc}", params={"q": question})
+        dt = time.time() - t0
+        console.print(
+            f"Response Status: [bold {'green' if resp.is_success else 'red'}]{resp.status_code}[/] (took {dt:.2f}s)")
+        resp.raise_for_status()
+        answer = resp.json().get("answer", "")
+        console.print(Panel(answer or "No answer returned",
+                      title="LLM answer", border_style="magenta", expand=False))
+    except Exception as e:
+        console.print(f"[bold red]Error hitting /llm:[/] {e}")
+
+
+# 9. /config/dump helpers --------------------------------------------------
+
+async def demo_config_dump_valid(client: httpx.AsyncClient):
+    """
+    Send a single top-level CrawlerRunConfig(...) expression and show the dump.
+    """
+    code_snippet = "CrawlerRunConfig(cache_mode='BYPASS', screenshot=True)"
+    payload = {"code": code_snippet}
+
+    console.rule("[bold blue]Demo 8a: /config/dump (valid)[/]", style="blue")
+    print_payload(payload)
+
+    try:
+        t0 = time.time()
+        resp = await client.post("/config/dump", json=payload)
+        dt = time.time() - t0
+        console.print(
+            f"Response Status: [bold {'green' if resp.is_success else 'red'}]{resp.status_code}[/] (took {dt:.2f}s)")
+        resp.raise_for_status()
+        dump_json = resp.json()
+        console.print(Panel(Syntax(json.dumps(dump_json, indent=2),
+                      "json", theme="monokai"), title="Dump()", border_style="cyan"))
+    except Exception as e:
+        console.print(f"[bold red]Error in valid /config/dump call:[/] {e}")
+
+
+async def demo_config_dump_invalid(client: httpx.AsyncClient):
+    """
+    Purposely break the rule (nested call) to show the 400 parse error.
+    """
+    bad_code = textwrap.dedent("""
+        BrowserConfig(headless=True); CrawlerRunConfig()
+    """).strip()
+    payload = {"code": bad_code}
+
+    console.rule(
+        "[bold magenta]Demo 8b: /config/dump (invalid)[/]", style="magenta")
+    print_payload(payload)
+
+    try:
+        resp = await client.post("/config/dump", json=payload)
+        console.print(
+            f"Response Status: [bold {'green' if resp.is_success else 'red'}]{resp.status_code}[/]")
+        resp.raise_for_status()   # should throw -> except
+    except httpx.HTTPStatusError as e:
+        console.print("[cyan]Expected parse/validation failure captured:[/]")
+        try:
+            console.print(Panel(Syntax(json.dumps(
+                e.response.json(), indent=2), "json", theme="fruity"), title="Error payload"))
+        except Exception:
+            console.print(e.response.text)
+    except Exception as e:
+        console.print(
+            f"[bold red]Unexpected error during invalid test:[/] {e}")
+
+
+# --- Update Main Runner to include new demo ---
+async def main_demo():
+    async with httpx.AsyncClient(base_url=BASE_URL, timeout=300.0) as client:
+        if not await check_server_health(client):
+            return
+
+        # --- Run Demos ---
+        # await demo_basic_single_url(client)
+        # await demo_basic_multi_url(client)
+        # await demo_streaming_multi_url(client)
+
+        # await demo_markdown_default(client)
+        # await demo_markdown_pruning(client)
+        # await demo_markdown_bm25(client)
+
+        # await demo_param_css_selector(client)
+        # await demo_param_js_execution(client)
+        # await demo_param_screenshot(client)
+        # await demo_param_ssl_fetch(client)
+        # await demo_param_proxy(client) # Skips if no PROXIES env var
+
+        # await demo_extract_css(client)
+        # await demo_extract_llm(client)  # Skips if no common LLM key env var
+
+        # await demo_deep_basic(client)
+        # await demo_deep_streaming(client)  # This need extra work
+
+        # await demo_deep_with_css_extraction(client)
+        # # Skips if no common LLM key env var
+        # await demo_deep_with_llm_extraction(client)
+        # await demo_deep_with_proxy(client)  # Skips if no PROXIES env var
+        # await demo_deep_with_ssl(client)   # Added the new demo
+
+        # --- Helper endpoints ---
+        await demo_markdown_endpoint(client)
+        await demo_llm_endpoint(client)
+
+        # --- /config/dump sanity checks ---
+        await demo_config_dump_valid(client)
+        await demo_config_dump_invalid(client)
+
+        console.rule("[bold green]Demo Complete[/]", style="green")
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main_demo())
+    except KeyboardInterrupt:
+        console.print("\n[yellow]Demo interrupted by user.[/]")
+    except Exception as e:
+        console.print(
+            f"\n[bold red]An error occurred during demo execution:[/]")
+        console.print_exception(show_locals=False)
diff --git a/docs/examples/docker/demo_docker_polling.py b/docs/examples/docker/demo_docker_polling.py
new file mode 100644
index 0000000..ee89572
--- /dev/null
+++ b/docs/examples/docker/demo_docker_polling.py
@@ -0,0 +1,149 @@
+
+#!/usr/bin/env python3
+"""
+demo_docker_polling.py
+Quick sanity-check for the asynchronous crawl job endpoints:
+
+  • POST  /crawl/job          – enqueue work, get task_id
+  • GET   /crawl/job/{id}     – poll status / fetch result
+
+The style matches demo_docker_api.py (console.rule banners, helper
+functions, coloured status lines).  Adjust BASE_URL as needed.
+
+Run:  python demo_docker_polling.py
+"""
+
+import asyncio, json, os, time, urllib.parse
+from typing import Dict, List
+
+import httpx
+from rich.console import Console
+from rich.panel   import Panel
+from rich.syntax  import Syntax
+
+console   = Console()
+BASE_URL  = os.getenv("BASE_URL", "http://localhost:11234")
+SIMPLE_URL = "https://example.org"
+LINKS_URL  = "https://httpbin.org/links/10/1"
+
+# --- helpers --------------------------------------------------------------
+
+
+def print_payload(payload: Dict):
+    console.print(Panel(Syntax(json.dumps(payload, indent=2),
+                               "json", theme="monokai", line_numbers=False),
+                        title="Payload", border_style="cyan", expand=False))
+
+
+async def check_server_health(client: httpx.AsyncClient) -> bool:
+    try:
+        resp = await client.get("/health")
+        if resp.is_success:
+            console.print("[green]Server healthy[/]")
+            return True
+    except Exception:
+        pass
+    console.print("[bold red]Server is not responding on /health[/]")
+    return False
+
+
+async def poll_for_result(client: httpx.AsyncClient, task_id: str,
+                          poll_interval: float = 1.5, timeout: float = 90.0):
+    """Hit /crawl/job/{id} until COMPLETED/FAILED or timeout."""
+    start = time.time()
+    while True:
+        resp = await client.get(f"/crawl/job/{task_id}")
+        resp.raise_for_status()
+        data = resp.json()
+        status = data.get("status")
+        if status.upper() in ("COMPLETED", "FAILED"):
+            return data
+        if time.time() - start > timeout:
+            raise TimeoutError(f"Task {task_id} did not finish in {timeout}s")
+        await asyncio.sleep(poll_interval)
+
+
+# --- demo functions -------------------------------------------------------
+
+
+async def demo_poll_single_url(client: httpx.AsyncClient):
+    payload = {
+        "urls": [SIMPLE_URL],
+        "browser_config": {"type": "BrowserConfig",
+                           "params": {"headless": True}},
+        "crawler_config": {"type": "CrawlerRunConfig",
+                           "params": {"cache_mode": "BYPASS"}}
+    }
+
+    console.rule("[bold blue]Demo A: /crawl/job Single URL[/]", style="blue")
+    print_payload(payload)
+
+    # enqueue
+    resp = await client.post("/crawl/job", json=payload)
+    console.print(f"Enqueue status: [bold]{resp.status_code}[/]")
+    resp.raise_for_status()
+    task_id = resp.json()["task_id"]
+    console.print(f"Task ID: [yellow]{task_id}[/]")
+
+    # poll
+    console.print("Polling…")
+    result = await poll_for_result(client, task_id)
+    console.print(Panel(Syntax(json.dumps(result, indent=2),
+                               "json", theme="fruity"),
+                        title="Final result", border_style="green"))
+    if result["status"] == "COMPLETED":
+        console.print("[green]✅ Crawl succeeded[/]")
+    else:
+        console.print("[red]❌ Crawl failed[/]")
+
+
+async def demo_poll_multi_url(client: httpx.AsyncClient):
+    payload = {
+        "urls": [SIMPLE_URL, LINKS_URL],
+        "browser_config": {"type": "BrowserConfig",
+                           "params": {"headless": True}},
+        "crawler_config": {"type": "CrawlerRunConfig",
+                           "params": {"cache_mode": "BYPASS"}}
+    }
+
+    console.rule("[bold magenta]Demo B: /crawl/job Multi-URL[/]",
+                 style="magenta")
+    print_payload(payload)
+
+    resp = await client.post("/crawl/job", json=payload)
+    console.print(f"Enqueue status: [bold]{resp.status_code}[/]")
+    resp.raise_for_status()
+    task_id = resp.json()["task_id"]
+    console.print(f"Task ID: [yellow]{task_id}[/]")
+
+    console.print("Polling…")
+    result = await poll_for_result(client, task_id)
+    console.print(Panel(Syntax(json.dumps(result, indent=2),
+                               "json", theme="fruity"),
+                        title="Final result", border_style="green"))
+    if result["status"] == "COMPLETED":
+        console.print(
+            f"[green]✅ {len(json.loads(result['result'])['results'])} URLs crawled[/]")
+    else:
+        console.print("[red]❌ Crawl failed[/]")
+
+
+# --- main runner ----------------------------------------------------------
+
+
+async def main_demo():
+    async with httpx.AsyncClient(base_url=BASE_URL, timeout=300.0) as client:
+        if not await check_server_health(client):
+            return
+        await demo_poll_single_url(client)
+        await demo_poll_multi_url(client)
+        console.rule("[bold green]Polling demos complete[/]", style="green")
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main_demo())
+    except KeyboardInterrupt:
+        console.print("\n[yellow]Interrupted by user[/]")
+    except Exception:
+        console.print_exception(show_locals=False)
diff --git a/docs/examples/docker_client_hooks_example.py b/docs/examples/docker_client_hooks_example.py
new file mode 100644
index 0000000..1aa27fd
--- /dev/null
+++ b/docs/examples/docker_client_hooks_example.py
@@ -0,0 +1,522 @@
+#!/usr/bin/env python3
+"""
+Comprehensive hooks examples using Docker Client with function objects.
+
+This approach is recommended because:
+- Write hooks as regular Python functions
+- Full IDE support (autocomplete, type checking)
+- Automatic conversion to API format
+- Reusable and testable code
+- Clean, readable syntax
+"""
+
+import asyncio
+from crawl4ai import Crawl4aiDockerClient
+
+# API_BASE_URL = "http://localhost:11235"
+API_BASE_URL = "http://localhost:11234"
+
+
+# ============================================================================
+# Hook Function Definitions
+# ============================================================================
+
+# --- All Hooks Demo ---
+async def browser_created_hook(browser, **kwargs):
+    """Called after browser is created"""
+    print("[HOOK] Browser created and ready")
+    return browser
+
+
+async def page_context_hook(page, context, **kwargs):
+    """Setup page environment"""
+    print("[HOOK] Setting up page environment")
+
+    # Set viewport
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+
+    # Add cookies
+    await context.add_cookies([{
+        "name": "test_session",
+        "value": "abc123xyz",
+        "domain": ".httpbin.org",
+        "path": "/"
+    }])
+
+    # Block resources
+    await context.route("**/*.{png,jpg,jpeg,gif}", lambda route: route.abort())
+    await context.route("**/analytics/*", lambda route: route.abort())
+
+    print("[HOOK] Environment configured")
+    return page
+
+
+async def user_agent_hook(page, context, user_agent, **kwargs):
+    """Called when user agent is updated"""
+    print(f"[HOOK] User agent: {user_agent[:50]}...")
+    return page
+
+
+async def before_goto_hook(page, context, url, **kwargs):
+    """Called before navigating to URL"""
+    print(f"[HOOK] Navigating to: {url}")
+
+    await page.set_extra_http_headers({
+        "X-Custom-Header": "crawl4ai-test",
+        "Accept-Language": "en-US"
+    })
+
+    return page
+
+
+async def after_goto_hook(page, context, url, response, **kwargs):
+    """Called after page loads"""
+    print(f"[HOOK] Page loaded: {url}")
+
+    await page.wait_for_timeout(1000)
+
+    try:
+        await page.wait_for_selector("body", timeout=2000)
+        print("[HOOK] Body element ready")
+    except:
+        print("[HOOK] Timeout, continuing")
+
+    return page
+
+
+async def execution_started_hook(page, context, **kwargs):
+    """Called when custom JS execution starts"""
+    print("[HOOK] JS execution started")
+    await page.evaluate("console.log('[HOOK] Custom JS');")
+    return page
+
+
+async def before_retrieve_hook(page, context, **kwargs):
+    """Called before retrieving HTML"""
+    print("[HOOK] Preparing HTML retrieval")
+
+    # Scroll for lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+    await page.wait_for_timeout(500)
+    await page.evaluate("window.scrollTo(0, 0);")
+
+    print("[HOOK] Scrolling complete")
+    return page
+
+
+async def before_return_hook(page, context, html, **kwargs):
+    """Called before returning HTML"""
+    print(f"[HOOK] HTML ready: {len(html)} chars")
+
+    metrics = await page.evaluate('''() => ({
+        images: document.images.length,
+        links: document.links.length,
+        scripts: document.scripts.length
+    })''')
+
+    print(f"[HOOK] Metrics - Images: {metrics['images']}, Links: {metrics['links']}")
+    return page
+
+
+# --- Authentication Hooks ---
+async def auth_context_hook(page, context, **kwargs):
+    """Setup authentication context"""
+    print("[HOOK] Setting up authentication")
+
+    # Add auth cookies
+    await context.add_cookies([{
+        "name": "auth_token",
+        "value": "fake_jwt_token",
+        "domain": ".httpbin.org",
+        "path": "/",
+        "httpOnly": True
+    }])
+
+    # Set localStorage
+    await page.evaluate('''
+        localStorage.setItem('user_id', '12345');
+        localStorage.setItem('auth_time', new Date().toISOString());
+    ''')
+
+    print("[HOOK] Auth context ready")
+    return page
+
+
+async def auth_headers_hook(page, context, url, **kwargs):
+    """Add authentication headers"""
+    print(f"[HOOK] Adding auth headers for {url}")
+
+    import base64
+    credentials = base64.b64encode(b"user:passwd").decode('ascii')
+
+    await page.set_extra_http_headers({
+        'Authorization': f'Basic {credentials}',
+        'X-API-Key': 'test-key-123'
+    })
+
+    return page
+
+
+# --- Performance Optimization Hooks ---
+async def performance_hook(page, context, **kwargs):
+    """Optimize page for performance"""
+    print("[HOOK] Optimizing for performance")
+
+    # Block resource-heavy content
+    await context.route("**/*.{png,jpg,jpeg,gif,webp,svg}", lambda r: r.abort())
+    await context.route("**/*.{woff,woff2,ttf}", lambda r: r.abort())
+    await context.route("**/*.{mp4,webm,ogg}", lambda r: r.abort())
+    await context.route("**/googletagmanager.com/*", lambda r: r.abort())
+    await context.route("**/google-analytics.com/*", lambda r: r.abort())
+    await context.route("**/facebook.com/*", lambda r: r.abort())
+
+    # Disable animations
+    await page.add_style_tag(content='''
+        *, *::before, *::after {
+            animation-duration: 0s !important;
+            transition-duration: 0s !important;
+        }
+    ''')
+
+    print("[HOOK] Optimizations applied")
+    return page
+
+
+async def cleanup_hook(page, context, **kwargs):
+    """Clean page before extraction"""
+    print("[HOOK] Cleaning page")
+
+    await page.evaluate('''() => {
+        const selectors = [
+            '.ad', '.ads', '.advertisement',
+            '.popup', '.modal', '.overlay',
+            '.cookie-banner', '.newsletter'
+        ];
+
+        selectors.forEach(sel => {
+            document.querySelectorAll(sel).forEach(el => el.remove());
+        });
+
+        document.querySelectorAll('script, style').forEach(el => el.remove());
+    }''')
+
+    print("[HOOK] Page cleaned")
+    return page
+
+
+# --- Content Extraction Hooks ---
+async def wait_dynamic_content_hook(page, context, url, response, **kwargs):
+    """Wait for dynamic content to load"""
+    print(f"[HOOK] Waiting for dynamic content on {url}")
+
+    await page.wait_for_timeout(2000)
+
+    # Click "Load More" if exists
+    try:
+        load_more = await page.query_selector('[class*="load-more"], button:has-text("Load More")')
+        if load_more:
+            await load_more.click()
+            await page.wait_for_timeout(1000)
+            print("[HOOK] Clicked 'Load More'")
+    except:
+        pass
+
+    return page
+
+
+async def extract_metadata_hook(page, context, **kwargs):
+    """Extract page metadata"""
+    print("[HOOK] Extracting metadata")
+
+    metadata = await page.evaluate('''() => {
+        const getMeta = (name) => {
+            const el = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`);
+            return el ? el.getAttribute('content') : null;
+        };
+
+        return {
+            title: document.title,
+            description: getMeta('description'),
+            author: getMeta('author'),
+            keywords: getMeta('keywords'),
+        };
+    }''')
+
+    print(f"[HOOK] Metadata: {metadata}")
+
+    # Infinite scroll
+    for i in range(3):
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+        await page.wait_for_timeout(1000)
+        print(f"[HOOK] Scroll {i+1}/3")
+
+    return page
+
+
+# --- Multi-URL Hooks ---
+async def url_specific_hook(page, context, url, **kwargs):
+    """Apply URL-specific logic"""
+    print(f"[HOOK] Processing URL: {url}")
+
+    # URL-specific headers
+    if 'html' in url:
+        await page.set_extra_http_headers({"X-Type": "HTML"})
+    elif 'json' in url:
+        await page.set_extra_http_headers({"X-Type": "JSON"})
+
+    return page
+
+
+async def track_progress_hook(page, context, url, response, **kwargs):
+    """Track crawl progress"""
+    status = response.status if response else 'unknown'
+    print(f"[HOOK] Loaded {url} - Status: {status}")
+    return page
+
+
+# ============================================================================
+# Test Functions
+# ============================================================================
+
+async def test_all_hooks_comprehensive():
+    """Test all 8 hook types"""
+    print("=" * 70)
+    print("Test 1: All Hooks Comprehensive Demo (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nCrawling with all 8 hooks...")
+
+        # Define hooks with function objects
+        hooks = {
+            "on_browser_created": browser_created_hook,
+            "on_page_context_created": page_context_hook,
+            "on_user_agent_updated": user_agent_hook,
+            "before_goto": before_goto_hook,
+            "after_goto": after_goto_hook,
+            "on_execution_started": execution_started_hook,
+            "before_retrieve_html": before_retrieve_hook,
+            "before_return_html": before_return_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks=hooks,
+            hooks_timeout=30
+        )
+
+        print("\n✅ Success!")
+        print(f"   URL: {result.url}")
+        print(f"   Success: {result.success}")
+        print(f"   HTML: {len(result.html)} chars")
+
+
+async def test_authentication_workflow():
+    """Test authentication with hooks"""
+    print("\n" + "=" * 70)
+    print("Test 2: Authentication Workflow (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting authentication...")
+
+        hooks = {
+            "on_page_context_created": auth_context_hook,
+            "before_goto": auth_headers_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/basic-auth/user/passwd"],
+            hooks=hooks,
+            hooks_timeout=15
+        )
+
+        print("\n✅ Authentication completed")
+
+        if result.success:
+            if '"authenticated"' in result.html and 'true' in result.html:
+                print("   ✅ Basic auth successful!")
+            else:
+                print("   ⚠️ Auth status unclear")
+        else:
+            print(f"   ❌ Failed: {result.error_message}")
+
+
+async def test_performance_optimization():
+    """Test performance optimization"""
+    print("\n" + "=" * 70)
+    print("Test 3: Performance Optimization (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting performance hooks...")
+
+        hooks = {
+            "on_page_context_created": performance_hook,
+            "before_retrieve_html": cleanup_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks=hooks,
+            hooks_timeout=10
+        )
+
+        print("\n✅ Optimization completed")
+        print(f"   HTML size: {len(result.html):,} chars")
+        print("   Resources blocked, ads removed")
+
+
+async def test_content_extraction():
+    """Test content extraction"""
+    print("\n" + "=" * 70)
+    print("Test 4: Content Extraction (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting extraction hooks...")
+
+        hooks = {
+            "after_goto": wait_dynamic_content_hook,
+            "before_retrieve_html": extract_metadata_hook
+        }
+
+        result = await client.crawl(
+            ["https://www.kidocode.com/"],
+            hooks=hooks,
+            hooks_timeout=20
+        )
+
+        print("\n✅ Extraction completed")
+        print(f"   URL: {result.url}")
+        print(f"   Success: {result.success}")
+        print(f"   Metadata: {result.metadata}")
+
+
+async def test_multi_url_crawl():
+    """Test hooks with multiple URLs"""
+    print("\n" + "=" * 70)
+    print("Test 5: Multi-URL Crawl (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nCrawling multiple URLs...")
+
+        hooks = {
+            "before_goto": url_specific_hook,
+            "after_goto": track_progress_hook
+        }
+
+        results = await client.crawl(
+            [
+                "https://httpbin.org/html",
+                "https://httpbin.org/json",
+                "https://httpbin.org/xml"
+            ],
+            hooks=hooks,
+            hooks_timeout=15
+        )
+
+        print("\n✅ Multi-URL crawl completed")
+        print(f"\n   Crawled {len(results)} URLs:")
+        for i, result in enumerate(results, 1):
+            status = "✅" if result.success else "❌"
+            print(f"   {status} {i}. {result.url}")
+
+
+async def test_reusable_hook_library():
+    """Test using reusable hook library"""
+    print("\n" + "=" * 70)
+    print("Test 6: Reusable Hook Library (Docker Client)")
+    print("=" * 70)
+
+    # Create a library of reusable hooks
+    class HookLibrary:
+        @staticmethod
+        async def block_images(page, context, **kwargs):
+            """Block all images"""
+            await context.route("**/*.{png,jpg,jpeg,gif}", lambda r: r.abort())
+            print("[LIBRARY] Images blocked")
+            return page
+
+        @staticmethod
+        async def block_analytics(page, context, **kwargs):
+            """Block analytics"""
+            await context.route("**/analytics/*", lambda r: r.abort())
+            await context.route("**/google-analytics.com/*", lambda r: r.abort())
+            print("[LIBRARY] Analytics blocked")
+            return page
+
+        @staticmethod
+        async def scroll_infinite(page, context, **kwargs):
+            """Handle infinite scroll"""
+            for i in range(5):
+                prev = await page.evaluate("document.body.scrollHeight")
+                await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+                await page.wait_for_timeout(1000)
+                curr = await page.evaluate("document.body.scrollHeight")
+                if curr == prev:
+                    break
+            print("[LIBRARY] Infinite scroll complete")
+            return page
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nUsing hook library...")
+
+        hooks = {
+            "on_page_context_created": HookLibrary.block_images,
+            "before_retrieve_html": HookLibrary.scroll_infinite
+        }
+
+        result = await client.crawl(
+            ["https://www.kidocode.com/"],
+            hooks=hooks,
+            hooks_timeout=20
+        )
+
+        print("\n✅ Library hooks completed")
+        print(f"   Success: {result.success}")
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+async def main():
+    """Run all Docker client hook examples"""
+    print("🔧 Crawl4AI Docker Client - Hooks Examples (Function-Based)")
+    print("Using Python function objects with automatic conversion")
+    print("=" * 70)
+
+    tests = [
+        ("All Hooks Demo", test_all_hooks_comprehensive),
+        ("Authentication", test_authentication_workflow),
+        ("Performance", test_performance_optimization),
+        ("Extraction", test_content_extraction),
+        ("Multi-URL", test_multi_url_crawl),
+        ("Hook Library", test_reusable_hook_library)
+    ]
+
+    for i, (name, test_func) in enumerate(tests, 1):
+        try:
+            await test_func()
+            print(f"\n✅ Test {i}/{len(tests)}: {name} completed\n")
+        except Exception as e:
+            print(f"\n❌ Test {i}/{len(tests)}: {name} failed: {e}\n")
+            import traceback
+            traceback.print_exc()
+
+    print("=" * 70)
+    print("🎉 All Docker client hook examples completed!")
+    print("\n💡 Key Benefits of Function-Based Hooks:")
+    print("   • Write as regular Python functions")
+    print("   • Full IDE support (autocomplete, types)")
+    print("   • Automatic conversion to API format")
+    print("   • Reusable across projects")
+    print("   • Clean, readable code")
+    print("   • Easy to test and debug")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/docker_config_obj.py b/docs/examples/docker_config_obj.py
new file mode 100644
index 0000000..6ddf157
--- /dev/null
+++ b/docs/examples/docker_config_obj.py
@@ -0,0 +1,249 @@
+from crawl4ai import BrowserConfig, CrawlerRunConfig, PruningContentFilter, DefaultMarkdownGenerator
+from crawl4ai.deep_crawling.filters import ContentTypeFilter, DomainFilter
+from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer, PathDepthScorer
+from crawl4ai.cache_context import CacheMode
+from crawl4ai.deep_crawling.bfs_strategy import BFSDeepCrawlStrategy
+from crawl4ai.deep_crawling.filters import FilterChain
+from crawl4ai.deep_crawling.scorers import CompositeScorer
+from crawl4ai.docker_client import Crawl4aiDockerClient
+import json
+from rich.console import Console
+from rich.syntax import Syntax
+
+console = Console()
+
+def print_json(data: dict, title: str = None):
+    """Helper to print JSON prettily with syntax highlighting"""
+    if title:
+        console.print(f"\n[bold blue]{title}[/bold blue]")
+    json_str = json.dumps(data, indent=2)
+    syntax = Syntax(json_str, "json", theme="monokai", line_numbers=True)
+    console.print(syntax)
+
+async def part1_basic_config():
+    """PART 1: Understanding Basic Configuration Objects
+    
+    Here we create simple configuration objects and examine their structure.
+    This helps understand the basic type-params pattern used throughout the API.
+    """
+    console.print("\n[bold green]Explanation:[/bold green] Configuration objects like BrowserConfig and CrawlerRunConfig are the foundation of Crawl4AI. They define how the crawler behaves—e.g., whether it runs headless or how it processes content. These objects use a 'type-params' pattern: 'type' identifies the object class, and 'params' holds its settings. This structure is key because it’s reusable and can be serialized into JSON for API calls.")
+    
+    # Create a simple browser config
+    browser_config = BrowserConfig(
+        headless=False,
+        viewport_width=500,
+        headers = {"User-Agent": "Mozilla/5.0"}
+    )
+    
+    # Show its structure
+    print_json(browser_config.dump(), "Simple Browser Config Structure")
+    
+    # Create a more complex config with nested objects
+    crawler_config = CrawlerRunConfig(
+        word_count_threshold=200,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(threshold=0.5)
+        )
+    )
+    
+    print_json(crawler_config.dump(), "Complex Config with Nested Objects")
+
+async def part2_manual_json():
+    """PART 2: Building JSON Manually
+    
+    Learn how to construct the JSON structure by hand.
+    This demonstrates deep understanding of the configuration format.
+    """
+    console.print("\n[bold green]Explanation:[/bold green] Manually building JSON configurations mirrors how the API expects data. It’s a hands-on way to learn the exact structure—each object has a 'type' and 'params' section. This is useful when you’re troubleshooting or working without the SDK, as it forces you to understand every detail of the config format.")
+    
+    # Manual browser config
+    manual_browser = {
+        "type": "BrowserConfig",
+        "params": {
+            "headless": True,
+            "viewport": {
+                "type": "dict",
+                "value": {
+                    "width": 1200,
+                    "height": 800
+                }
+            }
+        }
+    }
+    
+    # Validate by loading into BrowserConfig
+    loaded_config = BrowserConfig.load(manual_browser)
+    print_json(loaded_config.dump(), "Manually Created -> Loaded -> Dumped")
+    
+    # Show they're equivalent
+    original = BrowserConfig(headless=True, viewport={"width": 1200, "height": 800})
+    assert loaded_config.dump() == original.dump(), "Configs are equivalent!"
+    
+async def part3_complex_structures():
+    """PART 3: Working with Complex Nested Structures
+    
+    Explore more complex configurations with multiple levels of nesting.
+    This shows how the type-params pattern scales to complex scenarios.
+    """
+    console.print("\n[bold green]Explanation:[/bold green] Real-world crawling often requires detailed settings—like filtering content or customizing output. Here, we nest objects (e.g., a markdown generator with a content filter) using the same 'type-params' pattern. This nesting lets you fine-tune the crawler’s behavior at multiple levels, making it powerful and flexible.")
+    
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter()
+        ),
+        deep_crawl_strategy=BFSDeepCrawlStrategy(
+            max_depth=5,
+            filter_chain=FilterChain(
+                filters=[
+                    ContentTypeFilter(allowed_types=["text/html"]),
+                    DomainFilter(allowed_domains=["example.com"])
+                ]
+            ),
+            url_scorer=CompositeScorer(
+                scorers=[
+                    KeywordRelevanceScorer(keywords=["data", "analysis"]),
+                    PathDepthScorer(optimal_depth=3)
+                ]
+            )
+        )
+    )
+    
+    print_json(config.dump(), "Deep Nested Configuration")
+
+async def part4_client_sdk():
+    """PART 4: Using the Client SDK
+    
+    Demonstrate how the SDK makes working with the API simple by handling
+    all the complex serialization automatically.
+    """
+    console.print("\n[bold green]Explanation:[/bold green] The Crawl4aiDockerClient SDK is a time-saver—it takes your configuration objects and turns them into API-ready JSON automatically. This means less manual work and fewer mistakes. You just define your settings, pass them to the SDK, and it handles the rest, making crawling easier and faster.")
+    
+    async with Crawl4aiDockerClient(base_url="http://localhost:8000") as client:
+        # You would normally authenticate here if JWT is enabled
+        await client.authenticate("user@example.com")
+        
+        # Create configs
+        browser_config = BrowserConfig(headless=True)
+        crawler_config = CrawlerRunConfig(stream=False)
+        
+        # SDK handles all serialization
+        result = await client.crawl(
+            urls=["https://example.com"],
+            browser_config=browser_config,
+            crawler_config=crawler_config
+        )
+        
+        console.print("\n[bold green]🚀 Crawl completed successfully![/bold green]")
+        console.print(f"Markdown length: {len(result.markdown)} characters")
+
+async def part5_direct_api():
+    """PART 5: Using the API Directly
+    
+    Learn how to make direct API calls without the SDK.
+    This demonstrates the raw request structure and gives more control.
+    """
+    console.print("\n[bold green]Explanation:[/bold green] Skipping the SDK means you’re in full control—you build the JSON payload yourself and send it to the API. This is harder but gives you a deeper understanding of how Crawl4AI works under the hood. It’s also useful if you’re integrating with systems that don’t use the SDK.")
+    
+    import aiohttp
+    from datetime import datetime
+
+    # Prepare the request payload
+    payload = {
+        "urls": ["https://example.com"],
+        "browser_config": {
+            "type": "BrowserConfig",
+            "params": {
+                "headless": True,
+                "viewport": {
+                    "type": "dict",
+                    "value": {
+                        "width": 1200,
+                        "height": 800
+                    }
+                }
+            }
+        },
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "cache_mode": "bypass",
+                "markdown_generator": {
+                    "type": "DefaultMarkdownGenerator",
+                    "params": {
+                        "content_filter": {
+                            "type": "PruningContentFilter",
+                            "params": {
+                                "threshold": 0.48,
+                                "threshold_type": "fixed"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    print_json(payload, "Direct API Request Payload")
+
+    async with aiohttp.ClientSession() as session:
+        # If JWT is enabled, get token first
+        token_response = await session.post(
+            "http://localhost:8000/token",
+            json={"email": "user@example.com"}
+        )
+        token = (await token_response.json())["access_token"]
+        headers = {"Authorization": f"Bearer {token}"}
+
+        # Make the crawl request
+        start_time = datetime.now()
+        async with session.post(
+            "http://localhost:8000/crawl",
+            json=payload,
+            headers=headers  # comment if using JWT
+        ) as response:
+            result = await response.json()
+            duration = (datetime.now() - start_time).total_seconds()
+
+        console.print(f"\n[bold green]✅ API call completed in {duration:.2f}s[/bold green]")
+        print_json(result, "API Response")
+
+async def part6_wrap_up():
+    """PART 6: Wrap-Up and Key Takeaways
+    
+    Summarize the key concepts learned in this tutorial.
+    """
+    console.print("\n[bold yellow]🎓 Tutorial Wrap-Up[/bold yellow]")
+    console.print("[italic]Key Takeaways:[/italic]\n")
+    console.print("- **Configurations:** Use the type-params pattern to define settings flexibly.")
+    console.print("- **Manual JSON:** Build configs by hand to master the structure.")
+    console.print("- **Nesting:** Customize deeply with nested objects.")
+    console.print("- **SDK:** Simplify API calls with automatic serialization.")
+    console.print("- **Direct API:** Gain control by crafting raw requests.")
+    console.print("\n[bold green]🚀 You’re ready to crawl with Crawl4AI![/bold green]")
+
+async def main():
+    """Main tutorial runner that executes each part in sequence"""
+    console.print("\n[bold yellow]🎓 Crawl4AI Docker Tutorial[/bold yellow]")
+    console.print("[italic]Learn how to work with configuration objects and the Docker API[/italic]\n")
+    
+    parts = [
+        (part1_basic_config, "Understanding Basic Configurations"),
+        (part2_manual_json, "Manual JSON Construction"),
+        (part3_complex_structures, "Complex Nested Structures"),
+        (part4_client_sdk, "Using the Client SDK"),
+        (part5_direct_api, "Direct API Integration"),
+        (part6_wrap_up, "Wrap-Up and Key Takeaways")
+    ]
+    
+    for func, title in parts:
+        console.print(f"\n[bold cyan]📚 {title}[/bold cyan]")
+        console.print("[dim]" + func.__doc__.strip() + "[/dim]\n")
+        await func()
+        if func != part6_wrap_up:  # No pause after wrap-up
+            input("\nPress Enter to continue...\n")
+
+# Run the tutorial
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/docker_example.py b/docs/examples/docker_example.py
new file mode 100644
index 0000000..5925a8c
--- /dev/null
+++ b/docs/examples/docker_example.py
@@ -0,0 +1,407 @@
+import requests
+import json
+import time
+import sys
+import base64
+import os
+from typing import Dict, Any
+
+
+class Crawl4AiTester:
+    def __init__(self, base_url: str = "http://localhost:11235"):
+        self.base_url = base_url
+
+    def submit_and_wait(
+        self, request_data: Dict[str, Any], timeout: int = 300
+    ) -> Dict[str, Any]:
+        # Submit crawl job using async endpoint
+        response = requests.post(
+            f"{self.base_url}/crawl/job", json=request_data
+        )
+        response.raise_for_status()
+        job_response = response.json()
+        task_id = job_response["task_id"]
+        print(f"Submitted job with task_id: {task_id}")
+
+        # Poll for result
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Task {task_id} did not complete within {timeout} seconds"
+                )
+
+            result = requests.get(
+                f"{self.base_url}/crawl/job/{task_id}"
+            )
+            result.raise_for_status()
+            status = result.json()
+
+            if status["status"] == "failed":
+                print("Task failed:", status.get("error"))
+                raise Exception(f"Task failed: {status.get('error')}")
+
+            if status["status"] == "completed":
+                return status
+
+            time.sleep(2)
+
+    def submit_sync(self, request_data: Dict[str, Any]) -> Dict[str, Any]:
+        # Use synchronous crawl endpoint
+        response = requests.post(
+            f"{self.base_url}/crawl",
+            json=request_data,
+            timeout=60,
+        )
+        if response.status_code == 408:
+            raise TimeoutError("Task did not complete within server timeout")
+        response.raise_for_status()
+        return response.json()
+
+def test_docker_deployment(version="basic"):
+    tester = Crawl4AiTester(
+        base_url="http://localhost:11235",
+    )
+    print(f"Testing Crawl4AI Docker {version} version")
+
+    # Health check with timeout and retry
+    max_retries = 5
+    for i in range(max_retries):
+        try:
+            health = requests.get(f"{tester.base_url}/health", timeout=10)
+            print("Health check:", health.json())
+            break
+        except requests.exceptions.RequestException:
+            if i == max_retries - 1:
+                print(f"Failed to connect after {max_retries} attempts")
+                sys.exit(1)
+            print(f"Waiting for service to start (attempt {i+1}/{max_retries})...")
+            time.sleep(5)
+
+    # Test cases based on version
+    test_basic_crawl(tester)
+    test_basic_crawl_sync(tester)
+    if version in ["full", "transformer"]:
+        test_cosine_extraction(tester)
+
+    test_js_execution(tester)
+    test_css_selector(tester)
+    test_structured_extraction(tester)
+    test_llm_extraction(tester)
+    test_llm_with_ollama(tester)
+    test_screenshot(tester)
+
+
+def test_basic_crawl(tester: Crawl4AiTester):
+    print("\n=== Testing Basic Crawl (Async) ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {},
+        "crawler_config": {}
+    }
+
+    result = tester.submit_and_wait(request)
+    print(f"Basic crawl result count: {len(result['result']['results'])}")
+    assert result["result"]["success"]
+    assert len(result["result"]["results"]) > 0
+    assert len(result["result"]["results"][0]["markdown"]) > 0
+
+
+def test_basic_crawl_sync(tester: Crawl4AiTester):
+    print("\n=== Testing Basic Crawl (Sync) ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {},
+        "crawler_config": {}
+    }
+
+    result = tester.submit_sync(request)
+    print(f"Basic crawl result count: {len(result['results'])}")
+    assert result["success"]
+    assert len(result["results"]) > 0
+    assert len(result["results"][0]["markdown"]) > 0
+
+
+def test_js_execution(tester: Crawl4AiTester):
+    print("\n=== Testing JS Execution ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {"headless": True},
+        "crawler_config": {
+            "js_code": [
+                "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); if(loadMoreButton) loadMoreButton.click();"
+            ],
+            "wait_for": "wide-tease-item__wrapper df flex-column flex-row-m flex-nowrap-m enable-new-sports-feed-mobile-design(10)"
+        }
+    }
+
+    result = tester.submit_and_wait(request)
+    print(f"JS execution result count: {len(result['result']['results'])}")
+    assert result["result"]["success"]
+
+
+def test_css_selector(tester: Crawl4AiTester):
+    print("\n=== Testing CSS Selector ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {"headless": True},
+        "crawler_config": {
+            "css_selector": ".wide-tease-item__description",
+            "word_count_threshold": 10
+        }
+    }
+
+    result = tester.submit_and_wait(request)
+    print(f"CSS selector result count: {len(result['result']['results'])}")
+    assert result["result"]["success"]
+
+
+def test_structured_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing Structured Extraction ===")
+    schema = {
+        "name": "Cryptocurrency Prices",
+        "baseSelector": "table[data-testid=\"prices-table\"] tbody tr",
+        "fields": [
+            {
+                "name": "asset_name",
+                "selector": "td:nth-child(2) p.cds-headline-h4steop",
+                "type": "text"
+            },
+            {
+                "name": "asset_symbol",
+                "selector": "td:nth-child(2) p.cds-label2-l1sm09ec",
+                "type": "text"
+            },
+            {
+                "name": "asset_image_url",
+                "selector": "td:nth-child(2) img[alt=\"Asset Symbol\"]",
+                "type": "attribute",
+                "attribute": "src"
+            },
+            {
+                "name": "asset_url",
+                "selector": "td:nth-child(2) a[aria-label^=\"Asset page for\"]",
+                "type": "attribute",
+                "attribute": "href"
+            },
+            {
+                "name": "price",
+                "selector": "td:nth-child(3) div.cds-typographyResets-t6muwls.cds-body-bwup3gq",
+                "type": "text"
+            },
+            {
+                "name": "change",
+                "selector": "td:nth-child(7) p.cds-body-bwup3gq",
+                "type": "text"
+            }
+        ]
+    }
+
+    request = {
+        "urls": ["https://www.coinbase.com/explore"],
+        "browser_config": {},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "JsonCssExtractionStrategy",
+                    "params": {"schema": schema}
+                }
+            }
+        }
+    }
+
+    result = tester.submit_and_wait(request)
+    extracted = json.loads(result["result"]["results"][0]["extracted_content"])
+    print(f"Extracted {len(extracted)} items")
+    if extracted:
+        print("Sample item:", json.dumps(extracted[0], indent=2))
+    assert result["result"]["success"]
+    assert len(extracted) > 0
+
+
+def test_llm_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing LLM Extraction ===")
+    schema = {
+        "type": "object",
+        "properties": {
+            "asset_name": {
+                "type": "string",
+                "description": "Name of the asset.",
+            },
+            "price": {
+                "type": "string",
+                "description": "Price of the asset.",
+            },
+            "change": {
+                "type": "string",
+                "description": "Change in price of the asset.",
+            },
+        },
+        "required": ["asset_name", "price", "change"],
+    }
+
+    request = {
+        "urls": ["https://www.coinbase.com/en-in/explore"],
+        "browser_config": {},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "LLMExtractionStrategy",
+                    "params": {
+                        "llm_config": {
+                            "type": "LLMConfig",
+                            "params": {
+                                "provider": "gemini/gemini-2.0-flash-exp",
+                                "api_token": os.getenv("GEMINI_API_KEY")
+                            }
+                        },
+                        "schema": schema,
+                        "extraction_type": "schema",
+                        "instruction": "From the crawled content, extract asset names along with their prices and change in price.",
+                    }
+                },
+                "word_count_threshold": 1
+            }
+        }
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["results"][0]["extracted_content"])
+        print(f"Extracted {len(extracted)} asset pricing entries")
+        if extracted:
+            print("Sample entry:", json.dumps(extracted[0], indent=2))
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"LLM extraction test failed (might be due to missing API key): {str(e)}")
+
+
+def test_llm_with_ollama(tester: Crawl4AiTester):
+    print("\n=== Testing LLM with Ollama ===")
+    
+    # Check if Ollama is accessible first
+    try:
+        ollama_response = requests.get("http://localhost:11434/api/tags", timeout=5)
+        ollama_response.raise_for_status()
+        print("Ollama is accessible")
+    except:
+        print("Ollama is not accessible, skipping test")
+        return
+    
+    schema = {
+        "type": "object",
+        "properties": {
+            "article_title": {
+                "type": "string",
+                "description": "The main title of the news article",
+            },
+            "summary": {
+                "type": "string",
+                "description": "A brief summary of the article content",
+            },
+            "main_topics": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Main topics or themes discussed in the article",
+            },
+        },
+    }
+
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {"verbose": True},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "LLMExtractionStrategy",
+                    "params": {
+                        "llm_config": {
+                            "type": "LLMConfig",
+                            "params": {
+                                "provider": "ollama/llama3.2:latest",
+                            }
+                        },
+                        "schema": schema,
+                        "extraction_type": "schema",
+                        "instruction": "Extract the main article information including title, summary, and main topics.",
+                    }
+                },
+                "word_count_threshold": 1
+            }
+        }
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["results"][0]["extracted_content"])
+        print("Extracted content:", json.dumps(extracted, indent=2))
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"Ollama extraction test failed: {str(e)}")
+
+
+def test_cosine_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing Cosine Extraction ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "CosineStrategy",
+                    "params": {
+                        "semantic_filter": "business finance economy",
+                        "word_count_threshold": 10,
+                        "max_dist": 0.2,
+                        "top_k": 3,
+                    }
+                }
+            }
+        }
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["results"][0]["extracted_content"])
+        print(f"Extracted {len(extracted)} text clusters")
+        if extracted:
+            print("First cluster tags:", extracted[0]["tags"])
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"Cosine extraction test failed: {str(e)}")
+
+
+def test_screenshot(tester: Crawl4AiTester):
+    print("\n=== Testing Screenshot ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {"headless": True},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "screenshot": True
+            }
+        }
+    }
+
+    result = tester.submit_and_wait(request)
+    screenshot_data = result["result"]["results"][0]["screenshot"]
+    print("Screenshot captured:", bool(screenshot_data))
+
+    if screenshot_data:
+        # Save screenshot
+        screenshot_bytes = base64.b64decode(screenshot_data)
+        with open("test_screenshot.jpg", "wb") as f:
+            f.write(screenshot_bytes)
+        print("Screenshot saved as test_screenshot.jpg")
+
+    assert result["result"]["success"]
+
+
+if __name__ == "__main__":
+    version = sys.argv[1] if len(sys.argv) > 1 else "basic"
+    test_docker_deployment(version)
diff --git a/docs/examples/docker_hooks_examples.py b/docs/examples/docker_hooks_examples.py
new file mode 100644
index 0000000..b64caf0
--- /dev/null
+++ b/docs/examples/docker_hooks_examples.py
@@ -0,0 +1,627 @@
+#!/usr/bin/env python3
+"""
+🚀 Crawl4AI Docker Hooks System - Complete Examples
+====================================================
+
+This file demonstrates the Docker Hooks System with three different approaches:
+
+1. String-based hooks for REST API
+2. hooks_to_string() utility to convert functions
+3. Docker Client with automatic conversion (most convenient)
+
+Requirements:
+- Docker container running: docker run -p 11235:11235 unclecode/crawl4ai:latest
+- crawl4ai installed: pip install crawl4ai
+"""
+
+import asyncio
+import requests
+import json
+import time
+from typing import Dict, Any
+
+# Import Crawl4AI components
+from crawl4ai import hooks_to_string
+from crawl4ai.docker_client import Crawl4aiDockerClient
+
+# Configuration
+DOCKER_URL = "http://localhost:11235"
+TEST_URLS = [
+    "https://www.kidocode.com",
+    "https://quotes.toscrape.com",
+    "https://httpbin.org/html",
+]
+
+
+def print_section(title: str, description: str = ""):
+    """Print a formatted section header"""
+    print("\n" + "=" * 70)
+    print(f"  {title}")
+    if description:
+        print(f"  {description}")
+    print("=" * 70 + "\n")
+
+
+def check_docker_service() -> bool:
+    """Check if Docker service is running"""
+    try:
+        response = requests.get(f"{DOCKER_URL}/health", timeout=3)
+        return response.status_code == 200
+    except:
+        return False
+
+
+# ============================================================================
+# REUSABLE HOOK LIBRARY
+# ============================================================================
+
+async def performance_optimization_hook(page, context, **kwargs):
+    """
+    Performance Hook: Block unnecessary resources to speed up crawling
+    """
+    print("  [Hook] 🚀 Optimizing performance - blocking images and ads...")
+
+    # Block images
+    await context.route(
+        "**/*.{png,jpg,jpeg,gif,webp,svg,ico}",
+        lambda route: route.abort()
+    )
+
+    # Block ads and analytics
+    await context.route("**/analytics/*", lambda route: route.abort())
+    await context.route("**/ads/*", lambda route: route.abort())
+    await context.route("**/google-analytics.com/*", lambda route: route.abort())
+
+    print("  [Hook] ✓ Performance optimization applied")
+    return page
+
+
+async def viewport_setup_hook(page, context, **kwargs):
+    """
+    Viewport Hook: Set consistent viewport size for rendering
+    """
+    print("  [Hook] 🖥️  Setting viewport to 1920x1080...")
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    print("  [Hook] ✓ Viewport configured")
+    return page
+
+
+async def authentication_headers_hook(page, context, url, **kwargs):
+    """
+    Headers Hook: Add custom authentication and tracking headers
+    """
+    print(f"  [Hook] 🔐 Adding custom headers for {url[:50]}...")
+
+    await page.set_extra_http_headers({
+        'X-Crawl4AI': 'docker-hooks',
+        'X-Custom-Hook': 'function-based',
+        'Accept-Language': 'en-US,en;q=0.9',
+    })
+
+    print("  [Hook] ✓ Custom headers added")
+    return page
+
+
+async def lazy_loading_handler_hook(page, context, **kwargs):
+    """
+    Content Hook: Handle lazy-loaded content by scrolling
+    """
+    print("  [Hook] 📜 Scrolling to load lazy content...")
+
+    # Scroll to bottom
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+
+    # Scroll to middle
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2)")
+    await page.wait_for_timeout(500)
+
+    # Scroll back to top
+    await page.evaluate("window.scrollTo(0, 0)")
+    await page.wait_for_timeout(500)
+
+    print("  [Hook] ✓ Lazy content loaded")
+    return page
+
+
+async def page_analytics_hook(page, context, **kwargs):
+    """
+    Analytics Hook: Log page metrics before extraction
+    """
+    print("  [Hook] 📊 Collecting page analytics...")
+
+    metrics = await page.evaluate('''
+        () => ({
+            title: document.title,
+            images: document.images.length,
+            links: document.links.length,
+            scripts: document.scripts.length,
+            headings: document.querySelectorAll('h1, h2, h3').length,
+            paragraphs: document.querySelectorAll('p').length
+        })
+    ''')
+
+    print(f"  [Hook] 📈 Page: {metrics['title'][:50]}...")
+    print(f"         Links: {metrics['links']}, Images: {metrics['images']}, "
+          f"Headings: {metrics['headings']}, Paragraphs: {metrics['paragraphs']}")
+
+    return page
+
+
+# ============================================================================
+# APPROACH 1: String-Based Hooks (REST API)
+# ============================================================================
+
+def example_1_string_based_hooks():
+    """
+    Demonstrate string-based hooks with REST API
+    Use this when working with REST API directly or non-Python clients
+    """
+    print_section(
+        "APPROACH 1: String-Based Hooks (REST API)",
+        "Define hooks as strings for REST API requests"
+    )
+
+    # Define hooks as strings
+    hooks_config = {
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("  [String Hook] Setting up page context...")
+    # Block images for performance
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+""",
+
+        "before_goto": """
+async def hook(page, context, url, **kwargs):
+    print(f"  [String Hook] Navigating to {url[:50]}...")
+    await page.set_extra_http_headers({
+        'X-Crawl4AI': 'string-based-hooks',
+    })
+    return page
+""",
+
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    print("  [String Hook] Scrolling page...")
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    return page
+"""
+    }
+
+    # Prepare request payload
+    payload = {
+        "urls": [TEST_URLS[2]],  # httpbin.org
+        "hooks": {
+            "code": hooks_config,
+            "timeout": 30
+        },
+        "crawler_config": {
+            "cache_mode": "bypass"
+        }
+    }
+
+    print(f"🎯 Target URL: {TEST_URLS[2]}")
+    print(f"🔧 Configured {len(hooks_config)} string-based hooks")
+    print(f"📡 Sending request to Docker API...\n")
+
+    try:
+        start_time = time.time()
+        response = requests.post(f"{DOCKER_URL}/crawl", json=payload, timeout=60)
+        execution_time = time.time() - start_time
+
+        if response.status_code == 200:
+            result = response.json()
+
+            print(f"\n✅ Request successful! (took {execution_time:.2f}s)")
+
+            # Display results
+            if result.get('results') and result['results'][0].get('success'):
+                crawl_result = result['results'][0]
+                html_length = len(crawl_result.get('html', ''))
+                markdown_length = len(crawl_result.get('markdown', ''))
+
+                print(f"\n📊 Results:")
+                print(f"   • HTML length: {html_length:,} characters")
+                print(f"   • Markdown length: {markdown_length:,} characters")
+                print(f"   • URL: {crawl_result.get('url')}")
+
+                # Check hooks execution
+                if 'hooks' in result:
+                    hooks_info = result['hooks']
+                    print(f"\n🎣 Hooks Execution:")
+                    print(f"   • Status: {hooks_info['status']['status']}")
+                    print(f"   • Attached hooks: {len(hooks_info['status']['attached_hooks'])}")
+
+                    if 'summary' in hooks_info:
+                        summary = hooks_info['summary']
+                        print(f"   • Total executions: {summary['total_executions']}")
+                        print(f"   • Successful: {summary['successful']}")
+                        print(f"   • Success rate: {summary['success_rate']:.1f}%")
+            else:
+                print(f"⚠️ Crawl completed but no results")
+
+        else:
+            print(f"❌ Request failed with status {response.status_code}")
+            print(f"   Error: {response.text[:200]}")
+
+    except requests.exceptions.Timeout:
+        print("⏰ Request timed out after 60 seconds")
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+
+    print("\n" + "─" * 70)
+    print("✓ String-based hooks example complete\n")
+
+
+# ============================================================================
+# APPROACH 2: Function-Based Hooks with hooks_to_string() Utility
+# ============================================================================
+
+def example_2_hooks_to_string_utility():
+    """
+    Demonstrate the hooks_to_string() utility for converting functions
+    Use this when you want to write hooks as functions but use REST API
+    """
+    print_section(
+        "APPROACH 2: hooks_to_string() Utility",
+        "Convert Python functions to strings for REST API"
+    )
+
+    print("📦 Creating hook functions...")
+    print("   • performance_optimization_hook")
+    print("   • authentication_headers_hook")
+    print("   • lazy_loading_handler_hook")
+
+    # Convert function objects to strings using the utility
+    print("\n🔄 Converting functions to strings with hooks_to_string()...")
+
+    hooks_dict = {
+        "on_page_context_created": performance_optimization_hook,
+        "before_goto": authentication_headers_hook,
+        "before_retrieve_html": lazy_loading_handler_hook,
+    }
+
+    hooks_as_strings = hooks_to_string(hooks_dict)
+
+    print(f"✅ Successfully converted {len(hooks_as_strings)} functions to strings")
+
+    # Show a preview
+    print("\n📝 Sample converted hook (first 200 characters):")
+    print("─" * 70)
+    sample_hook = list(hooks_as_strings.values())[0]
+    print(sample_hook[:200] + "...")
+    print("─" * 70)
+
+    # Use the converted hooks with REST API
+    print("\n📡 Using converted hooks with REST API...")
+
+    payload = {
+        "urls": [TEST_URLS[2]],
+        "hooks": {
+            "code": hooks_as_strings,
+            "timeout": 30
+        }
+    }
+
+    try:
+        start_time = time.time()
+        response = requests.post(f"{DOCKER_URL}/crawl", json=payload, timeout=60)
+        execution_time = time.time() - start_time
+
+        if response.status_code == 200:
+            result = response.json()
+            print(f"\n✅ Request successful! (took {execution_time:.2f}s)")
+
+            if result.get('results') and result['results'][0].get('success'):
+                crawl_result = result['results'][0]
+                print(f"   • HTML length: {len(crawl_result.get('html', '')):,} characters")
+                print(f"   • Hooks executed successfully!")
+        else:
+            print(f"❌ Request failed: {response.status_code}")
+
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+
+    print("\n💡 Benefits of hooks_to_string():")
+    print("   ✓ Write hooks as regular Python functions")
+    print("   ✓ Full IDE support (autocomplete, syntax highlighting)")
+    print("   ✓ Type checking and linting")
+    print("   ✓ Easy to test and debug")
+    print("   ✓ Reusable across projects")
+    print("   ✓ Works with any REST API client")
+
+    print("\n" + "─" * 70)
+    print("✓ hooks_to_string() utility example complete\n")
+
+
+# ============================================================================
+# APPROACH 3: Docker Client with Automatic Conversion (RECOMMENDED)
+# ============================================================================
+
+async def example_3_docker_client_auto_conversion():
+    """
+    Demonstrate Docker Client with automatic hook conversion (RECOMMENDED)
+    Use this for the best developer experience with Python
+    """
+    print_section(
+        "APPROACH 3: Docker Client with Auto-Conversion (RECOMMENDED)",
+        "Pass function objects directly - conversion happens automatically!"
+    )
+
+    print("🐳 Initializing Crawl4AI Docker Client...")
+    client = Crawl4aiDockerClient(base_url=DOCKER_URL)
+
+    print("✅ Client ready!\n")
+
+    # Use our reusable hook library - just pass the function objects!
+    print("📚 Using reusable hook library:")
+    print("   • performance_optimization_hook")
+    print("   • authentication_headers_hook")
+    print("   • lazy_loading_handler_hook")
+    print("   • page_analytics_hook")
+
+    print("\n🎯 Target URL: " + TEST_URLS[0])
+    print("🚀 Starting crawl with automatic hook conversion...\n")
+
+    try:
+        start_time = time.time()
+
+        # Pass function objects directly - NO manual conversion needed! ✨
+        results = await client.crawl(
+            urls=[TEST_URLS[0]],
+            hooks={
+                "on_page_context_created": performance_optimization_hook,
+                "before_goto": authentication_headers_hook,
+                "before_retrieve_html": lazy_loading_handler_hook,
+                "before_return_html": page_analytics_hook,
+            },
+            hooks_timeout=30
+        )
+
+        execution_time = time.time() - start_time
+
+        print(f"\n✅ Crawl completed! (took {execution_time:.2f}s)\n")
+
+        # Display results
+        if results and results.success:
+            result = results
+            print(f"📊 Results:")
+            print(f"   • URL: {result.url}")
+            print(f"   • Success: {result.success}")
+            print(f"   • HTML length: {len(result.html):,} characters")
+            print(f"   • Markdown length: {len(result.markdown):,} characters")
+
+            # Show metadata
+            if result.metadata:
+                print(f"\n📋 Metadata:")
+                print(f"   • Title: {result.metadata.get('title', 'N/A')[:50]}...")
+
+            # Show links
+            if result.links:
+                internal_count = len(result.links.get('internal', []))
+                external_count = len(result.links.get('external', []))
+                print(f"\n🔗 Links Found:")
+                print(f"   • Internal: {internal_count}")
+                print(f"   • External: {external_count}")
+        else:
+            print(f"⚠️ Crawl completed but no successful results")
+            if results:
+                print(f"   Error: {results.error_message}")
+
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+        import traceback
+        traceback.print_exc()
+
+    print("\n🌟 Why Docker Client is RECOMMENDED:")
+    print("   ✓ Automatic function-to-string conversion")
+    print("   ✓ No manual hooks_to_string() calls needed")
+    print("   ✓ Cleaner, more Pythonic code")
+    print("   ✓ Full type hints and IDE support")
+    print("   ✓ Built-in error handling")
+    print("   ✓ Async/await support")
+
+    print("\n" + "─" * 70)
+    print("✓ Docker Client auto-conversion example complete\n")
+
+
+# ============================================================================
+# APPROACH 4: Authentication Example
+# ============================================================================
+
+def example_4_authentication_flow():
+    """
+    Demonstrate authentication flow with multiple hooks
+    """
+    print_section(
+        "EXAMPLE 4: Authentication Flow",
+        "Using hooks for authentication with cookies and headers"
+    )
+
+    hooks_code = {
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("[HOOK] Setting up authentication context")
+
+    # Add authentication cookies
+    await context.add_cookies([
+        {
+            "name": "auth_token",
+            "value": "fake_jwt_token_here",
+            "domain": ".httpbin.org",
+            "path": "/",
+            "httpOnly": True,
+            "secure": True
+        }
+    ])
+
+    return page
+""",
+
+        "before_goto": """
+async def hook(page, context, url, **kwargs):
+    print(f"[HOOK] Adding auth headers for {url}")
+
+    # Add Authorization header
+    import base64
+    credentials = base64.b64encode(b"user:passwd").decode('ascii')
+
+    await page.set_extra_http_headers({
+        'Authorization': f'Basic {credentials}',
+        'X-API-Key': 'test-api-key-123'
+    })
+
+    return page
+"""
+    }
+
+    payload = {
+        "urls": ["https://httpbin.org/basic-auth/user/passwd"],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 15
+        }
+    }
+
+    print("\nTesting authentication with httpbin endpoints...")
+    response = requests.post(f"{DOCKER_URL}/crawl", json=payload)
+
+    if response.status_code == 200:
+        data = response.json()
+        print("✅ Authentication test completed")
+
+        if 'results' in data:
+            for i, result in enumerate(data['results']):
+                print(f"\n  URL {i+1}: {result['url']}")
+                if result.get('success'):
+                    # Check for authentication success indicators
+                    html_content = result.get('html', '')
+                    if '"authenticated"' in html_content and 'true' in html_content:
+                        print("    ✅ Authentication successful! Basic auth worked.")
+                    else:
+                        print("    ⚠️ Page loaded but auth status unclear")
+                else:
+                    print(f"    ❌ Failed: {result.get('error_message', 'Unknown error')}")
+    else:
+        print(f"❌ Error: {response.status_code}")
+
+    print("\n" + "─" * 70)
+    print("✓ Authentication example complete\n")
+
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+async def main():
+    """
+    Run all example demonstrations
+    """
+    print("\n" + "=" * 70)
+    print("  🚀 Crawl4AI - Docker Hooks System Examples")
+    print("=" * 70)
+
+    # Check Docker service
+    print("\n🔍 Checking Docker service status...")
+    if not check_docker_service():
+        print("❌ Docker service is not running!")
+        print("\n📋 To start the Docker service:")
+        print("   docker run -p 11235:11235 unclecode/crawl4ai:latest")
+        print("\nPlease start the service and run this example again.")
+        return
+
+    print("✅ Docker service is running!\n")
+
+    # Run all examples
+    examples = [
+        ("String-Based Hooks (REST API)", example_1_string_based_hooks, False),
+        ("hooks_to_string() Utility", example_2_hooks_to_string_utility, False),
+        ("Docker Client Auto-Conversion (Recommended)", example_3_docker_client_auto_conversion, True),
+        ("Authentication Flow", example_4_authentication_flow, False),
+    ]
+
+    for i, (name, example_func, is_async) in enumerate(examples, 1):
+        print(f"\n{'🔷' * 35}")
+        print(f"Example {i}/{len(examples)}: {name}")
+        print(f"{'🔷' * 35}\n")
+
+        try:
+            if is_async:
+                await example_func()
+            else:
+                example_func()
+
+            print(f"✅ Example {i} completed successfully!")
+
+            # Pause between examples (except the last one)
+            if i < len(examples):
+                print("\n⏸️  Press Enter to continue to next example...")
+                input()
+
+        except KeyboardInterrupt:
+            print(f"\n⏹️  Examples interrupted by user")
+            break
+        except Exception as e:
+            print(f"\n❌ Example {i} failed: {str(e)}")
+            import traceback
+            traceback.print_exc()
+            print("\nContinuing to next example...\n")
+            continue
+
+    # Final summary
+    print("\n" + "=" * 70)
+    print("  🎉 All Examples Complete!")
+    print("=" * 70)
+
+    print("\n📊 Summary - Three Approaches to Docker Hooks:")
+
+    print("\n✨ 1. String-Based Hooks:")
+    print("   • Write hooks as strings directly in JSON")
+    print("   • Best for: REST API, non-Python clients, simple use cases")
+    print("   • Cons: No IDE support, harder to debug")
+
+    print("\n✨ 2. hooks_to_string() Utility:")
+    print("   • Write hooks as Python functions, convert to strings")
+    print("   • Best for: Python with REST API, reusable hook libraries")
+    print("   • Pros: IDE support, type checking, easy debugging")
+
+    print("\n✨ 3. Docker Client (RECOMMENDED):")
+    print("   • Pass function objects directly, automatic conversion")
+    print("   • Best for: Python applications, best developer experience")
+    print("   • Pros: All benefits of #2 + cleaner code, no manual conversion")
+
+    print("\n💡 Recommendation:")
+    print("   Use Docker Client (#3) for Python applications")
+    print("   Use hooks_to_string() (#2) when you need REST API flexibility")
+    print("   Use string-based (#1) for non-Python clients or simple scripts")
+
+    print("\n🎯 8 Hook Points Available:")
+    print("   • on_browser_created, on_page_context_created")
+    print("   • on_user_agent_updated, before_goto, after_goto")
+    print("   • on_execution_started, before_retrieve_html, before_return_html")
+
+    print("\n📚 Resources:")
+    print("   • Docs: https://docs.crawl4ai.com/core/docker-deployment")
+    print("   • GitHub: https://github.com/unclecode/crawl4ai")
+    print("   • Discord: https://discord.gg/jP8KfhDhyN")
+
+    print("\n" + "=" * 70)
+    print("  Happy Crawling! 🕷️")
+    print("=" * 70 + "\n")
+
+
+if __name__ == "__main__":
+    print("\n🎬 Starting Crawl4AI Docker Hooks Examples...")
+    print("Press Ctrl+C anytime to exit\n")
+
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\n\n👋 Examples stopped by user. Thanks for exploring Crawl4AI!")
+    except Exception as e:
+        print(f"\n\n❌ Error: {str(e)}")
+        import traceback
+        traceback.print_exc()
diff --git a/docs/examples/docker_python_rest_api.py b/docs/examples/docker_python_rest_api.py
new file mode 100644
index 0000000..6650f8d
--- /dev/null
+++ b/docs/examples/docker_python_rest_api.py
@@ -0,0 +1,214 @@
+import asyncio
+import json
+from typing import Optional
+from urllib.parse import quote
+
+async def get_token(session, email: str = "test@example.com") -> str:
+    """Fetch a JWT token from the /token endpoint."""
+    url = "http://localhost:8000/token"
+    payload = {"email": email}
+    print(f"\nFetching token from {url} with email: {email}")
+    try:
+        async with session.post(url, json=payload) as response:
+            status = response.status
+            data = await response.json()
+            print(f"Token Response Status: {status}")
+            print(f"Token Response: {json.dumps(data, indent=2)}")
+            if status == 200:
+                return data["access_token"]
+            else:
+                raise Exception(f"Failed to get token: {data.get('detail', 'Unknown error')}")
+    except Exception as e:
+        print(f"Error fetching token: {str(e)}")
+        raise
+
+async def test_endpoint(
+    session,
+    endpoint: str,
+    url: str,
+    token: str,
+    params: Optional[dict] = None,
+    expected_status: int = 200
+) -> Optional[dict]:
+    """Test an endpoint with token and print results."""
+    params = params or {}
+    param_str = "&".join(f"{k}={v}" for k, v in params.items())
+    full_url = f"http://localhost:8000/{endpoint}/{quote(url)}"
+    if param_str:
+        full_url += f"?{param_str}"
+    
+    headers = {"Authorization": f"Bearer {token}"}
+    print(f"\nTesting: {full_url}")
+    
+    try:
+        async with session.get(full_url, headers=headers) as response:
+            status = response.status
+            try:
+                data = await response.json()
+            except:
+                data = await response.text()
+            
+            print(f"Status: {status} (Expected: {expected_status})")
+            if isinstance(data, dict):
+                print(f"Response: {json.dumps(data, indent=2)}")
+            else:
+                print(f"Response: {data[:500]}...")  # First 500 chars
+            assert status == expected_status, f"Expected {expected_status}, got {status}"
+            return data
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        return None
+
+
+async def test_stream_crawl(session, token: str):
+    """Test the /crawl/stream endpoint with multiple URLs."""
+    url = "http://localhost:8000/crawl/stream"
+    payload = {
+        "urls": [
+            "https://example.com",
+            "https://example.com/page1",  # Replicated example.com with variation
+            "https://example.com/page2",  # Replicated example.com with variation
+            "https://example.com/page3",  # Replicated example.com with variation
+            # "https://www.python.org",
+            # "https://news.ycombinator.com/news"
+        ],
+        "browser_config": {"headless": True, "viewport": {"width": 1200}},
+        "crawler_config": {"stream": True, "cache_mode": "bypass"}
+    }
+    headers = {"Authorization": f"Bearer {token}"}
+    print(f"\nTesting Streaming Crawl: {url}")
+    print(f"Payload: {json.dumps(payload, indent=2)}")
+    
+    try:
+        async with session.post(url, json=payload, headers=headers) as response:
+            status = response.status
+            print(f"Status: {status} (Expected: 200)")
+            assert status == 200, f"Expected 200, got {status}"
+            
+            # Read streaming response line-by-line (NDJSON)
+            async for line in response.content:
+                if line:
+                    data = json.loads(line.decode('utf-8').strip())
+                    print(f"Streamed Result: {json.dumps(data, indent=2)}")
+    except Exception as e:
+        print(f"Error in streaming crawl test: {str(e)}")
+
+async def run_tests():
+    import aiohttp
+    print("Starting API Tests...")
+    
+    # Test URLs
+    urls = [
+        "example.com",
+        "https://www.python.org",
+        "https://news.ycombinator.com/news",
+        "https://github.com/trending"
+    ]
+    
+    async with aiohttp.ClientSession() as session:
+        token = "test_token"
+        # If jwt is enabled, authenticate first
+        # Fetch token once and reuse it
+        # token = await get_token(session)
+        # if not token:
+        #     print("Aborting tests due to token failure!")
+        #     return
+        
+        print("\n=== Testing Crawl Endpoint ===")
+        crawl_payload = {
+            "urls": ["https://example.com"],
+            "browser_config": {"headless": True},
+            "crawler_config": {"stream": False}
+        }
+        async with session.post(
+            "http://localhost:8000/crawl",
+            json=crawl_payload,
+            headers={"Authorization": f"Bearer {token}"}
+        ) as response:
+            status = response.status
+            data = await response.json()
+            print(f"\nCrawl Endpoint Status: {status}")
+            print(f"Crawl Response: {json.dumps(data, indent=2)}")
+        
+
+        print("\n=== Testing Crawl Stream Endpoint ===")
+        await test_stream_crawl(session, token)
+
+        print("\n=== Testing Markdown Endpoint ===")
+        for url in []: #urls:
+            for filter_type in ["raw", "fit", "bm25", "llm"]:
+                params = {"f": filter_type}
+                if filter_type in ["bm25", "llm"]:
+                    params["q"] = "extract main content"
+                
+                for cache in ["0", "1"]:
+                    params["c"] = cache
+                    await test_endpoint(session, "md", url, token, params)
+                    await asyncio.sleep(1)  # Be nice to the server
+
+        print("\n=== Testing LLM Endpoint ===")
+        for url in urls:
+            # Test basic extraction (direct response now)
+            result = await test_endpoint(
+                session,
+                "llm",
+                url,
+                token,
+                {"q": "Extract title and main content"}
+            )
+            
+            # Test with schema (direct response)
+            schema = {
+                "type": "object",
+                "properties": {
+                    "title": {"type": "string"},
+                    "content": {"type": "string"},
+                    "links": {"type": "array", "items": {"type": "string"}}
+                }
+            }
+            result = await test_endpoint(
+                session,
+                "llm",
+                url,
+                token,
+                {
+                    "q": "Extract content with links",
+                    "s": json.dumps(schema),
+                    "c": "1"  # Test with cache
+                }
+            )
+            await asyncio.sleep(2)  # Be nice to the server
+        
+        print("\n=== Testing Error Cases ===")
+        # Test invalid URL
+        await test_endpoint(
+            session,
+            "md",
+            "not_a_real_url",
+            token,
+            expected_status=500
+        )
+        
+        # Test invalid filter type
+        await test_endpoint(
+            session,
+            "md",
+            "example.com",
+            token,
+            {"f": "invalid"},
+            expected_status=422
+        )
+        
+        # Test LLM without query (should fail per your server logic)
+        await test_endpoint(
+            session,
+            "llm",
+            "example.com",
+            token,
+            expected_status=400
+        )
+        
+    print("\nAll tests completed!")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
\ No newline at end of file
diff --git a/docs/examples/docker_python_sdk.py b/docs/examples/docker_python_sdk.py
new file mode 100644
index 0000000..72091da
--- /dev/null
+++ b/docs/examples/docker_python_sdk.py
@@ -0,0 +1,35 @@
+import asyncio
+from crawl4ai.docker_client import Crawl4aiDockerClient
+from crawl4ai import (
+    BrowserConfig,
+    CrawlerRunConfig
+)
+
+async def main():
+    async with Crawl4aiDockerClient(base_url="http://localhost:8000", verbose=True) as client:
+        # If jwt is enabled, authenticate first
+        # await client.authenticate("test@example.com")
+        
+        # Non-streaming crawl
+        results = await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig()
+        )
+        print(f"Non-streaming results: {results}")
+        
+        # Streaming crawl
+        crawler_config = CrawlerRunConfig(stream=True)
+        async for result in await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=crawler_config
+        ):
+            print(f"Streamed result: {result}")
+        
+        # Get schema
+        schema = await client.get_schema()
+        print(f"Schema: {schema}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/docker_webhook_example.py b/docs/examples/docker_webhook_example.py
new file mode 100644
index 0000000..f05d350
--- /dev/null
+++ b/docs/examples/docker_webhook_example.py
@@ -0,0 +1,461 @@
+"""
+Docker Webhook Example for Crawl4AI
+
+This example demonstrates how to use webhooks with the Crawl4AI job queue API.
+Instead of polling for results, webhooks notify your application when jobs complete.
+
+Supports both:
+- /crawl/job - Raw crawling with markdown extraction
+- /llm/job - LLM-powered content extraction
+
+Prerequisites:
+1. Crawl4AI Docker container running on localhost:11235
+2. Flask installed: pip install flask requests
+3. LLM API key configured in .llm.env (for LLM extraction examples)
+
+Usage:
+1. Run this script: python docker_webhook_example.py
+2. The webhook server will start on http://localhost:8080
+3. Jobs will be submitted and webhooks will be received automatically
+"""
+
+import requests
+import json
+import time
+from flask import Flask, request, jsonify
+from threading import Thread
+
+# Configuration
+CRAWL4AI_BASE_URL = "http://localhost:11235"
+WEBHOOK_BASE_URL = "http://localhost:8080"  # Your webhook receiver URL
+
+# Initialize Flask app for webhook receiver
+app = Flask(__name__)
+
+# Store received webhook data for demonstration
+received_webhooks = []
+
+
+@app.route('/webhooks/crawl-complete', methods=['POST'])
+def handle_crawl_webhook():
+    """
+    Webhook handler that receives notifications when crawl jobs complete.
+
+    Payload structure:
+    {
+        "task_id": "crawl_abc123",
+        "task_type": "crawl",
+        "status": "completed" or "failed",
+        "timestamp": "2025-10-21T10:30:00.000000+00:00",
+        "urls": ["https://example.com"],
+        "error": "error message" (only if failed),
+        "data": {...} (only if webhook_data_in_payload=True)
+    }
+    """
+    payload = request.json
+    print(f"\n{'='*60}")
+    print(f"📬 Webhook received for task: {payload['task_id']}")
+    print(f"   Status: {payload['status']}")
+    print(f"   Timestamp: {payload['timestamp']}")
+    print(f"   URLs: {payload['urls']}")
+
+    if payload['status'] == 'completed':
+        # If data is in payload, process it directly
+        if 'data' in payload:
+            print(f"   ✅ Data included in webhook")
+            data = payload['data']
+            # Process the crawl results here
+            for result in data.get('results', []):
+                print(f"      - Crawled: {result.get('url')}")
+                print(f"      - Markdown length: {len(result.get('markdown', ''))}")
+        else:
+            # Fetch results from API if not included
+            print(f"   📥 Fetching results from API...")
+            task_id = payload['task_id']
+            result_response = requests.get(f"{CRAWL4AI_BASE_URL}/crawl/job/{task_id}")
+            if result_response.ok:
+                data = result_response.json()
+                print(f"   ✅ Results fetched successfully")
+                # Process the crawl results here
+                for result in data['result'].get('results', []):
+                    print(f"      - Crawled: {result.get('url')}")
+                    print(f"      - Markdown length: {len(result.get('markdown', ''))}")
+
+    elif payload['status'] == 'failed':
+        print(f"   ❌ Job failed: {payload.get('error', 'Unknown error')}")
+
+    print(f"{'='*60}\n")
+
+    # Store webhook for demonstration
+    received_webhooks.append(payload)
+
+    # Return 200 OK to acknowledge receipt
+    return jsonify({"status": "received"}), 200
+
+
+@app.route('/webhooks/llm-complete', methods=['POST'])
+def handle_llm_webhook():
+    """
+    Webhook handler that receives notifications when LLM extraction jobs complete.
+
+    Payload structure:
+    {
+        "task_id": "llm_1698765432_12345",
+        "task_type": "llm_extraction",
+        "status": "completed" or "failed",
+        "timestamp": "2025-10-21T10:30:00.000000+00:00",
+        "urls": ["https://example.com/article"],
+        "error": "error message" (only if failed),
+        "data": {"extracted_content": {...}} (only if webhook_data_in_payload=True)
+    }
+    """
+    payload = request.json
+    print(f"\n{'='*60}")
+    print(f"🤖 LLM Webhook received for task: {payload['task_id']}")
+    print(f"   Task Type: {payload['task_type']}")
+    print(f"   Status: {payload['status']}")
+    print(f"   Timestamp: {payload['timestamp']}")
+    print(f"   URL: {payload['urls'][0]}")
+
+    if payload['status'] == 'completed':
+        # If data is in payload, process it directly
+        if 'data' in payload:
+            print(f"   ✅ Data included in webhook")
+            data = payload['data']
+            # Webhook wraps extracted content in 'extracted_content' field
+            extracted = data.get('extracted_content', {})
+            print(f"      - Extracted content:")
+            print(f"        {json.dumps(extracted, indent=8)}")
+        else:
+            # Fetch results from API if not included
+            print(f"   📥 Fetching results from API...")
+            task_id = payload['task_id']
+            result_response = requests.get(f"{CRAWL4AI_BASE_URL}/llm/job/{task_id}")
+            if result_response.ok:
+                data = result_response.json()
+                print(f"   ✅ Results fetched successfully")
+                # API returns unwrapped content in 'result' field
+                extracted = data['result']
+                print(f"      - Extracted content:")
+                print(f"        {json.dumps(extracted, indent=8)}")
+
+    elif payload['status'] == 'failed':
+        print(f"   ❌ Job failed: {payload.get('error', 'Unknown error')}")
+
+    print(f"{'='*60}\n")
+
+    # Store webhook for demonstration
+    received_webhooks.append(payload)
+
+    # Return 200 OK to acknowledge receipt
+    return jsonify({"status": "received"}), 200
+
+
+def start_webhook_server():
+    """Start the Flask webhook server in a separate thread"""
+    app.run(host='0.0.0.0', port=8080, debug=False, use_reloader=False)
+
+
+def submit_crawl_job_with_webhook(urls, webhook_url, include_data=False):
+    """
+    Submit a crawl job with webhook notification.
+
+    Args:
+        urls: List of URLs to crawl
+        webhook_url: URL to receive webhook notifications
+        include_data: Whether to include full results in webhook payload
+
+    Returns:
+        task_id: The job's task identifier
+    """
+    payload = {
+        "urls": urls,
+        "browser_config": {"headless": True},
+        "crawler_config": {"cache_mode": "bypass"},
+        "webhook_config": {
+            "webhook_url": webhook_url,
+            "webhook_data_in_payload": include_data,
+            # Optional: Add custom headers for authentication
+            # "webhook_headers": {
+            #     "X-Webhook-Secret": "your-secret-token"
+            # }
+        }
+    }
+
+    print(f"\n🚀 Submitting crawl job...")
+    print(f"   URLs: {urls}")
+    print(f"   Webhook: {webhook_url}")
+    print(f"   Include data: {include_data}")
+
+    response = requests.post(
+        f"{CRAWL4AI_BASE_URL}/crawl/job",
+        json=payload,
+        headers={"Content-Type": "application/json"}
+    )
+
+    if response.ok:
+        data = response.json()
+        task_id = data['task_id']
+        print(f"   ✅ Job submitted successfully")
+        print(f"   Task ID: {task_id}")
+        return task_id
+    else:
+        print(f"   ❌ Failed to submit job: {response.text}")
+        return None
+
+
+def submit_llm_job_with_webhook(url, query, webhook_url, include_data=False, schema=None, provider=None):
+    """
+    Submit an LLM extraction job with webhook notification.
+
+    Args:
+        url: URL to extract content from
+        query: Instruction for the LLM (e.g., "Extract article title and author")
+        webhook_url: URL to receive webhook notifications
+        include_data: Whether to include full results in webhook payload
+        schema: Optional JSON schema for structured extraction
+        provider: Optional LLM provider (e.g., "openai/gpt-4o-mini")
+
+    Returns:
+        task_id: The job's task identifier
+    """
+    payload = {
+        "url": url,
+        "q": query,
+        "cache": False,
+        "webhook_config": {
+            "webhook_url": webhook_url,
+            "webhook_data_in_payload": include_data,
+            # Optional: Add custom headers for authentication
+            # "webhook_headers": {
+            #     "X-Webhook-Secret": "your-secret-token"
+            # }
+        }
+    }
+
+    if schema:
+        payload["schema"] = schema
+
+    if provider:
+        payload["provider"] = provider
+
+    print(f"\n🤖 Submitting LLM extraction job...")
+    print(f"   URL: {url}")
+    print(f"   Query: {query}")
+    print(f"   Webhook: {webhook_url}")
+    print(f"   Include data: {include_data}")
+    if provider:
+        print(f"   Provider: {provider}")
+
+    response = requests.post(
+        f"{CRAWL4AI_BASE_URL}/llm/job",
+        json=payload,
+        headers={"Content-Type": "application/json"}
+    )
+
+    if response.ok:
+        data = response.json()
+        task_id = data['task_id']
+        print(f"   ✅ Job submitted successfully")
+        print(f"   Task ID: {task_id}")
+        return task_id
+    else:
+        print(f"   ❌ Failed to submit job: {response.text}")
+        return None
+
+
+def submit_job_without_webhook(urls):
+    """
+    Submit a job without webhook (traditional polling approach).
+
+    Args:
+        urls: List of URLs to crawl
+
+    Returns:
+        task_id: The job's task identifier
+    """
+    payload = {
+        "urls": urls,
+        "browser_config": {"headless": True},
+        "crawler_config": {"cache_mode": "bypass"}
+    }
+
+    print(f"\n🚀 Submitting crawl job (without webhook)...")
+    print(f"   URLs: {urls}")
+
+    response = requests.post(
+        f"{CRAWL4AI_BASE_URL}/crawl/job",
+        json=payload
+    )
+
+    if response.ok:
+        data = response.json()
+        task_id = data['task_id']
+        print(f"   ✅ Job submitted successfully")
+        print(f"   Task ID: {task_id}")
+        return task_id
+    else:
+        print(f"   ❌ Failed to submit job: {response.text}")
+        return None
+
+
+def poll_job_status(task_id, timeout=60):
+    """
+    Poll for job status (used when webhook is not configured).
+
+    Args:
+        task_id: The job's task identifier
+        timeout: Maximum time to wait in seconds
+    """
+    print(f"\n⏳ Polling for job status...")
+    start_time = time.time()
+
+    while time.time() - start_time < timeout:
+        response = requests.get(f"{CRAWL4AI_BASE_URL}/crawl/job/{task_id}")
+
+        if response.ok:
+            data = response.json()
+            status = data.get('status', 'unknown')
+
+            if status == 'completed':
+                print(f"   ✅ Job completed!")
+                return data
+            elif status == 'failed':
+                print(f"   ❌ Job failed: {data.get('error', 'Unknown error')}")
+                return data
+            else:
+                print(f"   ⏳ Status: {status}, waiting...")
+                time.sleep(2)
+        else:
+            print(f"   ❌ Failed to get status: {response.text}")
+            return None
+
+    print(f"   ⏰ Timeout reached")
+    return None
+
+
+def main():
+    """Run the webhook demonstration"""
+
+    # Check if Crawl4AI is running
+    try:
+        health = requests.get(f"{CRAWL4AI_BASE_URL}/health", timeout=5)
+        print(f"✅ Crawl4AI is running: {health.json()}")
+    except:
+        print(f"❌ Cannot connect to Crawl4AI at {CRAWL4AI_BASE_URL}")
+        print("   Please make sure Docker container is running:")
+        print("   docker run -d -p 11235:11235 --name crawl4ai unclecode/crawl4ai:latest")
+        return
+
+    # Start webhook server in background thread
+    print(f"\n🌐 Starting webhook server at {WEBHOOK_BASE_URL}...")
+    webhook_thread = Thread(target=start_webhook_server, daemon=True)
+    webhook_thread.start()
+    time.sleep(2)  # Give server time to start
+
+    # Example 1: Job with webhook (notification only, fetch data separately)
+    print(f"\n{'='*60}")
+    print("Example 1: Webhook Notification Only")
+    print(f"{'='*60}")
+    task_id_1 = submit_crawl_job_with_webhook(
+        urls=["https://example.com"],
+        webhook_url=f"{WEBHOOK_BASE_URL}/webhooks/crawl-complete",
+        include_data=False
+    )
+
+    # Example 2: Job with webhook (data included in payload)
+    time.sleep(5)  # Wait a bit between requests
+    print(f"\n{'='*60}")
+    print("Example 2: Webhook with Full Data")
+    print(f"{'='*60}")
+    task_id_2 = submit_crawl_job_with_webhook(
+        urls=["https://www.python.org"],
+        webhook_url=f"{WEBHOOK_BASE_URL}/webhooks/crawl-complete",
+        include_data=True
+    )
+
+    # Example 3: LLM extraction with webhook (notification only)
+    time.sleep(5)  # Wait a bit between requests
+    print(f"\n{'='*60}")
+    print("Example 3: LLM Extraction with Webhook (Notification Only)")
+    print(f"{'='*60}")
+    task_id_3 = submit_llm_job_with_webhook(
+        url="https://www.example.com",
+        query="Extract the main heading and description from this page.",
+        webhook_url=f"{WEBHOOK_BASE_URL}/webhooks/llm-complete",
+        include_data=False,
+        provider="openai/gpt-4o-mini"
+    )
+
+    # Example 4: LLM extraction with webhook (data included + schema)
+    time.sleep(5)  # Wait a bit between requests
+    print(f"\n{'='*60}")
+    print("Example 4: LLM Extraction with Schema and Full Data")
+    print(f"{'='*60}")
+
+    # Define a schema for structured extraction
+    schema = json.dumps({
+        "type": "object",
+        "properties": {
+            "title": {"type": "string", "description": "Page title"},
+            "description": {"type": "string", "description": "Page description"}
+        },
+        "required": ["title"]
+    })
+
+    task_id_4 = submit_llm_job_with_webhook(
+        url="https://www.python.org",
+        query="Extract the title and description of this website",
+        webhook_url=f"{WEBHOOK_BASE_URL}/webhooks/llm-complete",
+        include_data=True,
+        schema=schema,
+        provider="openai/gpt-4o-mini"
+    )
+
+    # Example 5: Traditional polling (no webhook)
+    time.sleep(5)  # Wait a bit between requests
+    print(f"\n{'='*60}")
+    print("Example 5: Traditional Polling (No Webhook)")
+    print(f"{'='*60}")
+    task_id_5 = submit_job_without_webhook(
+        urls=["https://github.com"]
+    )
+    if task_id_5:
+        result = poll_job_status(task_id_5)
+        if result and result.get('status') == 'completed':
+            print(f"   ✅ Results retrieved via polling")
+
+    # Wait for webhooks to arrive
+    print(f"\n⏳ Waiting for webhooks to be received...")
+    time.sleep(30)  # Give jobs time to complete and webhooks to arrive (longer for LLM)
+
+    # Summary
+    print(f"\n{'='*60}")
+    print("Summary")
+    print(f"{'='*60}")
+    print(f"Total webhooks received: {len(received_webhooks)}")
+
+    crawl_webhooks = [w for w in received_webhooks if w['task_type'] == 'crawl']
+    llm_webhooks = [w for w in received_webhooks if w['task_type'] == 'llm_extraction']
+
+    print(f"\n📊 Breakdown:")
+    print(f"   - Crawl webhooks: {len(crawl_webhooks)}")
+    print(f"   - LLM extraction webhooks: {len(llm_webhooks)}")
+
+    print(f"\n📋 Details:")
+    for i, webhook in enumerate(received_webhooks, 1):
+        task_type = webhook['task_type']
+        icon = "🕷️" if task_type == "crawl" else "🤖"
+        print(f"{i}. {icon} Task {webhook['task_id']}: {webhook['status']} ({task_type})")
+
+    print(f"\n✅ Demo completed!")
+    print(f"\n💡 Pro tips:")
+    print(f"   - In production, your webhook URL should be publicly accessible")
+    print(f"     (e.g., https://myapp.com/webhooks) or use ngrok for testing")
+    print(f"   - Both /crawl/job and /llm/job support the same webhook configuration")
+    print(f"   - Use webhook_data_in_payload=true to get results directly in the webhook")
+    print(f"   - LLM jobs may take longer, adjust timeouts accordingly")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/docs/examples/domain_mapper/domain_mapper_demo.py b/docs/examples/domain_mapper/domain_mapper_demo.py
new file mode 100644
index 0000000..0270d47
--- /dev/null
+++ b/docs/examples/domain_mapper/domain_mapper_demo.py
@@ -0,0 +1,82 @@
+"""
+DomainMapper Demo — Comprehensive domain URL discovery
+
+Discovers all URLs under a domain using 8 sources:
+sitemap, Common Crawl, Wayback Machine, Certificate Transparency,
+path probing, robots.txt mining, RSS/Atom feeds, homepage link extraction.
+
+Usage:
+    python domain_mapper_demo.py [domain] [--source SOURCE] [--query QUERY]
+
+Examples:
+    python domain_mapper_demo.py superdesign.dev
+    python domain_mapper_demo.py example.com --source sitemap+crt+probe
+    python domain_mapper_demo.py docs.crawl4ai.com --query "extraction tutorial"
+"""
+
+import asyncio
+import argparse
+from collections import defaultdict
+from crawl4ai import DomainMapper, DomainMapperConfig
+
+
+async def main():
+    parser = argparse.ArgumentParser(description="DomainMapper Demo")
+    parser.add_argument("domain", help="Domain to scan (e.g., example.com)")
+    parser.add_argument("--source", default="sitemap+cc+crt+probe+robots+homepage",
+                        help="Discovery sources (default: sitemap+cc+crt+probe+robots+homepage)")
+    parser.add_argument("--query", default=None, help="BM25 relevance query")
+    parser.add_argument("--max-urls", type=int, default=-1, help="Max URLs to return")
+    parser.add_argument("--no-head", action="store_true", help="Skip head extraction")
+    args = parser.parse_args()
+
+    config = DomainMapperConfig(
+        source=args.source,
+        extract_head=not args.no_head,
+        query=args.query,
+        max_urls=args.max_urls,
+        verbose=True,
+        force=True,
+    )
+
+    async with DomainMapper() as mapper:
+        results = await mapper.scan(args.domain, config)
+
+    # Group by host
+    by_host = defaultdict(list)
+    for r in results:
+        by_host[r["host"]].append(r)
+
+    # Print results
+    print("\n" + "=" * 70)
+    print(f"RESULTS: {len(results)} URLs across {len(by_host)} hosts")
+    print("=" * 70)
+
+    for host in sorted(by_host.keys()):
+        urls = by_host[host]
+        print(f"\n  {host} ({len(urls)} URLs):")
+        for r in urls[:10]:
+            title = r.get("head_data", {}).get("title", "")
+            score = r.get("relevance_score")
+            line = f"    [{r['source']}] {r['url']}"
+            if title:
+                line += f"\n      Title: {title}"
+            if score is not None:
+                line += f"\n      Score: {score:.3f}"
+            print(line)
+        if len(urls) > 10:
+            print(f"    ... and {len(urls) - 10} more")
+
+    # Source breakdown
+    source_counts = defaultdict(int)
+    for r in results:
+        for s in r["source"].split("+"):
+            source_counts[s] += 1
+
+    print(f"\nSource breakdown:")
+    for source, count in sorted(source_counts.items(), key=lambda x: -x[1]):
+        print(f"  {source}: {count} URLs")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/extraction_strategies_examples.py b/docs/examples/extraction_strategies_examples.py
new file mode 100644
index 0000000..5f072f8
--- /dev/null
+++ b/docs/examples/extraction_strategies_examples.py
@@ -0,0 +1,125 @@
+"""
+Example demonstrating different extraction strategies with various input formats.
+This example shows how to:
+1. Use different input formats (markdown, HTML, fit_markdown)
+2. Work with JSON-based extractors (CSS and XPath)
+3. Use LLM-based extraction with different input formats
+4. Configure browser and crawler settings properly
+"""
+
+import asyncio
+import os
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai import LLMConfig
+from crawl4ai import (
+    LLMExtractionStrategy,
+    JsonCssExtractionStrategy,
+    JsonXPathExtractionStrategy,
+)
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+
+async def run_extraction(crawler: AsyncWebCrawler, url: str, strategy, name: str):
+    """Helper function to run extraction with proper configuration"""
+    try:
+        # Configure the crawler run settings
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            extraction_strategy=strategy,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter()  # For fit_markdown support
+            ),
+        )
+
+        # Run the crawler
+        result = await crawler.arun(url=url, config=config)
+
+        if result.success:
+            print(f"\n=== {name} Results ===")
+            print(f"Extracted Content: {result.extracted_content}")
+            print(f"Raw Markdown Length: {len(result.markdown.raw_markdown)}")
+            print(
+                f"Citations Markdown Length: {len(result.markdown.markdown_with_citations)}"
+            )
+        else:
+            print(f"Error in {name}: Crawl failed")
+
+    except Exception as e:
+        print(f"Error in {name}: {str(e)}")
+
+
+async def main():
+    # Example URL (replace with actual URL)
+    url = "https://example.com/product-page"
+
+    # Configure browser settings
+    browser_config = BrowserConfig(headless=True, verbose=True)
+
+    # Initialize extraction strategies
+
+    # 1. LLM Extraction with different input formats
+    markdown_strategy = LLMExtractionStrategy(
+        llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY")),
+        instruction="Extract product information including name, price, and description",
+    )
+
+    html_strategy = LLMExtractionStrategy(
+        input_format="html",
+        llm_config=LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY")),
+        instruction="Extract product information from HTML including structured data",
+    )
+
+    fit_markdown_strategy = LLMExtractionStrategy(
+        input_format="fit_markdown",
+        llm_config=LLMConfig(provider="openai/gpt-4o-mini",api_token=os.getenv("OPENAI_API_KEY")),
+        instruction="Extract product information from cleaned markdown",
+    )
+
+    # 2. JSON CSS Extraction (automatically uses HTML input)
+    css_schema = {
+        "baseSelector": ".product",
+        "fields": [
+            {"name": "title", "selector": "h1.product-title", "type": "text"},
+            {"name": "price", "selector": ".price", "type": "text"},
+            {"name": "description", "selector": ".description", "type": "text"},
+        ],
+    }
+    css_strategy = JsonCssExtractionStrategy(schema=css_schema)
+
+    # 3. JSON XPath Extraction (automatically uses HTML input)
+    xpath_schema = {
+        "baseSelector": "//div[@class='product']",
+        "fields": [
+            {
+                "name": "title",
+                "selector": ".//h1[@class='product-title']/text()",
+                "type": "text",
+            },
+            {
+                "name": "price",
+                "selector": ".//span[@class='price']/text()",
+                "type": "text",
+            },
+            {
+                "name": "description",
+                "selector": ".//div[@class='description']/text()",
+                "type": "text",
+            },
+        ],
+    }
+    xpath_strategy = JsonXPathExtractionStrategy(schema=xpath_schema)
+
+    # Use context manager for proper resource handling
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Run all strategies
+        await run_extraction(crawler, url, markdown_strategy, "Markdown LLM")
+        await run_extraction(crawler, url, html_strategy, "HTML LLM")
+        await run_extraction(crawler, url, fit_markdown_strategy, "Fit Markdown LLM")
+        await run_extraction(crawler, url, css_strategy, "CSS Extraction")
+        await run_extraction(crawler, url, xpath_strategy, "XPath Extraction")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/full_page_screenshot_and_pdf_export.md b/docs/examples/full_page_screenshot_and_pdf_export.md
new file mode 100644
index 0000000..6d7c632
--- /dev/null
+++ b/docs/examples/full_page_screenshot_and_pdf_export.md
@@ -0,0 +1,108 @@
+# Capturing Full-Page Screenshots and PDFs from Massive Webpages with Crawl4AI
+
+When dealing with very long web pages, traditional full-page screenshots can be slow or fail entirely. For large pages (like extensive Wikipedia articles), generating a single massive screenshot often leads to delays, memory issues, or style differences.
+
+**The New Approach:**
+We’ve introduced a new feature that effortlessly handles even the biggest pages by first exporting them as a PDF, then converting that PDF into a high-quality image. This approach leverages the browser’s built-in PDF rendering, making it both stable and efficient for very long content. You also have the option to directly save the PDF for your own usage—no need for multiple passes or complex stitching logic.
+
+**Key Benefits:**
+- **Reliability:** The PDF export never times out and works regardless of page length.
+- **Versatility:** Get both the PDF and a screenshot in one crawl, without reloading or reprocessing.
+- **Performance:** Skips manual scrolling and stitching images, reducing complexity and runtime.
+
+**Simple Example:**
+```python
+import os
+import sys
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
+
+# Adjust paths as needed
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        # Request both PDF and screenshot
+        result = await crawler.arun(
+            url='https://en.wikipedia.org/wiki/List_of_common_misconceptions',
+            config=CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS,
+                pdf=True,
+                screenshot=True
+            )
+        )
+        
+        if result.success:
+            # Save screenshot
+            if result.screenshot:
+                from base64 import b64decode
+                with open(os.path.join(__location__, "screenshot.png"), "wb") as f:
+                    f.write(b64decode(result.screenshot))
+            
+            # Save PDF
+            if result.pdf:
+                with open(os.path.join(__location__, "page.pdf"), "wb") as f:
+                    f.write(result.pdf)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**What Happens Under the Hood:**
+- Crawl4AI navigates to the target page.
+- If `pdf=True`, it exports the current page as a full PDF, capturing all of its content no matter the length.
+- If `screenshot=True`, and a PDF is already available, it directly converts the first page of that PDF to an image for you—no repeated loading or scrolling.
+- Finally, you get your PDF and/or screenshot ready to use.
+
+**Controlling scroll speed for full-page screenshots:**  
+When a page is taller than `screenshot_height_threshold` (default ~20,000px) and no PDF is available, Crawl4AI scrolls through the page to capture a stitched full-page screenshot. Use `scroll_delay` to control the pause between scroll steps:
+
+```python
+config = CrawlerRunConfig(
+    screenshot=True,
+    scroll_delay=0.5,  # Wait 0.5s between scroll steps (default: 0.2)
+)
+```
+
+This is particularly useful for pages with lazy-loaded images or animations that need time to render during scrolling.
+
+---
+
+## Viewport-Only Screenshots
+
+If you only need a screenshot of the visible viewport (not the entire page), use the `force_viewport_screenshot` option. This is significantly faster and produces smaller images, especially for long pages.
+
+```python
+import os
+import asyncio
+from base64 import b64decode
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url='https://en.wikipedia.org/wiki/List_of_common_misconceptions',
+            config=CrawlerRunConfig(
+                screenshot=True,
+                force_viewport_screenshot=True  # Only capture the visible viewport
+            )
+        )
+
+        if result.success and result.screenshot:
+            with open("viewport_screenshot.png", "wb") as f:
+                f.write(b64decode(result.screenshot))
+            print("Viewport screenshot saved!")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**When to use viewport-only screenshots:**
+- You only need what's visible above the fold
+- You want faster screenshot capture on long pages
+- You need smaller image sizes for thumbnails or previews
+
+**Conclusion:**
+With this feature, Crawl4AI becomes even more robust and versatile for large-scale content extraction. Whether you need a PDF snapshot or a quick screenshot, you now have a reliable solution for even the most extensive webpages.
\ No newline at end of file
diff --git a/docs/examples/hello_world.py b/docs/examples/hello_world.py
new file mode 100644
index 0000000..2ba2e85
--- /dev/null
+++ b/docs/examples/hello_world.py
@@ -0,0 +1,29 @@
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    DefaultMarkdownGenerator,
+    PruningContentFilter,
+    CrawlResult
+)
+
+
+async def main():
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True,
+    )
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        crawler_config = CrawlerRunConfig(
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter()
+            ),
+        )
+        result: CrawlResult = await crawler.arun(
+            url="https://www.helloworld.org", config=crawler_config
+        )
+        print(result.markdown.raw_markdown[:500])
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/hello_world_undetected.py b/docs/examples/hello_world_undetected.py
new file mode 100644
index 0000000..83ce51e
--- /dev/null
+++ b/docs/examples/hello_world_undetected.py
@@ -0,0 +1,57 @@
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    DefaultMarkdownGenerator,
+    PruningContentFilter,
+    CrawlResult,
+    UndetectedAdapter
+)
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+
+async def main():
+    # Create browser config
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True,
+    )
+    
+    # Create the undetected adapter
+    undetected_adapter = UndetectedAdapter()
+    
+    # Create the crawler strategy with the undetected adapter
+    crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=undetected_adapter
+    )
+    
+    # Create the crawler with our custom strategy
+    async with AsyncWebCrawler(
+        crawler_strategy=crawler_strategy,
+        config=browser_config
+    ) as crawler:
+        # Configure the crawl
+        crawler_config = CrawlerRunConfig(
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter()
+            ),
+            capture_console_messages=True,  # Enable console capture to test adapter
+        )
+        
+        # Test on a site that typically detects bots
+        print("Testing undetected adapter...")
+        result: CrawlResult = await crawler.arun(
+            url="https://www.helloworld.org", 
+            config=crawler_config
+        )
+        
+        print(f"Status: {result.status_code}")
+        print(f"Success: {result.success}")
+        print(f"Console messages captured: {len(result.console_messages or [])}")
+        print(f"Markdown content (first 500 chars):\n{result.markdown.raw_markdown[:500]}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/hooks_example.py b/docs/examples/hooks_example.py
new file mode 100644
index 0000000..de0aa6e
--- /dev/null
+++ b/docs/examples/hooks_example.py
@@ -0,0 +1,118 @@
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from playwright.async_api import Page, BrowserContext
+
+
+async def main():
+    print("🔗 Hooks Example: Demonstrating different hook use cases")
+
+    # Configure browser settings
+    browser_config = BrowserConfig(headless=True)
+
+    # Configure crawler settings
+    crawler_run_config = CrawlerRunConfig(
+        js_code="window.scrollTo(0, document.body.scrollHeight);",
+        wait_for="body",
+        cache_mode=CacheMode.BYPASS,
+    )
+
+    # Create crawler instance
+    crawler = AsyncWebCrawler(config=browser_config)
+
+    # Define and set hook functions
+    async def on_browser_created(browser, context: BrowserContext, **kwargs):
+        """Hook called after the browser is created"""
+        print("[HOOK] on_browser_created - Browser is ready!")
+        # Example: Set a cookie that will be used for all requests
+        return browser
+
+    async def on_page_context_created(page: Page, context: BrowserContext, **kwargs):
+        """Hook called after a new page and context are created"""
+        print("[HOOK] on_page_context_created - New page created!")
+        # Example: Set default viewport size
+        await context.add_cookies(
+            [
+                {
+                    "name": "session_id",
+                    "value": "example_session",
+                    "domain": ".example.com",
+                    "path": "/",
+                }
+            ]
+        )
+        await page.set_viewport_size({"width": 1080, "height": 800})
+        return page
+
+    async def on_user_agent_updated(
+        page: Page, context: BrowserContext, user_agent: str, **kwargs
+    ):
+        """Hook called when the user agent is updated"""
+        print(f"[HOOK] on_user_agent_updated - New user agent: {user_agent}")
+        return page
+
+    async def on_execution_started(page: Page, context: BrowserContext, **kwargs):
+        """Hook called after custom JavaScript execution"""
+        print("[HOOK] on_execution_started - Custom JS executed!")
+        return page
+
+    async def before_goto(page: Page, context: BrowserContext, url: str, **kwargs):
+        """Hook called before navigating to each URL"""
+        print(f"[HOOK] before_goto - About to visit: {url}")
+        # Example: Add custom headers for the request
+        await page.set_extra_http_headers({"Custom-Header": "my-value"})
+        return page
+
+    async def after_goto(
+        page: Page, context: BrowserContext, url: str, response: dict, **kwargs
+    ):
+        """Hook called after navigating to each URL"""
+        print(f"[HOOK] after_goto - Successfully loaded: {url}")
+        # Example: Wait for a specific element to be loaded
+        try:
+            await page.wait_for_selector(".content", timeout=1000)
+            print("Content element found!")
+        except:
+            print("Content element not found, continuing anyway")
+        return page
+
+    async def before_retrieve_html(page: Page, context: BrowserContext, **kwargs):
+        """Hook called before retrieving the HTML content"""
+        print("[HOOK] before_retrieve_html - About to get HTML content")
+        # Example: Scroll to bottom to trigger lazy loading
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+        return page
+
+    async def before_return_html(
+        page: Page, context: BrowserContext, html: str, **kwargs
+    ):
+        """Hook called before returning the HTML content"""
+        print(f"[HOOK] before_return_html - Got HTML content (length: {len(html)})")
+        # Example: You could modify the HTML content here if needed
+        return page
+
+    # Set all the hooks
+    crawler.crawler_strategy.set_hook("on_browser_created", on_browser_created)
+    crawler.crawler_strategy.set_hook(
+        "on_page_context_created", on_page_context_created
+    )
+    crawler.crawler_strategy.set_hook("on_user_agent_updated", on_user_agent_updated)
+    crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+    crawler.crawler_strategy.set_hook("before_goto", before_goto)
+    crawler.crawler_strategy.set_hook("after_goto", after_goto)
+    crawler.crawler_strategy.set_hook("before_retrieve_html", before_retrieve_html)
+    crawler.crawler_strategy.set_hook("before_return_html", before_return_html)
+
+    await crawler.start()
+
+    # Example usage: crawl a simple website
+    url = "https://example.com"
+    result = await crawler.arun(url, config=crawler_run_config)
+    print(f"\nCrawled URL: {result.url}")
+    print(f"HTML length: {len(result.html)}")
+
+    await crawler.close()
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())
diff --git a/docs/examples/identity_based_browsing.py b/docs/examples/identity_based_browsing.py
new file mode 100644
index 0000000..0159694
--- /dev/null
+++ b/docs/examples/identity_based_browsing.py
@@ -0,0 +1,108 @@
+"""
+Identity-Based Browsing Example with Crawl4AI
+
+This example demonstrates how to:
+1. Create a persistent browser profile interactively
+2. List available profiles
+3. Use a saved profile for crawling authenticated sites
+4. Delete profiles when no longer needed
+
+Uses the new BrowserProfiler class for profile management.
+"""
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+from crawl4ai.browser_profiler import BrowserProfiler
+from crawl4ai.async_logger import AsyncLogger
+from colorama import Fore, Style, init
+
+# Initialize colorama
+init()
+
+# Create a shared logger instance
+logger = AsyncLogger(verbose=True)
+
+# Create a shared BrowserProfiler instance
+profiler = BrowserProfiler(logger=logger)
+
+
+async def crawl_with_profile(profile_path, url):
+    """Use a profile to crawl an authenticated page"""
+    logger.info(f"\nCrawling {Fore.CYAN}{url}{Style.RESET_ALL} using profile at {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="CRAWL")
+    
+    # Create browser config with the profile path
+    browser_config = BrowserConfig(
+        headless=False,  # Set to False if you want to see the browser window
+        use_managed_browser=True,  # Required for persistent profiles
+        user_data_dir=profile_path
+    )
+    
+    start_time = asyncio.get_event_loop().time()
+    
+    # Initialize crawler with the browser config
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Crawl the URL - You should have access to authenticated content now
+        result = await crawler.arun(url)
+        
+        elapsed_time = asyncio.get_event_loop().time() - start_time
+        
+        if result.success:
+            # Use url_status method for consistent logging
+            logger.url_status(url, True, elapsed_time, tag="CRAWL")
+            
+            # Print page title or some indication of success
+            title = result.metadata.get("title", "")
+            logger.success(f"Page title: {Fore.GREEN}{title}{Style.RESET_ALL}", tag="CRAWL")
+            return result
+        else:
+            # Log error status
+            logger.error_status(url, result.error_message, tag="CRAWL")
+            return None
+
+
+async def main():
+    logger.info(f"{Fore.CYAN}Identity-Based Browsing Example with Crawl4AI{Style.RESET_ALL}", tag="DEMO")
+    logger.info("This example demonstrates using profiles for authenticated browsing", tag="DEMO")
+    
+    # Choose between interactive mode and automatic mode
+    mode = input(f"{Fore.CYAN}Run in [i]nteractive mode or [a]utomatic mode? (i/a): {Style.RESET_ALL}").lower()
+    
+    if mode == 'i':
+        # Interactive profile management - use the interactive_manager method
+        # Pass the crawl_with_profile function as the callback for the "crawl a website" option
+        await profiler.interactive_manager(crawl_callback=crawl_with_profile)
+    else:
+        # Automatic mode - simplified example
+        profiles = profiler.list_profiles()
+        
+        if not profiles:
+            # Create a new profile if none exists
+            logger.info("No profiles found. Creating a new one...", tag="DEMO")
+            profile_path = await profiler.create_profile()
+            if not profile_path:
+                logger.error("Cannot proceed without a valid profile", tag="DEMO")
+                return
+        else:
+            # Use the first (most recent) profile
+            profile_path = profiles[0]["path"]
+            logger.info(f"Using existing profile: {Fore.CYAN}{profiles[0]['name']}{Style.RESET_ALL}", tag="DEMO")
+        
+        # Example: Crawl an authenticated page
+        urls_to_crawl = [
+            "https://github.com/settings/profile",  # GitHub requires login
+            # "https://twitter.com/home",  # Twitter requires login
+            # "https://www.linkedin.com/feed/",  # LinkedIn requires login
+        ]
+        
+        for url in urls_to_crawl:
+            await crawl_with_profile(profile_path, url)
+
+
+if __name__ == "__main__":
+    try:
+        # Run the async main function
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        logger.warning("Example interrupted by user", tag="DEMO")
+    except Exception as e:
+        logger.error(f"Error in example: {str(e)}", tag="DEMO")
\ No newline at end of file
diff --git a/docs/examples/language_support_example.py b/docs/examples/language_support_example.py
new file mode 100644
index 0000000..712db2c
--- /dev/null
+++ b/docs/examples/language_support_example.py
@@ -0,0 +1,50 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler, AsyncPlaywrightCrawlerStrategy
+
+
+async def main():
+    # Example 1: Setting language when creating the crawler
+    crawler1 = AsyncWebCrawler(
+        crawler_strategy=AsyncPlaywrightCrawlerStrategy(
+            headers={"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7"}
+        )
+    )
+    result1 = await crawler1.arun("https://www.example.com")
+    print(
+        "Example 1 result:", result1.extracted_content[:100]
+    )  # Print first 100 characters
+
+    # Example 2: Setting language before crawling
+    crawler2 = AsyncWebCrawler()
+    crawler2.crawler_strategy.headers[
+        "Accept-Language"
+    ] = "es-ES,es;q=0.9,en-US;q=0.8,en;q=0.7"
+    result2 = await crawler2.arun("https://www.example.com")
+    print("Example 2 result:", result2.extracted_content[:100])
+
+    # Example 3: Setting language when calling arun method
+    crawler3 = AsyncWebCrawler()
+    result3 = await crawler3.arun(
+        "https://www.example.com",
+        headers={"Accept-Language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"},
+    )
+    print("Example 3 result:", result3.extracted_content[:100])
+
+    # Example 4: Crawling multiple pages with different languages
+    urls = [
+        ("https://www.example.com", "fr-FR,fr;q=0.9"),
+        ("https://www.example.org", "es-ES,es;q=0.9"),
+        ("https://www.example.net", "de-DE,de;q=0.9"),
+    ]
+
+    crawler4 = AsyncWebCrawler()
+    results = await asyncio.gather(
+        *[crawler4.arun(url, headers={"Accept-Language": lang}) for url, lang in urls]
+    )
+
+    for url, result in zip([u for u, _ in urls], results):
+        print(f"Result for {url}:", result.extracted_content[:100])
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/link_head_extraction_example.py b/docs/examples/link_head_extraction_example.py
new file mode 100644
index 0000000..500566a
--- /dev/null
+++ b/docs/examples/link_head_extraction_example.py
@@ -0,0 +1,376 @@
+#!/usr/bin/env python3
+"""
+Link Head Extraction & Scoring Example
+
+This example demonstrates Crawl4AI's advanced link analysis capabilities:
+1. Basic link head extraction
+2. Three-layer scoring system (intrinsic, contextual, total)
+3. Pattern-based filtering
+4. Multiple practical use cases
+
+Requirements:
+- crawl4ai installed
+- Internet connection
+
+Usage:
+    python link_head_extraction_example.py
+"""
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai import LinkPreviewConfig
+
+
+async def basic_link_head_extraction():
+    """
+    Basic example: Extract head content from internal links with scoring
+    """
+    print("🔗 Basic Link Head Extraction Example")
+    print("=" * 50)
+    
+    config = CrawlerRunConfig(
+        # Enable link head extraction
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,           # Process internal links
+            include_external=False,          # Skip external links for this demo
+            max_links=5,                    # Limit to 5 links
+            concurrency=3,                  # Process 3 links simultaneously
+            timeout=10,                     # 10 second timeout per link
+            query="API documentation guide", # Query for relevance scoring
+            verbose=True                    # Show detailed progress
+        ),
+        # Enable intrinsic link scoring
+        score_links=True,
+        only_text=True
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://docs.python.org/3/", config=config)
+        
+        if result.success:
+            print(f"\n✅ Successfully crawled: {result.url}")
+            
+            internal_links = result.links.get("internal", [])
+            links_with_head = [link for link in internal_links 
+                             if link.get("head_data") is not None]
+            
+            print(f"🧠 Links with head data: {len(links_with_head)}")
+            
+            # Show detailed results
+            for i, link in enumerate(links_with_head[:3]):
+                print(f"\n📄 Link {i+1}: {link['href']}")
+                print(f"   Text: '{link.get('text', 'No text')[:50]}...'")
+                
+                # Show all three score types
+                intrinsic = link.get('intrinsic_score')
+                contextual = link.get('contextual_score') 
+                total = link.get('total_score')
+                
+                print(f"   📊 Scores:")
+                if intrinsic is not None:
+                    print(f"      • Intrinsic: {intrinsic:.2f}/10.0")
+                if contextual is not None:
+                    print(f"      • Contextual: {contextual:.3f}")
+                if total is not None:
+                    print(f"      • Total: {total:.3f}")
+                
+                # Show head data
+                head_data = link.get("head_data", {})
+                if head_data:
+                    title = head_data.get("title", "No title")
+                    description = head_data.get("meta", {}).get("description", "")
+                    print(f"   📰 Title: {title[:60]}...")
+                    if description:
+                        print(f"   📝 Description: {description[:80]}...")
+        else:
+            print(f"❌ Crawl failed: {result.error_message}")
+
+
+async def research_assistant_example():
+    """
+    Research Assistant: Find highly relevant documentation pages
+    """
+    print("\n\n🔍 Research Assistant Example")
+    print("=" * 50)
+    
+    config = CrawlerRunConfig(
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,
+            include_external=True,
+            include_patterns=["*/docs/*", "*/tutorial/*", "*/guide/*"],
+            exclude_patterns=["*/login*", "*/admin*"],
+            query="machine learning neural networks deep learning",
+            max_links=15,
+            score_threshold=0.4,  # Only include high-relevance links
+            concurrency=8,
+            verbose=False  # Clean output for this example
+        ),
+        score_links=True
+    )
+    
+    # Test with scikit-learn documentation
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://scikit-learn.org/stable/", config=config)
+        
+        if result.success:
+            print(f"✅ Analyzed: {result.url}")
+            
+            all_links = result.links.get("internal", []) + result.links.get("external", [])
+            
+            # Filter for high-scoring links
+            high_scoring_links = [link for link in all_links 
+                                if link.get("total_score", 0) > 0.6]
+            
+            # Sort by total score (highest first)
+            high_scoring_links.sort(key=lambda x: x.get("total_score", 0), reverse=True)
+            
+            print(f"\n🎯 Found {len(high_scoring_links)} highly relevant links:")
+            print("   (Showing top 5 by relevance score)")
+            
+            for i, link in enumerate(high_scoring_links[:5]):
+                score = link.get("total_score", 0)
+                title = link.get("head_data", {}).get("title", "No title")
+                print(f"\n{i+1}. ⭐ {score:.3f} - {title[:70]}...")
+                print(f"   🔗 {link['href']}")
+                
+                # Show score breakdown
+                intrinsic = link.get('intrinsic_score', 0)
+                contextual = link.get('contextual_score', 0)
+                print(f"   📊 Quality: {intrinsic:.1f}/10 | Relevance: {contextual:.3f}")
+        else:
+            print(f"❌ Research failed: {result.error_message}")
+
+
+async def api_discovery_example():
+    """
+    API Discovery: Find API endpoints and references
+    """
+    print("\n\n🔧 API Discovery Example")
+    print("=" * 50)
+    
+    config = CrawlerRunConfig(
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,
+            include_patterns=["*/api/*", "*/reference/*", "*/endpoint/*"],
+            exclude_patterns=["*/deprecated/*", "*/v1/*"],  # Skip old versions
+            max_links=25,
+            concurrency=10,
+            timeout=8,
+            verbose=False
+        ),
+        score_links=True
+    )
+    
+    # Example with a documentation site that has API references
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://httpbin.org/", config=config)
+        
+        if result.success:
+            print(f"✅ Discovered APIs at: {result.url}")
+            
+            api_links = result.links.get("internal", [])
+            
+            # Categorize by detected content
+            endpoints = {"GET": [], "POST": [], "PUT": [], "DELETE": [], "OTHER": []}
+            
+            for link in api_links:
+                if link.get("head_data"):
+                    title = link.get("head_data", {}).get("title", "").upper()
+                    text = link.get("text", "").upper()
+                    
+                    # Simple categorization based on content
+                    if "GET" in title or "GET" in text:
+                        endpoints["GET"].append(link)
+                    elif "POST" in title or "POST" in text:
+                        endpoints["POST"].append(link)
+                    elif "PUT" in title or "PUT" in text:
+                        endpoints["PUT"].append(link)
+                    elif "DELETE" in title or "DELETE" in text:
+                        endpoints["DELETE"].append(link)
+                    else:
+                        endpoints["OTHER"].append(link)
+            
+            # Display results
+            total_found = sum(len(links) for links in endpoints.values())
+            print(f"\n📡 Found {total_found} API-related links:")
+            
+            for method, links in endpoints.items():
+                if links:
+                    print(f"\n{method} Endpoints ({len(links)}):")
+                    for link in links[:3]:  # Show first 3 of each type
+                        title = link.get("head_data", {}).get("title", "No title")
+                        score = link.get("intrinsic_score", 0)
+                        print(f"  • [{score:.1f}] {title[:50]}...")
+                        print(f"    {link['href']}")
+        else:
+            print(f"❌ API discovery failed: {result.error_message}")
+
+
+async def link_quality_analysis():
+    """
+    Link Quality Analysis: Analyze website structure and link quality
+    """
+    print("\n\n📊 Link Quality Analysis Example")
+    print("=" * 50)
+    
+    config = CrawlerRunConfig(
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,
+            max_links=30,  # Analyze more links for better statistics
+            concurrency=15,
+            timeout=6,
+            verbose=False
+        ),
+        score_links=True
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        # Test with a content-rich site
+        result = await crawler.arun("https://docs.python.org/3/", config=config)
+        
+        if result.success:
+            print(f"✅ Analyzed: {result.url}")
+            
+            links = result.links.get("internal", [])
+            
+            # Extract intrinsic scores for analysis
+            scores = [link.get('intrinsic_score', 0) for link in links if link.get('intrinsic_score') is not None]
+            
+            if scores:
+                avg_score = sum(scores) / len(scores)
+                high_quality = len([s for s in scores if s >= 7.0])
+                medium_quality = len([s for s in scores if 4.0 <= s < 7.0])
+                low_quality = len([s for s in scores if s < 4.0])
+                
+                print(f"\n📈 Quality Analysis Results:")
+                print(f"   📊 Average Score: {avg_score:.2f}/10.0")
+                print(f"   🟢 High Quality (≥7.0): {high_quality} links")
+                print(f"   🟡 Medium Quality (4.0-6.9): {medium_quality} links")
+                print(f"   🔴 Low Quality (<4.0): {low_quality} links")
+                
+                # Show best and worst links
+                scored_links = [(link, link.get('intrinsic_score', 0)) for link in links 
+                              if link.get('intrinsic_score') is not None]
+                scored_links.sort(key=lambda x: x[1], reverse=True)
+                
+                print(f"\n🏆 Top 3 Quality Links:")
+                for i, (link, score) in enumerate(scored_links[:3]):
+                    text = link.get('text', 'No text')[:40]
+                    print(f"   {i+1}. [{score:.1f}] {text}...")
+                    print(f"      {link['href']}")
+                
+                print(f"\n⚠️  Bottom 3 Quality Links:")
+                for i, (link, score) in enumerate(scored_links[-3:]):
+                    text = link.get('text', 'No text')[:40]
+                    print(f"   {i+1}. [{score:.1f}] {text}...")
+                    print(f"      {link['href']}")
+            else:
+                print("❌ No scoring data available")
+        else:
+            print(f"❌ Analysis failed: {result.error_message}")
+
+
+async def pattern_filtering_example():
+    """
+    Pattern Filtering: Demonstrate advanced filtering capabilities
+    """
+    print("\n\n🎯 Pattern Filtering Example")
+    print("=" * 50)
+    
+    # Example with multiple filtering strategies
+    filters = [
+        {
+            "name": "Documentation Only",
+            "config": LinkPreviewConfig(
+                include_internal=True,
+                max_links=10,
+                concurrency=5,
+                verbose=False,
+                include_patterns=["*/docs/*", "*/documentation/*"],
+                exclude_patterns=["*/api/*"]
+            )
+        },
+        {
+            "name": "API References Only", 
+            "config": LinkPreviewConfig(
+                include_internal=True,
+                max_links=10,
+                concurrency=5,
+                verbose=False,
+                include_patterns=["*/api/*", "*/reference/*"],
+                exclude_patterns=["*/tutorial/*"]
+            )
+        },
+        {
+            "name": "Exclude Admin Areas",
+            "config": LinkPreviewConfig(
+                include_internal=True,
+                max_links=10,
+                concurrency=5,
+                verbose=False,
+                exclude_patterns=["*/admin/*", "*/login/*", "*/dashboard/*"]
+            )
+        }
+    ]
+    
+    async with AsyncWebCrawler() as crawler:
+        for filter_example in filters:
+            print(f"\n🔍 Testing: {filter_example['name']}")
+            
+            config = CrawlerRunConfig(
+                link_preview_config=filter_example['config'],
+                score_links=True
+            )
+            
+            result = await crawler.arun("https://docs.python.org/3/", config=config)
+            
+            if result.success:
+                links = result.links.get("internal", [])
+                links_with_head = [link for link in links if link.get("head_data")]
+                
+                print(f"   📊 Found {len(links_with_head)} matching links")
+                
+                if links_with_head:
+                    # Show sample matches
+                    for link in links_with_head[:2]:
+                        title = link.get("head_data", {}).get("title", "No title")
+                        print(f"   • {title[:50]}...")
+                        print(f"     {link['href']}")
+            else:
+                print(f"   ❌ Failed: {result.error_message}")
+
+
+async def main():
+    """
+    Run all examples
+    """
+    print("🚀 Crawl4AI Link Head Extraction Examples")
+    print("=" * 60)
+    print("This will demonstrate various link analysis capabilities.\n")
+    
+    try:
+        # Run all examples
+        await basic_link_head_extraction()
+        await research_assistant_example()
+        await api_discovery_example() 
+        await link_quality_analysis()
+        await pattern_filtering_example()
+        
+        print("\n" + "=" * 60)
+        print("✨ All examples completed successfully!")
+        print("\nNext steps:")
+        print("1. Try modifying the queries and patterns above")
+        print("2. Test with your own websites") 
+        print("3. Experiment with different score thresholds")
+        print("4. Check out the full documentation for more options")
+        
+    except KeyboardInterrupt:
+        print("\n⏹️  Examples interrupted by user")
+    except Exception as e:
+        print(f"\n💥 Error running examples: {str(e)}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/llm_extraction_openai_pricing.py b/docs/examples/llm_extraction_openai_pricing.py
new file mode 100644
index 0000000..de9c1c4
--- /dev/null
+++ b/docs/examples/llm_extraction_openai_pricing.py
@@ -0,0 +1,55 @@
+import asyncio
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, LLMConfig, BrowserConfig, CacheMode
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+from typing import Dict
+import os
+
+
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(..., description="Fee for output token for the OpenAI model.")
+
+
+async def extract_structured_data_using_llm(provider: str, api_token: str = None, extra_headers: Dict[str, str] = None):
+    print(f"\n--- Extracting Structured Data with {provider} ---")
+
+    if api_token is None and provider != "ollama":
+        print(f"API token is required for {provider}. Skipping this example.")
+        return
+
+    browser_config = BrowserConfig(headless=True)
+
+    extra_args = {"temperature": 0, "top_p": 0.9, "max_tokens": 2000}
+    if extra_headers:
+        extra_args["extra_headers"] = extra_headers
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        word_count_threshold=1,
+        page_timeout=80000,
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider=provider, api_token=api_token),
+            schema=OpenAIModelFee.model_json_schema(),
+            extraction_type="schema",
+            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+            Do not miss any models in the entire content.""",
+            extra_args=extra_args,
+        ),
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://openai.com/api/pricing/", 
+            config=crawler_config
+        )
+        print(result.extracted_content)
+
+
+if __name__ == "__main__":
+    asyncio.run(
+        extract_structured_data_using_llm(
+            provider="openai/gpt-4o", api_token=os.getenv("OPENAI_API_KEY")
+        )
+    )
diff --git a/docs/examples/llm_markdown_generator.py b/docs/examples/llm_markdown_generator.py
new file mode 100644
index 0000000..777c59b
--- /dev/null
+++ b/docs/examples/llm_markdown_generator.py
@@ -0,0 +1,87 @@
+import os
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai import LLMConfig
+from crawl4ai.content_filter_strategy import LLMContentFilter
+
+async def test_llm_filter():
+    # Create an HTML source that needs intelligent filtering
+    url = "https://docs.python.org/3/tutorial/classes.html"
+    
+    browser_config = BrowserConfig(
+        headless=True,
+        verbose=True
+    )
+    
+    # run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED)
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # First get the raw HTML
+        result = await crawler.arun(url, config=run_config)
+        html = result.cleaned_html
+
+        # Initialize LLM filter with focused instruction
+        filter = LLMContentFilter(
+            llm_config=LLMConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')),
+            instruction="""
+            Focus on extracting the core educational content about Python classes.
+            Include:
+            - Key concepts and their explanations
+            - Important code examples
+            - Essential technical details
+            Exclude:
+            - Navigation elements
+            - Sidebars
+            - Footer content
+            - Version information
+            - Any non-essential UI elements
+            
+            Format the output as clean markdown with proper code blocks and headers.
+            """,
+            verbose=True
+        )
+        
+        filter = LLMContentFilter(
+            llm_config=LLMConfig(provider="openai/gpt-4o",api_token=os.getenv('OPENAI_API_KEY')),
+            chunk_token_threshold=2 ** 12 * 2, # 2048 * 2
+            ignore_cache = True,
+            instruction="""
+            Extract the main educational content while preserving its original wording and substance completely. Your task is to:
+
+            1. Maintain the exact language and terminology used in the main content
+            2. Keep all technical explanations, examples, and educational content intact
+            3. Preserve the original flow and structure of the core content
+            4. Remove only clearly irrelevant elements like:
+            - Navigation menus
+            - Advertisement sections
+            - Cookie notices
+            - Footers with site information
+            - Sidebars with external links
+            - Any UI elements that don't contribute to learning
+
+            The goal is to create a clean markdown version that reads exactly like the original article, 
+            keeping all valuable content but free from distracting elements. Imagine you're creating 
+            a perfect reading experience where nothing valuable is lost, but all noise is removed.
+            """,
+            verbose=True
+        )        
+
+        # Apply filtering
+        filtered_content = filter.filter_content(html)
+        
+        # Show results
+        print("\nFiltered Content Length:", len(filtered_content))
+        print("\nFirst 500 chars of filtered content:")
+        if filtered_content:
+            print(filtered_content[0][:500])
+        
+        # Save on disc the markdown version
+        with open("filtered_content.md", "w", encoding="utf-8") as f:
+            f.write("\n".join(filtered_content))
+        
+        # Show token usage
+        filter.show_usage()
+
+if __name__ == "__main__":
+    asyncio.run(test_llm_filter())
\ No newline at end of file
diff --git a/docs/examples/llm_table_extraction_example.py b/docs/examples/llm_table_extraction_example.py
new file mode 100644
index 0000000..b97d2bb
--- /dev/null
+++ b/docs/examples/llm_table_extraction_example.py
@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+"""
+Example demonstrating LLM-based table extraction in Crawl4AI.
+
+This example shows how to use the LLMTableExtraction strategy to extract
+complex tables from web pages, including handling rowspan, colspan, and nested tables.
+"""
+
+import os
+import sys
+
+# Get the grandparent directory
+grandparent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(grandparent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+
+
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    LLMConfig,
+    LLMTableExtraction,
+    CacheMode
+)
+import pandas as pd
+
+
+# Example 1: Basic LLM Table Extraction
+async def basic_llm_extraction():
+    """Extract tables using LLM with default settings."""
+    print("\n=== Example 1: Basic LLM Table Extraction ===")
+    
+    # Configure LLM (using OpenAI GPT-4o-mini for cost efficiency)
+    llm_config = LLMConfig(
+        provider="openai/gpt-4.1-mini",
+        api_token="env:OPENAI_API_KEY",  # Uses environment variable
+        temperature=0.1,  # Low temperature for consistency
+        max_tokens=32000
+    )
+    
+    # Create LLM table extraction strategy
+    table_strategy = LLMTableExtraction(
+        llm_config=llm_config,
+        verbose=True,
+        # css_selector="div.mw-content-ltr",
+        max_tries=2,
+        enable_chunking=True,
+        chunk_token_threshold=5000,  # Lower threshold to force chunking
+        min_rows_per_chunk=10,
+        max_parallel_chunks=3
+    )
+    
+    # Configure crawler with the strategy
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        table_extraction=table_strategy
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        # Extract tables from a Wikipedia page
+        result = await crawler.arun(
+            url="https://en.wikipedia.org/wiki/List_of_chemical_elements",
+            config=config
+        )
+        
+        if result.success:
+            print(f"✓ Found {len(result.tables)} tables")
+            
+            # Display first table
+            if result.tables:
+                first_table = result.tables[0]
+                print(f"\nFirst table:")
+                print(f"  Headers: {first_table['headers'][:5]}...")
+                print(f"  Rows: {len(first_table['rows'])}")
+                
+                # Convert to pandas DataFrame
+                df = pd.DataFrame(
+                    first_table['rows'],
+                    columns=first_table['headers']
+                )
+                print(f"\nDataFrame shape: {df.shape}")
+                print(df.head())
+        else:
+            print(f"✗ Extraction failed: {result.error}")
+
+
+# Example 2: Focused Extraction with CSS Selector
+async def focused_extraction():
+    """Extract tables from specific page sections using CSS selectors."""
+    print("\n=== Example 2: Focused Extraction with CSS Selector ===")
+    
+    # HTML with multiple tables
+    test_html = """
+    
+    
+        

+            
+                
+            
Navigation

+        

+        
+        

+            
+                
+                
+                    
+                        
+                        
+                    
+                    
+                        
+                        
+                        
+                    
+                
+                
+                    
+                        
+                        
+                        
+                        
+                    
+                    
+                        
+                        
+                        
+                        
+                    
+                
+            
Quarterly Sales Report
ProductQ1 2024
JanFebMar
Widget A100120140
Widget B200180220

+        

+    
+    
+    """
+    
+    llm_config = LLMConfig(
+        provider="openai/gpt-4.1-mini",
+        api_token="env:OPENAI_API_KEY"
+    )
+    
+    # Focus only on main content area
+    table_strategy = LLMTableExtraction(
+        llm_config=llm_config,
+        css_selector=".main-content",  # Only extract from main content
+        verbose=True
+    )
+    
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        table_extraction=table_strategy
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url=f"raw:{test_html}",
+            config=config
+        )
+        
+        if result.success and result.tables:
+            table = result.tables[0]
+            print(f"✓ Extracted table: {table.get('caption', 'No caption')}")
+            print(f"  Headers: {table['headers']}")
+            print(f"  Metadata: {table['metadata']}")
+            
+            # The LLM should have handled the rowspan/colspan correctly
+            print("\nProcessed data (rowspan/colspan handled):")
+            for i, row in enumerate(table['rows']):
+                print(f"  Row {i+1}: {row}")
+
+
+# Example 3: Comparing with Default Extraction
+async def compare_strategies():
+    """Compare LLM extraction with default extraction on complex tables."""
+    print("\n=== Example 3: Comparing LLM vs Default Extraction ===")
+    
+    # Complex table with nested structure
+    complex_html = """
+    
+    
+        
+            
+                
+                
+                
+            
+            
+                
+                
+                
+                
+            
+            
+                
+            
+            
+                
+                
+                
+                
+                
+            
+            
+                
+                
+                
+                
+                
+            
+        
Category20232024
H1H2H1H2
All values in millions
Revenue100120130145
Profit20252832

+    
+    
+    """
+    
+    async with AsyncWebCrawler() as crawler:
+        # Test with default extraction
+        from crawl4ai import DefaultTableExtraction
+        
+        default_strategy = DefaultTableExtraction(
+            table_score_threshold=3,
+            verbose=True
+        )
+        
+        config_default = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            table_extraction=default_strategy
+        )
+        
+        result_default = await crawler.arun(
+            url=f"raw:{complex_html}",
+            config=config_default
+        )
+        
+        # Test with LLM extraction
+        llm_strategy = LLMTableExtraction(
+            llm_config=LLMConfig(
+                provider="openai/gpt-4.1-mini",
+                api_token="env:OPENAI_API_KEY"
+            ),
+            verbose=True
+        )
+        
+        config_llm = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            table_extraction=llm_strategy
+        )
+        
+        result_llm = await crawler.arun(
+            url=f"raw:{complex_html}",
+            config=config_llm
+        )
+        
+        # Compare results
+        print("\nDefault Extraction:")
+        if result_default.tables:
+            table = result_default.tables[0]
+            print(f"  Headers: {table.get('headers', [])}")
+            print(f"  Rows: {len(table.get('rows', []))}")
+            for i, row in enumerate(table.get('rows', [])[:3]):
+                print(f"    Row {i+1}: {row}")
+        
+        print("\nLLM Extraction (handles complex structure better):")
+        if result_llm.tables:
+            table = result_llm.tables[0]
+            print(f"  Headers: {table.get('headers', [])}")
+            print(f"  Rows: {len(table.get('rows', []))}")
+            for i, row in enumerate(table.get('rows', [])):
+                print(f"    Row {i+1}: {row}")
+            print(f"  Metadata: {table.get('metadata', {})}")
+
+# Example 4: Batch Processing Multiple Pages
+async def batch_extraction():
+    """Extract tables from multiple pages efficiently."""
+    print("\n=== Example 4: Batch Table Extraction ===")
+    
+    urls = [
+        "https://www.worldometers.info/geography/alphabetical-list-of-countries/",
+        # "https://en.wikipedia.org/wiki/List_of_chemical_elements",
+    ]
+    
+    llm_config = LLMConfig(
+        provider="openai/gpt-4.1-mini",
+        api_token="env:OPENAI_API_KEY",
+        temperature=0.1,
+        max_tokens=1500
+    )
+    
+    table_strategy = LLMTableExtraction(
+        llm_config=llm_config,
+        css_selector="div.datatable-container",  # Wikipedia data tables
+        verbose=False,
+        enable_chunking=True,
+        chunk_token_threshold=5000,  # Lower threshold to force chunking
+        min_rows_per_chunk=10,
+        max_parallel_chunks=3
+    )
+    
+    config = CrawlerRunConfig(
+        table_extraction=table_strategy,
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    all_tables = []
+    
+    async with AsyncWebCrawler() as crawler:
+        for url in urls:
+            print(f"\nProcessing: {url.split('/')[-1][:50]}...")
+            result = await crawler.arun(url=url, config=config)
+            
+            if result.success and result.tables:
+                print(f"  ✓ Found {len(result.tables)} tables")
+                # Store first table from each page
+                if result.tables:
+                    all_tables.append({
+                        'url': url,
+                        'table': result.tables[0]
+                    })
+    
+    # Summary
+    print(f"\n=== Summary ===")
+    print(f"Extracted {len(all_tables)} tables from {len(urls)} pages")
+    for item in all_tables:
+        table = item['table']
+        print(f"\nFrom {item['url'].split('/')[-1][:30]}:")
+        print(f"  Columns: {len(table['headers'])}")
+        print(f"  Rows: {len(table['rows'])}")
+
+
+async def main():
+    """Run all examples."""
+    print("=" * 60)
+    print("LLM TABLE EXTRACTION EXAMPLES")
+    print("=" * 60)
+    
+    # Run examples (comment out ones you don't want to run)
+    
+    # Basic extraction
+    await basic_llm_extraction()
+    
+    # # Focused extraction with CSS
+    # await focused_extraction()
+    
+    # # Compare strategies
+    # await compare_strategies()
+    
+    # # Batch processing
+    # await batch_extraction()
+    
+    print("\n" + "=" * 60)
+    print("ALL EXAMPLES COMPLETED")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/markdown/content_source_example.py b/docs/examples/markdown/content_source_example.py
new file mode 100644
index 0000000..5d83676
--- /dev/null
+++ b/docs/examples/markdown/content_source_example.py
@@ -0,0 +1,64 @@
+"""
+Example showing how to use the content_source parameter to control HTML input for markdown generation.
+"""
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, DefaultMarkdownGenerator
+
+async def demo_content_source():
+    """Demonstrates different content_source options for markdown generation."""
+    url = "https://example.com"  # Simple demo site
+    
+    print("Crawling with different content_source options...")
+    
+    # --- Example 1: Default Behavior (cleaned_html) ---
+    # This uses the HTML after it has been processed by the scraping strategy
+    # The HTML is cleaned, simplified, and optimized for readability
+    default_generator = DefaultMarkdownGenerator()  # content_source="cleaned_html" is default
+    default_config = CrawlerRunConfig(markdown_generator=default_generator)
+    
+    # --- Example 2: Raw HTML ---
+    # This uses the original HTML directly from the webpage
+    # Preserves more original content but may include navigation, ads, etc.
+    raw_generator = DefaultMarkdownGenerator(content_source="raw_html")
+    raw_config = CrawlerRunConfig(markdown_generator=raw_generator)
+    
+    # --- Example 3: Fit HTML ---
+    # This uses preprocessed HTML optimized for schema extraction
+    # Better for structured data extraction but may lose some formatting
+    fit_generator = DefaultMarkdownGenerator(content_source="fit_html")
+    fit_config = CrawlerRunConfig(markdown_generator=fit_generator)
+    
+    # Execute all three crawlers in sequence
+    async with AsyncWebCrawler() as crawler:
+        # Default (cleaned_html)
+        result_default = await crawler.arun(url=url, config=default_config)
+        
+        # Raw HTML
+        result_raw = await crawler.arun(url=url, config=raw_config)
+        
+        # Fit HTML
+        result_fit = await crawler.arun(url=url, config=fit_config)
+    
+    # Print a summary of the results
+    print("\nMarkdown Generation Results:\n")
+    
+    print("1. Default (cleaned_html):")
+    print(f"   Length: {len(result_default.markdown.raw_markdown)} chars")
+    print(f"   First 80 chars: {result_default.markdown.raw_markdown[:80]}...\n")
+    
+    print("2. Raw HTML:")
+    print(f"   Length: {len(result_raw.markdown.raw_markdown)} chars")
+    print(f"   First 80 chars: {result_raw.markdown.raw_markdown[:80]}...\n")
+    
+    print("3. Fit HTML:")
+    print(f"   Length: {len(result_fit.markdown.raw_markdown)} chars")
+    print(f"   First 80 chars: {result_fit.markdown.raw_markdown[:80]}...\n")
+    
+    # Demonstrate differences in output
+    print("\nKey Takeaways:")
+    print("- cleaned_html: Best for readable, focused content")
+    print("- raw_html: Preserves more original content, but may include noise")
+    print("- fit_html: Optimized for schema extraction and structured data")
+
+if __name__ == "__main__":
+    asyncio.run(demo_content_source())
\ No newline at end of file
diff --git a/docs/examples/markdown/content_source_short_example.py b/docs/examples/markdown/content_source_short_example.py
new file mode 100644
index 0000000..83c3ecb
--- /dev/null
+++ b/docs/examples/markdown/content_source_short_example.py
@@ -0,0 +1,42 @@
+"""
+Example demonstrating how to use the content_source parameter in MarkdownGenerationStrategy
+"""
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, DefaultMarkdownGenerator
+
+async def demo_markdown_source_config():
+    print("\n=== Demo: Configuring Markdown Source ===")
+
+    # Example 1: Generate markdown from cleaned HTML (default behavior)
+    cleaned_md_generator = DefaultMarkdownGenerator(content_source="cleaned_html")
+    config_cleaned = CrawlerRunConfig(markdown_generator=cleaned_md_generator)
+
+    async with AsyncWebCrawler() as crawler:
+        result_cleaned = await crawler.arun(url="https://example.com", config=config_cleaned)
+        print("Markdown from Cleaned HTML (default):")
+        print(f"  Length: {len(result_cleaned.markdown.raw_markdown)}")
+        print(f"  Start: {result_cleaned.markdown.raw_markdown[:100]}...")
+
+    # Example 2: Generate markdown directly from raw HTML
+    raw_md_generator = DefaultMarkdownGenerator(content_source="raw_html")
+    config_raw = CrawlerRunConfig(markdown_generator=raw_md_generator)
+
+    async with AsyncWebCrawler() as crawler:
+        result_raw = await crawler.arun(url="https://example.com", config=config_raw)
+        print("\nMarkdown from Raw HTML:")
+        print(f"  Length: {len(result_raw.markdown.raw_markdown)}")
+        print(f"  Start: {result_raw.markdown.raw_markdown[:100]}...")
+
+    # Example 3: Generate markdown from preprocessed 'fit' HTML
+    fit_md_generator = DefaultMarkdownGenerator(content_source="fit_html")
+    config_fit = CrawlerRunConfig(markdown_generator=fit_md_generator)
+
+    async with AsyncWebCrawler() as crawler:
+        result_fit = await crawler.arun(url="https://example.com", config=config_fit)
+        print("\nMarkdown from Fit HTML:")
+        print(f"  Length: {len(result_fit.markdown.raw_markdown)}")
+        print(f"  Start: {result_fit.markdown.raw_markdown[:100]}...")
+
+if __name__ == "__main__":
+    asyncio.run(demo_markdown_source_config())
\ No newline at end of file
diff --git a/docs/examples/network_console_capture_example.py b/docs/examples/network_console_capture_example.py
new file mode 100644
index 0000000..0208bdc
--- /dev/null
+++ b/docs/examples/network_console_capture_example.py
@@ -0,0 +1,477 @@
+import asyncio
+import json
+import os
+import base64
+from pathlib import Path
+from typing import List, Dict, Any
+from datetime import datetime
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, CrawlResult
+from crawl4ai import BrowserConfig
+
+__cur_dir__ = Path(__file__).parent
+
+# Create temp directory if it doesn't exist
+os.makedirs(os.path.join(__cur_dir__, "tmp"), exist_ok=True)
+
+async def demo_basic_network_capture():
+    """Basic network request capturing example"""
+    print("\n=== 1. Basic Network Request Capturing ===")
+    
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            capture_network_requests=True,
+            wait_until="networkidle"  # Wait for network to be idle
+        )
+        
+        result = await crawler.arun(
+            url="https://example.com/",
+            config=config
+        )
+        
+        if result.success and result.network_requests:
+            print(f"Captured {len(result.network_requests)} network events")
+            
+            # Count by event type
+            event_types = {}
+            for req in result.network_requests:
+                event_type = req.get("event_type", "unknown")
+                event_types[event_type] = event_types.get(event_type, 0) + 1
+            
+            print("Event types:")
+            for event_type, count in event_types.items():
+                print(f"  - {event_type}: {count}")
+            
+            # Show a sample request and response
+            request = next((r for r in result.network_requests if r.get("event_type") == "request"), None)
+            response = next((r for r in result.network_requests if r.get("event_type") == "response"), None)
+            
+            if request:
+                print("\nSample request:")
+                print(f"  URL: {request.get('url')}")
+                print(f"  Method: {request.get('method')}")
+                print(f"  Headers: {list(request.get('headers', {}).keys())}")
+            
+            if response:
+                print("\nSample response:")
+                print(f"  URL: {response.get('url')}")
+                print(f"  Status: {response.get('status')} {response.get('status_text', '')}")
+                print(f"  Headers: {list(response.get('headers', {}).keys())}")
+
+async def demo_basic_console_capture():
+    """Basic console message capturing example"""
+    print("\n=== 2. Basic Console Message Capturing ===")
+    
+    # Create a simple HTML file with console messages
+    html_file = os.path.join(__cur_dir__, "tmp", "console_test.html")
+    with open(html_file, "w") as f:
+        f.write("""
+        
+        
+        
+            Console Test
+        
+        
+            
Console Message Test

+            
+        
+        
+        """)
+    
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            capture_console_messages=True,
+            wait_until="networkidle"  # Wait to make sure all scripts execute
+        )
+        
+        result = await crawler.arun(
+            url=f"file://{html_file}",
+            config=config
+        )
+        
+        if result.success and result.console_messages:
+            print(f"Captured {len(result.console_messages)} console messages")
+            
+            # Count by message type
+            message_types = {}
+            for msg in result.console_messages:
+                msg_type = msg.get("type", "unknown")
+                message_types[msg_type] = message_types.get(msg_type, 0) + 1
+            
+            print("Message types:")
+            for msg_type, count in message_types.items():
+                print(f"  - {msg_type}: {count}")
+            
+            # Show all messages
+            print("\nAll console messages:")
+            for i, msg in enumerate(result.console_messages, 1):
+                print(f"  {i}. [{msg.get('type', 'unknown')}] {msg.get('text', '')}")
+
+async def demo_combined_capture():
+    """Capturing both network requests and console messages"""
+    print("\n=== 3. Combined Network and Console Capture ===")
+    
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            capture_network_requests=True,
+            capture_console_messages=True,
+            wait_until="networkidle"
+        )
+        
+        result = await crawler.arun(
+            url="https://httpbin.org/html",
+            config=config
+        )
+        
+        if result.success:
+            network_count = len(result.network_requests) if result.network_requests else 0
+            console_count = len(result.console_messages) if result.console_messages else 0
+            
+            print(f"Captured {network_count} network events and {console_count} console messages")
+            
+            # Save the captured data to a JSON file for analysis
+            output_file = os.path.join(__cur_dir__, "tmp", "capture_data.json")
+            with open(output_file, "w") as f:
+                json.dump({
+                    "url": result.url,
+                    "timestamp": datetime.now().isoformat(),
+                    "network_requests": result.network_requests,
+                    "console_messages": result.console_messages
+                }, f, indent=2)
+            
+            print(f"Full capture data saved to {output_file}")
+
+async def analyze_spa_network_traffic():
+    """Analyze network traffic of a Single-Page Application"""
+    print("\n=== 4. Analyzing SPA Network Traffic ===")
+    
+    async with AsyncWebCrawler(config=BrowserConfig(
+        headless=True,
+        viewport_width=1280,
+        viewport_height=800
+    )) as crawler:
+        config = CrawlerRunConfig(
+            capture_network_requests=True,
+            capture_console_messages=True,
+            # Wait longer to ensure all resources are loaded
+            wait_until="networkidle",
+            page_timeout=60000,  # 60 seconds
+        )
+        
+        result = await crawler.arun(
+            url="https://weather.com",
+            config=config
+        )
+        
+        if result.success and result.network_requests:
+            # Extract different types of requests
+            requests = []
+            responses = []
+            failures = []
+            
+            for event in result.network_requests:
+                event_type = event.get("event_type")
+                if event_type == "request":
+                    requests.append(event)
+                elif event_type == "response":
+                    responses.append(event)
+                elif event_type == "request_failed":
+                    failures.append(event)
+            
+            print(f"Captured {len(requests)} requests, {len(responses)} responses, and {len(failures)} failures")
+            
+            # Analyze request types
+            resource_types = {}
+            for req in requests:
+                resource_type = req.get("resource_type", "unknown")
+                resource_types[resource_type] = resource_types.get(resource_type, 0) + 1
+            
+            print("\nResource types:")
+            for resource_type, count in sorted(resource_types.items(), key=lambda x: x[1], reverse=True):
+                print(f"  - {resource_type}: {count}")
+            
+            # Analyze API calls
+            api_calls = [r for r in requests if "api" in r.get("url", "").lower()]
+            if api_calls:
+                print(f"\nDetected {len(api_calls)} API calls:")
+                for i, call in enumerate(api_calls[:5], 1):  # Show first 5
+                    print(f"  {i}. {call.get('method')} {call.get('url')}")
+                if len(api_calls) > 5:
+                    print(f"     ... and {len(api_calls) - 5} more")
+            
+            # Analyze response status codes
+            status_codes = {}
+            for resp in responses:
+                status = resp.get("status", 0)
+                status_codes[status] = status_codes.get(status, 0) + 1
+            
+            print("\nResponse status codes:")
+            for status, count in sorted(status_codes.items()):
+                print(f"  - {status}: {count}")
+            
+            # Analyze failures
+            if failures:
+                print("\nFailed requests:")
+                for i, failure in enumerate(failures[:5], 1):  # Show first 5
+                    print(f"  {i}. {failure.get('url')} - {failure.get('failure_text')}")
+                if len(failures) > 5:
+                    print(f"     ... and {len(failures) - 5} more")
+            
+            # Check for console errors
+            if result.console_messages:
+                errors = [msg for msg in result.console_messages if msg.get("type") == "error"]
+                if errors:
+                    print(f"\nDetected {len(errors)} console errors:")
+                    for i, error in enumerate(errors[:3], 1):  # Show first 3
+                        print(f"  {i}. {error.get('text', '')[:100]}...")
+                    if len(errors) > 3:
+                        print(f"     ... and {len(errors) - 3} more")
+            
+            # Save analysis to file
+            output_file = os.path.join(__cur_dir__, "tmp", "weather_network_analysis.json")
+            with open(output_file, "w") as f:
+                json.dump({
+                    "url": result.url,
+                    "timestamp": datetime.now().isoformat(),
+                    "statistics": {
+                        "request_count": len(requests),
+                        "response_count": len(responses),
+                        "failure_count": len(failures),
+                        "resource_types": resource_types,
+                        "status_codes": {str(k): v for k, v in status_codes.items()},
+                        "api_call_count": len(api_calls),
+                        "console_error_count": len(errors) if result.console_messages else 0
+                    },
+                    "network_requests": result.network_requests,
+                    "console_messages": result.console_messages
+                }, f, indent=2)
+            
+            print(f"\nFull analysis saved to {output_file}")
+
+async def demo_security_analysis():
+    """Using network capture for security analysis"""
+    print("\n=== 5. Security Analysis with Network Capture ===")
+    
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            capture_network_requests=True,
+            capture_console_messages=True,
+            wait_until="networkidle"
+        )
+        
+        # A site that makes multiple third-party requests
+        result = await crawler.arun(
+            url="https://www.nytimes.com/",
+            config=config
+        )
+        
+        if result.success and result.network_requests:
+            print(f"Captured {len(result.network_requests)} network events")
+            
+            # Extract all domains
+            domains = set()
+            for req in result.network_requests:
+                if req.get("event_type") == "request":
+                    url = req.get("url", "")
+                    try:
+                        from urllib.parse import urlparse
+                        domain = urlparse(url).netloc
+                        if domain:
+                            domains.add(domain)
+                    except:
+                        pass
+            
+            print(f"\nDetected requests to {len(domains)} unique domains:")
+            main_domain = urlparse(result.url).netloc
+            
+            # Separate first-party vs third-party domains
+            first_party = [d for d in domains if main_domain in d]
+            third_party = [d for d in domains if main_domain not in d]
+            
+            print(f"  - First-party domains: {len(first_party)}")
+            print(f"  - Third-party domains: {len(third_party)}")
+            
+            # Look for potential trackers/analytics
+            tracking_keywords = ["analytics", "tracker", "pixel", "tag", "stats", "metric", "collect", "beacon"]
+            potential_trackers = []
+            
+            for domain in third_party:
+                if any(keyword in domain.lower() for keyword in tracking_keywords):
+                    potential_trackers.append(domain)
+            
+            if potential_trackers:
+                print(f"\nPotential tracking/analytics domains ({len(potential_trackers)}):")
+                for i, domain in enumerate(sorted(potential_trackers)[:10], 1):
+                    print(f"  {i}. {domain}")
+                if len(potential_trackers) > 10:
+                    print(f"     ... and {len(potential_trackers) - 10} more")
+            
+            # Check for insecure (HTTP) requests
+            insecure_requests = [
+                req.get("url") for req in result.network_requests 
+                if req.get("event_type") == "request" and req.get("url", "").startswith("http://")
+            ]
+            
+            if insecure_requests:
+                print(f"\nWarning: Found {len(insecure_requests)} insecure (HTTP) requests:")
+                for i, url in enumerate(insecure_requests[:5], 1):
+                    print(f"  {i}. {url}")
+                if len(insecure_requests) > 5:
+                    print(f"     ... and {len(insecure_requests) - 5} more")
+            
+            # Save security analysis to file
+            output_file = os.path.join(__cur_dir__, "tmp", "security_analysis.json")
+            with open(output_file, "w") as f:
+                json.dump({
+                    "url": result.url,
+                    "main_domain": main_domain,
+                    "timestamp": datetime.now().isoformat(),
+                    "analysis": {
+                        "total_requests": len([r for r in result.network_requests if r.get("event_type") == "request"]),
+                        "unique_domains": len(domains),
+                        "first_party_domains": first_party,
+                        "third_party_domains": third_party,
+                        "potential_trackers": potential_trackers,
+                        "insecure_requests": insecure_requests
+                    }
+                }, f, indent=2)
+            
+            print(f"\nFull security analysis saved to {output_file}")
+
+async def demo_performance_analysis():
+    """Using network capture for performance analysis"""
+    print("\n=== 6. Performance Analysis with Network Capture ===")
+    
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            capture_network_requests=True,
+            page_timeout=60 * 2 * 1000  # 120 seconds
+        )
+        
+        result = await crawler.arun(
+            url="https://www.cnn.com/",
+            config=config
+        )
+        
+        if result.success and result.network_requests:
+            # Filter only response events with timing information
+            responses_with_timing = [
+                r for r in result.network_requests 
+                if r.get("event_type") == "response" and r.get("request_timing")
+            ]
+            
+            if responses_with_timing:
+                print(f"Analyzing timing for {len(responses_with_timing)} network responses")
+                
+                # Group by resource type
+                resource_timings = {}
+                for resp in responses_with_timing:
+                    url = resp.get("url", "")
+                    timing = resp.get("request_timing", {})
+                    
+                    # Determine resource type from URL extension
+                    ext = url.split(".")[-1].lower() if "." in url.split("/")[-1] else "unknown"
+                    if ext in ["jpg", "jpeg", "png", "gif", "webp", "svg", "ico"]:
+                        resource_type = "image"
+                    elif ext in ["js"]:
+                        resource_type = "javascript"
+                    elif ext in ["css"]:
+                        resource_type = "css"
+                    elif ext in ["woff", "woff2", "ttf", "otf", "eot"]:
+                        resource_type = "font"
+                    else:
+                        resource_type = "other"
+                    
+                    if resource_type not in resource_timings:
+                        resource_timings[resource_type] = []
+                    
+                    # Calculate request duration if timing information is available
+                    if isinstance(timing, dict) and "requestTime" in timing and "receiveHeadersEnd" in timing:
+                        # Convert to milliseconds
+                        duration = (timing["receiveHeadersEnd"] - timing["requestTime"]) * 1000
+                        resource_timings[resource_type].append({
+                            "url": url,
+                            "duration_ms": duration
+                        })
+                    if isinstance(timing, dict) and "requestStart" in timing and "responseStart" in timing and "startTime" in timing:
+                        # Convert to milliseconds
+                        duration = (timing["responseStart"] - timing["requestStart"]) * 1000
+                        resource_timings[resource_type].append({
+                            "url": url,
+                            "duration_ms": duration
+                        })
+                
+                # Calculate statistics for each resource type
+                print("\nPerformance by resource type:")
+                for resource_type, timings in resource_timings.items():
+                    if timings:
+                        durations = [t["duration_ms"] for t in timings]
+                        avg_duration = sum(durations) / len(durations)
+                        max_duration = max(durations)
+                        slowest_resource = next(t["url"] for t in timings if t["duration_ms"] == max_duration)
+                        
+                        print(f"  {resource_type.upper()}:")
+                        print(f"    - Count: {len(timings)}")
+                        print(f"    - Avg time: {avg_duration:.2f} ms")
+                        print(f"    - Max time: {max_duration:.2f} ms")
+                        print(f"    - Slowest: {slowest_resource}")
+                
+                # Identify the slowest resources overall
+                all_timings = []
+                for resource_type, timings in resource_timings.items():
+                    for timing in timings:
+                        timing["type"] = resource_type
+                        all_timings.append(timing)
+                
+                all_timings.sort(key=lambda x: x["duration_ms"], reverse=True)
+                
+                print("\nTop 5 slowest resources:")
+                for i, timing in enumerate(all_timings[:5], 1):
+                    print(f"  {i}. [{timing['type']}] {timing['url']} - {timing['duration_ms']:.2f} ms")
+                
+                # Save performance analysis to file
+                output_file = os.path.join(__cur_dir__, "tmp", "performance_analysis.json")
+                with open(output_file, "w") as f:
+                    json.dump({
+                        "url": result.url,
+                        "timestamp": datetime.now().isoformat(),
+                        "resource_timings": resource_timings,
+                        "slowest_resources": all_timings[:10]  # Save top 10
+                    }, f, indent=2)
+                
+                print(f"\nFull performance analysis saved to {output_file}")
+
+async def main():
+    """Run all demo functions sequentially"""
+    print("=== Network and Console Capture Examples ===")
+    
+    # Make sure tmp directory exists
+    os.makedirs(os.path.join(__cur_dir__, "tmp"), exist_ok=True)
+    
+    # Run basic examples
+    # await demo_basic_network_capture()
+    await demo_basic_console_capture()
+    # await demo_combined_capture()
+    
+    # Run advanced examples
+    # await analyze_spa_network_traffic()
+    # await demo_security_analysis()
+    # await demo_performance_analysis()
+    
+    print("\n=== Examples Complete ===")
+    print(f"Check the tmp directory for output files: {os.path.join(__cur_dir__, 'tmp')}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/nst_proxy/api_proxy_example.py b/docs/examples/nst_proxy/api_proxy_example.py
new file mode 100644
index 0000000..1184769
--- /dev/null
+++ b/docs/examples/nst_proxy/api_proxy_example.py
@@ -0,0 +1,48 @@
+"""
+NSTProxy Integration Examples for crawl4ai
+------------------------------------------
+
+NSTProxy is a premium residential proxy provider.
+👉 Purchase Proxies: https://nstproxy.com
+💰 Use coupon code "crawl4ai" for 10% off your plan.
+
+"""
+import asyncio, requests
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+
+async def main():
+    """
+    Example: Dynamically fetch a proxy from NSTProxy API before crawling.
+    """
+    NST_TOKEN = "YOUR_NST_PROXY_TOKEN"  # Get from https://app.nstproxy.com/profile
+    CHANNEL_ID = "YOUR_NST_PROXY_CHANNEL_ID"  # Your NSTProxy Channel ID
+    country = "ANY"  # e.g. "ANY", "US", "DE"
+
+    # Fetch proxy from NSTProxy API
+    api_url = (
+        f"https://api.nstproxy.com/api/v1/generate/apiproxies"
+        f"?fType=2&channelId={CHANNEL_ID}&country={country}"
+        f"&protocol=http&sessionDuration=10&count=1&token={NST_TOKEN}"
+    )
+    response = requests.get(api_url, timeout=10).json()
+    proxy = response[0]
+
+    ip = proxy.get("ip")
+    port = proxy.get("port")
+    username = proxy.get("username", "")
+    password = proxy.get("password", "")
+
+    browser_config = BrowserConfig(proxy_config={
+        "server": f"http://{ip}:{port}",
+        "username": username,
+        "password": password,
+    })
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[API Proxy] Status:", result.status_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/nst_proxy/auth_proxy_example.py b/docs/examples/nst_proxy/auth_proxy_example.py
new file mode 100644
index 0000000..6fb838b
--- /dev/null
+++ b/docs/examples/nst_proxy/auth_proxy_example.py
@@ -0,0 +1,31 @@
+"""
+NSTProxy Integration Examples for crawl4ai
+------------------------------------------
+
+NSTProxy is a premium residential proxy provider.
+👉 Purchase Proxies: https://nstproxy.com
+💰 Use coupon code "crawl4ai" for 10% off your plan.
+
+"""
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+
+async def main():
+    """
+    Example: Use NSTProxy with manual username/password authentication.
+    """
+
+    browser_config = BrowserConfig(proxy_config={
+        "server": "http://gate.nstproxy.io:24125",
+        "username": "your_username",
+        "password": "your_password",
+    })
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[Auth Proxy] Status:", result.status_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/nst_proxy/basic_proxy_example.py b/docs/examples/nst_proxy/basic_proxy_example.py
new file mode 100644
index 0000000..5a79525
--- /dev/null
+++ b/docs/examples/nst_proxy/basic_proxy_example.py
@@ -0,0 +1,29 @@
+"""
+NSTProxy Integration Examples for crawl4ai
+------------------------------------------
+
+NSTProxy is a premium residential proxy provider.
+👉 Purchase Proxies: https://nstproxy.com
+💰 Use coupon code "crawl4ai" for 10% off your plan.
+
+"""
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+
+async def main():
+    # Using HTTP proxy
+    browser_config = BrowserConfig(proxy_config={"server": "http://gate.nstproxy.io:24125"})
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[HTTP Proxy] Status:", result.status_code)
+
+    # Using SOCKS proxy
+    browser_config = BrowserConfig(proxy_config={"server": "socks5://gate.nstproxy.io:24125"})
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[SOCKS5 Proxy] Status:", result.status_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/nst_proxy/nstproxy_example.py b/docs/examples/nst_proxy/nstproxy_example.py
new file mode 100644
index 0000000..4e8587b
--- /dev/null
+++ b/docs/examples/nst_proxy/nstproxy_example.py
@@ -0,0 +1,39 @@
+"""
+NSTProxy Integration Examples for crawl4ai
+------------------------------------------
+
+NSTProxy is a premium residential proxy provider.
+👉 Purchase Proxies: https://nstproxy.com
+💰 Use coupon code "crawl4ai" for 10% off your plan.
+
+"""
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+
+async def main():
+    """
+    Example: Using NSTProxy with AsyncWebCrawler.
+    """
+
+    NST_TOKEN = "YOUR_NST_PROXY_TOKEN"  # Get from https://app.nstproxy.com/profile
+    CHANNEL_ID = "YOUR_NST_PROXY_CHANNEL_ID"  # Your NSTProxy Channel ID
+
+    browser_config = BrowserConfig()
+    browser_config.set_nstproxy(
+        token=NST_TOKEN,
+        channel_id=CHANNEL_ID,
+        country="ANY",  # e.g. "US", "JP", or "ANY"
+        state="",  # optional, leave empty if not needed
+        city="",  # optional, leave empty if not needed
+        session_duration=0  # Session duration in minutes,0 = rotate on every request
+    )
+
+    # === Run crawler ===
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[Nstproxy] Status:", result.status_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/prefetch_two_phase_crawl.py b/docs/examples/prefetch_two_phase_crawl.py
new file mode 100644
index 0000000..61ca1d9
--- /dev/null
+++ b/docs/examples/prefetch_two_phase_crawl.py
@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+Prefetch Mode and Two-Phase Crawling Example
+
+Prefetch mode is a fast path that skips heavy processing and returns
+only HTML + links. This is ideal for:
+
+- Site mapping: Quickly discover all URLs
+- Selective crawling: Find URLs first, then process only what you need
+- Link validation: Check which pages exist without full processing
+- Crawl planning: Estimate size before committing resources
+
+Key concept:
+- `prefetch=True` in CrawlerRunConfig enables fast link-only extraction
+- Skips: markdown generation, content scraping, media extraction, LLM extraction
+- Returns: HTML and links dictionary
+
+Performance benefit: ~5-10x faster than full processing
+"""
+
+import asyncio
+import time
+from typing import List, Dict
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+
+async def example_basic_prefetch():
+    """
+    Example 1: Basic prefetch mode.
+
+    Shows how prefetch returns HTML and links without heavy processing.
+    """
+    print("\n" + "=" * 60)
+    print("Example 1: Basic Prefetch Mode")
+    print("=" * 60)
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # Enable prefetch mode
+        config = CrawlerRunConfig(prefetch=True)
+
+        print("\nFetching with prefetch=True...")
+        result = await crawler.arun("https://books.toscrape.com", config=config)
+
+        print(f"\nResult summary:")
+        print(f"  Success: {result.success}")
+        print(f"  HTML length: {len(result.html) if result.html else 0} chars")
+        print(f"  Internal links: {len(result.links.get('internal', []))}")
+        print(f"  External links: {len(result.links.get('external', []))}")
+
+        # These should be None/empty in prefetch mode
+        print(f"\n  Skipped processing:")
+        print(f"    Markdown: {result.markdown}")
+        print(f"    Cleaned HTML: {result.cleaned_html}")
+        print(f"    Extracted content: {result.extracted_content}")
+
+        # Show some discovered links
+        internal_links = result.links.get("internal", [])
+        if internal_links:
+            print(f"\n  Sample internal links:")
+            for link in internal_links[:5]:
+                print(f"    - {link['href'][:60]}...")
+
+
+async def example_performance_comparison():
+    """
+    Example 2: Compare prefetch vs full processing performance.
+    """
+    print("\n" + "=" * 60)
+    print("Example 2: Performance Comparison")
+    print("=" * 60)
+
+    url = "https://books.toscrape.com"
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # Warm up - first request is slower due to browser startup
+        await crawler.arun(url, config=CrawlerRunConfig())
+
+        # Prefetch mode timing
+        start = time.time()
+        prefetch_result = await crawler.arun(url, config=CrawlerRunConfig(prefetch=True))
+        prefetch_time = time.time() - start
+
+        # Full processing timing
+        start = time.time()
+        full_result = await crawler.arun(url, config=CrawlerRunConfig())
+        full_time = time.time() - start
+
+        print(f"\nTiming comparison:")
+        print(f"  Prefetch mode: {prefetch_time:.3f}s")
+        print(f"  Full processing: {full_time:.3f}s")
+        print(f"  Speedup: {full_time / prefetch_time:.1f}x faster")
+
+        print(f"\nOutput comparison:")
+        print(f"  Prefetch - Links found: {len(prefetch_result.links.get('internal', []))}")
+        print(f"  Full - Links found: {len(full_result.links.get('internal', []))}")
+        print(f"  Full - Markdown length: {len(full_result.markdown.raw_markdown) if full_result.markdown else 0}")
+
+
+async def example_two_phase_crawl():
+    """
+    Example 3: Two-phase crawling pattern.
+
+    Phase 1: Fast discovery with prefetch
+    Phase 2: Full processing on selected URLs
+    """
+    print("\n" + "=" * 60)
+    print("Example 3: Two-Phase Crawling")
+    print("=" * 60)
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # ═══════════════════════════════════════════════════════════
+        # Phase 1: Fast URL discovery
+        # ═══════════════════════════════════════════════════════════
+        print("\n--- Phase 1: Fast Discovery ---")
+
+        prefetch_config = CrawlerRunConfig(prefetch=True)
+        start = time.time()
+        discovery = await crawler.arun("https://books.toscrape.com", config=prefetch_config)
+        discovery_time = time.time() - start
+
+        all_urls = [link["href"] for link in discovery.links.get("internal", [])]
+        print(f"  Discovered {len(all_urls)} URLs in {discovery_time:.2f}s")
+
+        # Filter to URLs we care about (e.g., book detail pages)
+        # On books.toscrape.com, book pages contain "catalogue/" but not "category/"
+        book_urls = [
+            url for url in all_urls
+            if "catalogue/" in url and "category/" not in url
+        ][:5]  # Limit to 5 for demo
+
+        print(f"  Filtered to {len(book_urls)} book pages")
+
+        # ═══════════════════════════════════════════════════════════
+        # Phase 2: Full processing on selected URLs
+        # ═══════════════════════════════════════════════════════════
+        print("\n--- Phase 2: Full Processing ---")
+
+        full_config = CrawlerRunConfig(
+            word_count_threshold=10,
+            remove_overlay_elements=True,
+        )
+
+        results = []
+        start = time.time()
+
+        for url in book_urls:
+            result = await crawler.arun(url, config=full_config)
+            if result.success:
+                results.append(result)
+                title = result.url.split("/")[-2].replace("-", " ").title()[:40]
+                md_len = len(result.markdown.raw_markdown) if result.markdown else 0
+                print(f"    Processed: {title}... ({md_len} chars)")
+
+        processing_time = time.time() - start
+        print(f"\n  Processed {len(results)} pages in {processing_time:.2f}s")
+
+        # ═══════════════════════════════════════════════════════════
+        # Summary
+        # ═══════════════════════════════════════════════════════════
+        print(f"\n--- Summary ---")
+        print(f"  Discovery phase: {discovery_time:.2f}s ({len(all_urls)} URLs)")
+        print(f"  Processing phase: {processing_time:.2f}s ({len(results)} pages)")
+        print(f"  Total time: {discovery_time + processing_time:.2f}s")
+        print(f"  URLs skipped: {len(all_urls) - len(book_urls)} (not matching filter)")
+
+
+async def example_prefetch_with_deep_crawl():
+    """
+    Example 4: Combine prefetch with deep crawl strategy.
+
+    Use prefetch mode during deep crawl for maximum speed.
+    """
+    print("\n" + "=" * 60)
+    print("Example 4: Prefetch with Deep Crawl")
+    print("=" * 60)
+
+    from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # Deep crawl with prefetch - maximum discovery speed
+        config = CrawlerRunConfig(
+            prefetch=True,  # Fast mode
+            deep_crawl_strategy=BFSDeepCrawlStrategy(
+                max_depth=1,
+                max_pages=10,
+            )
+        )
+
+        print("\nDeep crawling with prefetch mode...")
+        start = time.time()
+
+        result_container = await crawler.arun("https://books.toscrape.com", config=config)
+
+        # Handle iterator result from deep crawl
+        if hasattr(result_container, '__iter__'):
+            results = list(result_container)
+        else:
+            results = [result_container]
+
+        elapsed = time.time() - start
+
+        # Collect all discovered links
+        all_internal_links = set()
+        all_external_links = set()
+
+        for result in results:
+            for link in result.links.get("internal", []):
+                all_internal_links.add(link["href"])
+            for link in result.links.get("external", []):
+                all_external_links.add(link["href"])
+
+        print(f"\nResults:")
+        print(f"  Pages crawled: {len(results)}")
+        print(f"  Total internal links discovered: {len(all_internal_links)}")
+        print(f"  Total external links discovered: {len(all_external_links)}")
+        print(f"  Time: {elapsed:.2f}s")
+
+
+async def example_prefetch_with_raw_html():
+    """
+    Example 5: Prefetch with raw HTML input.
+
+    You can also use prefetch mode with raw: URLs for cached content.
+    """
+    print("\n" + "=" * 60)
+    print("Example 5: Prefetch with Raw HTML")
+    print("=" * 60)
+
+    sample_html = """
+    
+        Sample Page
+        
+            
Hello World

+            
+            

+                
This is the main content with another link.

+            

+        
+    
+    """
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        config = CrawlerRunConfig(
+            prefetch=True,
+            base_url="https://mysite.com",  # For resolving relative links
+        )
+
+        result = await crawler.arun(f"raw:{sample_html}", config=config)
+
+        print(f"\nExtracted from raw HTML:")
+        print(f"  Internal links: {len(result.links.get('internal', []))}")
+        for link in result.links.get("internal", []):
+            print(f"    - {link['href']} ({link['text']})")
+
+        print(f"\n  External links: {len(result.links.get('external', []))}")
+        for link in result.links.get("external", []):
+            print(f"    - {link['href']} ({link['text']})")
+
+
+async def main():
+    """Run all examples."""
+    print("=" * 60)
+    print("Prefetch Mode and Two-Phase Crawling Examples")
+    print("=" * 60)
+
+    await example_basic_prefetch()
+    await example_performance_comparison()
+    await example_two_phase_crawl()
+    await example_prefetch_with_deep_crawl()
+    await example_prefetch_with_raw_html()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/proxy_rotation_demo.py b/docs/examples/proxy_rotation_demo.py
new file mode 100644
index 0000000..7efa974
--- /dev/null
+++ b/docs/examples/proxy_rotation_demo.py
@@ -0,0 +1,161 @@
+import os
+import re
+from typing import List, Dict
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    RoundRobinProxyStrategy
+)
+
+def load_proxies_from_env() -> List[Dict]:
+    """Load proxies from PROXIES environment variable"""
+    proxies = []
+    try:
+        proxy_list = os.getenv("PROXIES", "").split(",")
+        for proxy in proxy_list:
+            if not proxy:
+                continue
+            ip, port, username, password = proxy.split(":")
+            proxies.append({
+                "server": f"http://{ip}:{port}",
+                "username": username,
+                "password": password,
+                "ip": ip  # Store original IP for verification
+            })
+    except Exception as e:
+        print(f"Error loading proxies from environment: {e}")
+    return proxies
+
+async def demo_proxy_rotation():
+    """
+    Proxy Rotation Demo using RoundRobinProxyStrategy
+    ===============================================
+    Demonstrates proxy rotation using the strategy pattern.
+    """
+    print("\n=== Proxy Rotation Demo (Round Robin) ===")
+    
+    # Load proxies and create rotation strategy
+    proxies = load_proxies_from_env()
+    if not proxies:
+        print("No proxies found in environment. Set PROXIES env variable!")
+        return
+        
+    proxy_strategy = RoundRobinProxyStrategy(proxies)
+    
+    # Create configs
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        proxy_rotation_strategy=proxy_strategy
+    )
+    
+    # Test URLs
+    urls = ["https://httpbin.org/ip"] * len(proxies)  # Test each proxy once
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        for url in urls:
+            result = await crawler.arun(url=url, config=run_config)
+            
+            if result.success:
+                # Extract IP from response
+                ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
+                current_proxy = run_config.proxy_config if run_config.proxy_config else None
+                
+                if current_proxy:
+                    print(f"Proxy {current_proxy['server']} -> Response IP: {ip_match.group(0) if ip_match else 'Not found'}")
+                    verified = ip_match and ip_match.group(0) == current_proxy['ip']
+                    if verified:
+                        print(f"✅ Proxy working! IP matches: {current_proxy['ip']}")
+                    else:
+                        print("❌ Proxy failed or IP mismatch!")
+            else:
+                print(f"Request failed: {result.error_message}")
+
+async def demo_proxy_rotation_batch():
+    """
+    Proxy Rotation Demo with Batch Processing
+    =======================================
+    Demonstrates proxy rotation using arun_many with memory dispatcher.
+    """
+    print("\n=== Proxy Rotation Batch Demo ===")
+    
+    try:
+        # Load proxies and create rotation strategy
+        proxies = load_proxies_from_env()
+        if not proxies:
+            print("No proxies found in environment. Set PROXIES env variable!")
+            return
+            
+        proxy_strategy = RoundRobinProxyStrategy(proxies)
+        
+        # Configurations
+        browser_config = BrowserConfig(headless=True, verbose=False)
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            proxy_rotation_strategy=proxy_strategy,
+            markdown_generator=DefaultMarkdownGenerator()
+        )
+
+        # Test URLs - multiple requests to test rotation
+        urls = ["https://httpbin.org/ip"] * (len(proxies) * 2)  # Test each proxy twice
+
+        print("\n📈 Initializing crawler with proxy rotation...")
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            monitor = CrawlerMonitor(
+                max_visible_rows=10,
+                display_mode=DisplayMode.DETAILED
+            )
+            
+            dispatcher = MemoryAdaptiveDispatcher(
+                memory_threshold_percent=80.0,
+                check_interval=0.5,
+                max_session_permit=1, #len(proxies),  # Match concurrent sessions to proxy count
+                # monitor=monitor
+            )
+            
+            print("\n🚀 Starting batch crawl with proxy rotation...")
+            results = await crawler.arun_many(
+                urls=urls,
+                config=run_config,
+                dispatcher=dispatcher
+            )
+
+            # Verify results
+            success_count = 0
+            for result in results:
+                if result.success:
+                    ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
+                    current_proxy = run_config.proxy_config if run_config.proxy_config else None
+                    
+                    if current_proxy and ip_match:
+                        print(f"URL {result.url}")
+                        print(f"Proxy {current_proxy['server']} -> Response IP: {ip_match.group(0)}")
+                        verified = ip_match.group(0) == current_proxy['ip']
+                        if verified:
+                            print(f"✅ Proxy working! IP matches: {current_proxy['ip']}")
+                            success_count += 1
+                        else:
+                            print("❌ Proxy failed or IP mismatch!")
+                    print("---")
+                    
+            print(f"\n✅ Completed {len(results)} requests with {success_count} successful proxy verifications")
+            
+    except Exception as e:
+        print(f"\n❌ Error in proxy rotation batch demo: {str(e)}")
+
+if __name__ == "__main__":
+    import asyncio
+    from crawl4ai import (
+        CrawlerMonitor, 
+        DisplayMode,
+        MemoryAdaptiveDispatcher,
+        DefaultMarkdownGenerator
+    )
+    
+    async def run_demos():
+        # await demo_proxy_rotation()  # Original single-request demo
+        await demo_proxy_rotation_batch()  # New batch processing demo
+        
+    asyncio.run(run_demos())
diff --git a/docs/examples/quickstart.ipynb b/docs/examples/quickstart.ipynb
new file mode 100644
index 0000000..cc5be00
--- /dev/null
+++ b/docs/examples/quickstart.ipynb
@@ -0,0 +1,664 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0cba38e5",
+   "metadata": {},
+   "source": [
+    "# Crawl4AI 🕷️🤖\n",
+    "\"unclecode%2Fcrawl4ai\n",
+    "\n",
+    "[![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)\n",
+    "![PyPI - Downloads](https://img.shields.io/pypi/dm/Crawl4AI)\n",
+    "[![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members)\n",
+    "[![GitHub Issues](https://img.shields.io/github/issues/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/issues)\n",
+    "[![GitHub Pull Requests](https://img.shields.io/github/issues-pr/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/pulls)\n",
+    "[![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)\n",
+    "\n",
+    "Crawl4AI simplifies asynchronous web crawling and data extraction, making it accessible for large language models (LLMs) and AI applications. 🆓🌐\n",
+    "\n",
+    "- GitHub Repository: [https://github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)\n",
+    "- Twitter: [@unclecode](https://twitter.com/unclecode)\n",
+    "- Website: [https://crawl4ai.com](https://crawl4ai.com)\n",
+    "\n",
+    "## 🌟 Meet the Crawl4AI Assistant: Your Copilot for Crawling\n",
+    "Use the [Crawl4AI GPT Assistant](https://tinyurl.com/crawl4ai-gpt) as your AI-powered copilot! With this assistant, you can:\n",
+    "- 🧑‍💻 Generate code for complex crawling and extraction tasks\n",
+    "- 💡 Get tailored support and examples\n",
+    "- 📘 Learn Crawl4AI faster with step-by-step guidance"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "41de6458",
+   "metadata": {},
+   "source": [
+    "### **Quickstart with Crawl4AI**"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1380e951",
+   "metadata": {},
+   "source": [
+    "#### 1. **Installation**\n",
+    "Install Crawl4AI and necessary dependencies:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05fecfad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# %%capture\n",
+    "!pip install crawl4ai\n",
+    "!pip install nest_asyncio\n",
+    "!playwright install  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "2c2a74c8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import asyncio\n",
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f3c558d7",
+   "metadata": {},
+   "source": [
+    "#### 2. **Basic Setup and Simple Crawl**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "003376f3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business, success: True, time taken: 1.49 seconds\n",
+      "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.10 seconds\n",
+      "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business, Strategy: AsyncWebCrawler\n",
+      "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.10 seconds.\n",
+      "IE 11 is not supported. For an optimal experience visit our site on another browser.\n",
+      "\n",
+      "[Morning Rundown: Trump and Harris' vastly different closing pitches, why Kim Jong Un is helping Russia, and an ancient city is discovered by accident](https://www.nbcnews.com/news/harris-speech-ellipse-ancient-mayan-city-morning-rundown-rcna177973)[](https://www.nbcnews.com/news/harris-speech-ellipse-ancient-mayan-city-morning-rundown-rcna177973)\n",
+      "\n",
+      "Skip to Content\n",
+      "\n",
+      "[NBC News Logo](https://www.nbcnews.com)\n",
+      "\n",
+      "Spon\n"
+     ]
+    }
+   ],
+   "source": [
+    "import asyncio\n",
+    "from crawl4ai import AsyncWebCrawler\n",
+    "\n",
+    "async def simple_crawl():\n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://www.nbcnews.com/business\",\n",
+    "            bypass_cache=True # By default this is False, meaning the cache will be used\n",
+    "        )\n",
+    "        print(result.markdown.raw_markdown[:500])  # Print the first 500 characters\n",
+    "        \n",
+    "asyncio.run(simple_crawl())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "da9b4d50",
+   "metadata": {},
+   "source": [
+    "#### 3. **Dynamic Content Handling**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5bb8c1e4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+      "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+      "[LOG] 🕸️ Crawling https://www.nbcnews.com/business using AsyncPlaywrightCrawlerStrategy...\n",
+      "[LOG] ✅ Crawled https://www.nbcnews.com/business successfully!\n",
+      "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business, success: True, time taken: 4.52 seconds\n",
+      "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.15 seconds\n",
+      "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business, Strategy: AsyncWebCrawler\n",
+      "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.15 seconds.\n",
+      "IE 11 is not supported. For an optimal experience visit our site on another browser.\n",
+      "\n",
+      "[Morning Rundown: Trump and Harris' vastly different closing pitches, why Kim Jong Un is helping Russia, and an ancient city is discovered by accident](https://www.nbcnews.com/news/harris-speech-ellipse-ancient-mayan-city-morning-rundown-rcna177973)[](https://www.nbcnews.com/news/harris-speech-ellipse-ancient-mayan-city-morning-rundown-rcna177973)\n",
+      "\n",
+      "Skip to Content\n",
+      "\n",
+      "[NBC News Logo](https://www.nbcnews.com)\n",
+      "\n",
+      "Spon\n"
+     ]
+    }
+   ],
+   "source": [
+    "async def crawl_dynamic_content():\n",
+    "    # You can use wait_for to wait for a condition to be met before returning the result\n",
+    "    # wait_for = \"\"\"() => {\n",
+    "    #     return Array.from(document.querySelectorAll('article.tease-card')).length > 10;\n",
+    "    # }\"\"\"\n",
+    "\n",
+    "    # wait_for can be also just a css selector\n",
+    "    # wait_for = \"article.tease-card:nth-child(10)\"\n",
+    "\n",
+    "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+    "        js_code = [\n",
+    "            \"const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();\"\n",
+    "        ]\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://www.nbcnews.com/business\",\n",
+    "            js_code=js_code,\n",
+    "            # wait_for=wait_for,\n",
+    "            bypass_cache=True,\n",
+    "        )\n",
+    "        print(result.markdown.raw_markdown[:500])  # Print first 500 characters\n",
+    "\n",
+    "asyncio.run(crawl_dynamic_content())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "86febd8d",
+   "metadata": {},
+   "source": [
+    "#### 4. **Content Cleaning and Fit Markdown**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8e8ab01f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "async def clean_content():\n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://janineintheworld.com/places-to-visit-in-central-mexico\",\n",
+    "            excluded_tags=['nav', 'footer', 'aside'],\n",
+    "            remove_overlay_elements=True,\n",
+    "            word_count_threshold=10,\n",
+    "            bypass_cache=True\n",
+    "        )\n",
+    "        full_markdown_length = len(result.markdown.raw_markdown)\n",
+    "        fit_markdown_length = len(result.markdown.fit_markdown)\n",
+    "        print(f\"Full Markdown Length: {full_markdown_length}\")\n",
+    "        print(f\"Fit Markdown Length: {fit_markdown_length}\")\n",
+    "        print(result.markdown.fit_markdown[:1000])\n",
+    "        \n",
+    "\n",
+    "asyncio.run(clean_content())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "55715146",
+   "metadata": {},
+   "source": [
+    "#### 5. **Link Analysis and Smart Filtering**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "2ae47c69",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business, success: True, time taken: 0.93 seconds\n",
+      "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.11 seconds\n",
+      "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business, Strategy: AsyncWebCrawler\n",
+      "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.11 seconds.\n",
+      "Found 107 internal links\n",
+      "Found 58 external links\n",
+      "Href: https://www.nbcnews.com/news/harris-speech-ellipse-ancient-mayan-city-morning-rundown-rcna177973\n",
+      "Text: Morning Rundown: Trump and Harris' vastly different closing pitches, why Kim Jong Un is helping Russia, and an ancient city is discovered by accident\n",
+      "\n",
+      "Href: https://www.nbcnews.com\n",
+      "Text: NBC News Logo\n",
+      "\n",
+      "Href: https://www.nbcnews.com/politics/2024-election/live-blog/kamala-harris-donald-trump-rally-election-live-updates-rcna177529\n",
+      "Text: 2024 Election\n",
+      "\n",
+      "Href: https://www.nbcnews.com/politics\n",
+      "Text: Politics\n",
+      "\n",
+      "Href: https://www.nbcnews.com/us-news\n",
+      "Text: U.S. News\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "async def link_analysis():\n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://www.nbcnews.com/business\",\n",
+    "            bypass_cache=True,\n",
+    "            exclude_external_links=True,\n",
+    "            exclude_social_media_links=True,\n",
+    "            # exclude_domains=[\"facebook.com\", \"twitter.com\"]\n",
+    "        )\n",
+    "        print(f\"Found {len(result.links['internal'])} internal links\")\n",
+    "        print(f\"Found {len(result.links['external'])} external links\")\n",
+    "\n",
+    "        for link in result.links['internal'][:5]:\n",
+    "            print(f\"Href: {link['href']}\\nText: {link['text']}\\n\")\n",
+    "                \n",
+    "\n",
+    "asyncio.run(link_analysis())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "80cceef3",
+   "metadata": {},
+   "source": [
+    "#### 6. **Media Handling**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "1fed7f99",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business, success: True, time taken: 1.42 seconds\n",
+      "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.11 seconds\n",
+      "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business, Strategy: AsyncWebCrawler\n",
+      "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.12 seconds.\n",
+      "Image URL: https://media-cldnry.s-nbcnews.com/image/upload/t_focal-762x508,f_auto,q_auto:best/rockcms/2024-10/241023-NM-Chilccare-jg-27b982.jpg, Alt: , Score: 4\n",
+      "Image URL: https://media-cldnry.s-nbcnews.com/image/upload/t_focal-80x80,f_auto,q_auto:best/rockcms/2024-10/241030-china-ev-electric-mb-0746-cae05c.jpg, Alt: Volkswagen Workshop in Hefei, Score: 5\n",
+      "Image URL: https://media-cldnry.s-nbcnews.com/image/upload/t_focal-80x80,f_auto,q_auto:best/rockcms/2024-10/241029-nyc-subway-sandwich-2021-ac-922p-a92374.jpg, Alt: A sub is prepared at a Subway restaurant in Manhattan, New York City, Score: 5\n",
+      "Image URL: https://media-cldnry.s-nbcnews.com/image/upload/t_focal-80x80,f_auto,q_auto:best/rockcms/2024-10/241029-suv-gravity-ch-1618-752415.jpg, Alt: The Lucid Gravity car., Score: 5\n",
+      "Image URL: https://media-cldnry.s-nbcnews.com/image/upload/t_focal-80x80,f_auto,q_auto:best/rockcms/2024-10/241029-dearborn-michigan-f-150-ford-ranger-trucks-assembly-line-ac-426p-614f0b.jpg, Alt: Ford Introduces new F-150 And Ranger Trucks At Their Dearborn Plant, Score: 5\n"
+     ]
+    }
+   ],
+   "source": [
+    "async def media_handling():\n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://www.nbcnews.com/business\", \n",
+    "            bypass_cache=True,\n",
+    "            exclude_external_images=False,\n",
+    "            screenshot=True\n",
+    "        )\n",
+    "        for img in result.media['images'][:5]:\n",
+    "            print(f\"Image URL: {img['src']}, Alt: {img['alt']}, Score: {img['score']}\")\n",
+    "        \n",
+    "asyncio.run(media_handling())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9290499a",
+   "metadata": {},
+   "source": [
+    "#### 7. **Using Hooks for Custom Workflow**"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9d069c2b",
+   "metadata": {},
+   "source": [
+    "Hooks in Crawl4AI allow you to run custom logic at specific stages of the crawling process. This can be invaluable for scenarios like setting custom headers, logging activities, or processing content before it is returned. Below is an example of a basic workflow using a hook, followed by a complete list of available hooks and explanations on their usage."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bc4d2fc8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[Hook] Preparing to navigate...\n",
+      "[LOG] 🚀 Crawling done for https://crawl4ai.com, success: True, time taken: 3.49 seconds\n",
+      "[LOG] 🚀 Content extracted for https://crawl4ai.com, success: True, time taken: 0.03 seconds\n",
+      "[LOG] 🔥 Extracting semantic blocks for https://crawl4ai.com, Strategy: AsyncWebCrawler\n",
+      "[LOG] 🚀 Extraction done for https://crawl4ai.com, time taken: 0.03 seconds.\n",
+      "[Crawl4AI Documentation](https://docs.crawl4ai.com/)\n",
+      "\n",
+      "  * [ Home ](.)\n",
+      "  * [ Installation ](basic/installation/)\n",
+      "  * [ Quick Start ](basic/quickstart/)\n",
+      "  * [ Search ](#)\n",
+      "\n",
+      "\n",
+      "\n",
+      "  * Home\n",
+      "  * [Installation](basic/installation/)\n",
+      "  * [Quick Start](basic/quickstart/)\n",
+      "  * Basic\n",
+      "    * [Simple Crawling](basic/simple-crawling/)\n",
+      "    * [Output Formats](basic/output-formats/)\n",
+      "    * [Browser Configuration](basic/browser-config/)\n",
+      "    * [Page Interaction](basic/page-interaction/)\n",
+      "    * [Content Selection](basic/con\n"
+     ]
+    }
+   ],
+   "source": [
+    "async def custom_hook_workflow():\n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        # Set a 'before_goto' hook to run custom code just before navigation\n",
+    "        crawler.crawler_strategy.set_hook(\"before_goto\", lambda page: print(\"[Hook] Preparing to navigate...\"))\n",
+    "        \n",
+    "        # Perform the crawl operation\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://crawl4ai.com\",\n",
+    "            bypass_cache=True\n",
+    "        )\n",
+    "        print(result.markdown.raw_markdown[:500])  # Display the first 500 characters\n",
+    "\n",
+    "asyncio.run(custom_hook_workflow())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3ff45e21",
+   "metadata": {},
+   "source": [
+    "List of available hooks and examples for each stage of the crawling process:\n",
+    "\n",
+    "- **on_browser_created**\n",
+    "    ```python\n",
+    "    async def on_browser_created_hook(browser):\n",
+    "        print(\"[Hook] Browser created\")\n",
+    "    ```\n",
+    "\n",
+    "- **before_goto**\n",
+    "    ```python\n",
+    "    async def before_goto_hook(page):\n",
+    "        await page.set_extra_http_headers({\"X-Test-Header\": \"test\"})\n",
+    "    ```\n",
+    "\n",
+    "- **after_goto**\n",
+    "    ```python\n",
+    "    async def after_goto_hook(page):\n",
+    "        print(f\"[Hook] Navigated to {page.url}\")\n",
+    "    ```\n",
+    "\n",
+    "- **on_execution_started**\n",
+    "    ```python\n",
+    "    async def on_execution_started_hook(page):\n",
+    "        print(\"[Hook] JavaScript execution started\")\n",
+    "    ```\n",
+    "\n",
+    "- **before_return_html**\n",
+    "    ```python\n",
+    "    async def before_return_html_hook(page, html):\n",
+    "        print(f\"[Hook] HTML length: {len(html)}\")\n",
+    "    ```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2d56ebb1",
+   "metadata": {},
+   "source": [
+    "#### 8. **Session-Based Crawling**\n",
+    "\n",
+    "When to Use Session-Based Crawling: \n",
+    "Session-based crawling is especially beneficial when navigating through multi-page content where each page load needs to maintain the same session context. For instance, in cases where a “Next Page” button must be clicked to load subsequent data, the new data often replaces the previous content. Here, session-based crawling keeps the browser state intact across each interaction, allowing for sequential actions within the same session.\n",
+    "\n",
+    "Example: Multi-Page Navigation Using JavaScript\n",
+    "In this example, we’ll navigate through multiple pages by clicking a \"Next Page\" button. After each page load, we extract the new content and repeat the process."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e7bfebae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "async def multi_page_session_crawl():\n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        session_id = \"page_navigation_session\"\n",
+    "        url = \"https://example.com/paged-content\"\n",
+    "\n",
+    "        for page_number in range(1, 4):\n",
+    "            result = await crawler.arun(\n",
+    "                url=url,\n",
+    "                session_id=session_id,\n",
+    "                js_code=\"document.querySelector('.next-page-button').click();\" if page_number > 1 else None,\n",
+    "                css_selector=\".content-section\",\n",
+    "                bypass_cache=True\n",
+    "            )\n",
+    "            print(f\"Page {page_number} Content:\")\n",
+    "            print(result.markdown.raw_markdown[:500])  # Print first 500 characters\n",
+    "\n",
+    "# asyncio.run(multi_page_session_crawl())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ad32a778",
+   "metadata": {},
+   "source": [
+    "#### 9. **Using Extraction Strategies**\n",
+    "\n",
+    "**LLM Extraction**\n",
+    "\n",
+    "This example demonstrates how to use language model-based extraction to retrieve structured data from a pricing page on OpenAI’s site."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "3011a7c5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "--- Extracting Structured Data with openai/gpt-4o-mini ---\n",
+      "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+      "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+      "[LOG] 🕸️ Crawling https://openai.com/api/pricing/ using AsyncPlaywrightCrawlerStrategy...\n",
+      "[LOG] ✅ Crawled https://openai.com/api/pricing/ successfully!\n",
+      "[LOG] 🚀 Crawling done for https://openai.com/api/pricing/, success: True, time taken: 1.29 seconds\n",
+      "[LOG] 🚀 Content extracted for https://openai.com/api/pricing/, success: True, time taken: 0.13 seconds\n",
+      "[LOG] 🔥 Extracting semantic blocks for https://openai.com/api/pricing/, Strategy: AsyncWebCrawler\n",
+      "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 0\n",
+      "[LOG] Extracted 26 blocks from URL: https://openai.com/api/pricing/ block index: 0\n",
+      "[LOG] 🚀 Extraction done for https://openai.com/api/pricing/, time taken: 15.12 seconds.\n",
+      "[{'model_name': 'gpt-4o', 'input_fee': '$2.50 / 1M input tokens', 'output_fee': '$10.00 / 1M output tokens', 'error': False}, {'model_name': 'gpt-4o-2024-08-06', 'input_fee': '$2.50 / 1M input tokens', 'output_fee': '$10.00 / 1M output tokens', 'error': False}, {'model_name': 'gpt-4o-audio-preview', 'input_fee': '$2.50 / 1M input tokens', 'output_fee': '$10.00 / 1M output tokens', 'error': False}, {'model_name': 'gpt-4o-audio-preview-2024-10-01', 'input_fee': '$2.50 / 1M input tokens', 'output_fee': '$10.00 / 1M output tokens', 'error': False}, {'model_name': 'gpt-4o-2024-05-13', 'input_fee': '$5.00 / 1M input tokens', 'output_fee': '$15.00 / 1M output tokens', 'error': False}]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/unclecode/devs/crawl4ai/venv/lib/python3.10/site-packages/pydantic/main.py:347: UserWarning: Pydantic serializer warnings:\n",
+      "  Expected `PromptTokensDetails` but got `dict` - serialized value may not be as expected\n",
+      "  return self.__pydantic_serializer__.to_python(\n"
+     ]
+    }
+   ],
+   "source": [
+    "from crawl4ai import LLMExtractionStrategy\n",
+    "from pydantic import BaseModel, Field\n",
+    "import os, json\n",
+    "\n",
+    "class OpenAIModelFee(BaseModel):\n",
+    "    model_name: str = Field(..., description=\"Name of the OpenAI model.\")\n",
+    "    input_fee: str = Field(..., description=\"Fee for input token for the OpenAI model.\")\n",
+    "    output_fee: str = Field(\n",
+    "        ..., description=\"Fee for output token for the OpenAI model.\"\n",
+    "    )\n",
+    "\n",
+    "async def extract_structured_data_using_llm(provider: str, api_token: str = None, extra_headers: dict = None):\n",
+    "    print(f\"\\n--- Extracting Structured Data with {provider} ---\")\n",
+    "    \n",
+    "    # Skip if API token is missing (for providers that require it)\n",
+    "    if api_token is None and provider != \"ollama\":\n",
+    "        print(f\"API token is required for {provider}. Skipping this example.\")\n",
+    "        return\n",
+    "\n",
+    "    extra_args = {\"extra_headers\": extra_headers} if extra_headers else {}\n",
+    "\n",
+    "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://openai.com/api/pricing/\",\n",
+    "            word_count_threshold=1,\n",
+    "            extraction_strategy=LLMExtractionStrategy(\n",
+    "                provider=provider,\n",
+    "                api_token=api_token,\n",
+    "                schema=OpenAIModelFee.schema(),\n",
+    "                extraction_type=\"schema\",\n",
+    "                instruction=\"\"\"Extract all model names along with fees for input and output tokens.\"\n",
+    "                \"{model_name: 'GPT-4', input_fee: 'US$10.00 / 1M tokens', output_fee: 'US$30.00 / 1M tokens'}.\"\"\",\n",
+    "                **extra_args\n",
+    "            ),\n",
+    "            bypass_cache=True,\n",
+    "        )\n",
+    "        print(json.loads(result.extracted_content)[:5])\n",
+    "\n",
+    "# Usage:\n",
+    "await extract_structured_data_using_llm(\"openai/gpt-4o-mini\", os.getenv(\"OPENAI_API_KEY\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6532db9d",
+   "metadata": {},
+   "source": [
+    "**Cosine Similarity Strategy**\n",
+    "\n",
+    "This strategy uses semantic clustering to extract relevant content based on contextual similarity, which is helpful when extracting related sections from a single topic."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "ec079108",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[LOG] Loading Extraction Model for mps device.\n",
+      "[LOG] Loading Multilabel Classifier for mps device.\n",
+      "[LOG] Model loaded sentence-transformers/all-MiniLM-L6-v2, models/reuters, took 5.193778038024902 seconds\n",
+      "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business/consumer/how-mcdonalds-e-coli-crisis-inflation-politics-reflect-american-story-rcna177156, success: True, time taken: 1.37 seconds\n",
+      "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business/consumer/how-mcdonalds-e-coli-crisis-inflation-politics-reflect-american-story-rcna177156, success: True, time taken: 0.07 seconds\n",
+      "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business/consumer/how-mcdonalds-e-coli-crisis-inflation-politics-reflect-american-story-rcna177156, Strategy: AsyncWebCrawler\n",
+      "[LOG] 🚀 Assign tags using mps\n",
+      "[LOG] 🚀 Categorization done in 0.55 seconds\n",
+      "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business/consumer/how-mcdonalds-e-coli-crisis-inflation-politics-reflect-american-story-rcna177156, time taken: 6.63 seconds.\n",
+      "[{'index': 1, 'tags': ['news_&_social_concern'], 'content': \"McDonald's 2024 combo: Inflation, a health crisis and a side of politics # McDonald's 2024 combo: Inflation, a health crisis and a side of politics\"}, {'index': 2, 'tags': ['business_&_entrepreneurs', 'news_&_social_concern'], 'content': 'Like many major brands, McDonald’s raked in big profits as the economy reopened from the pandemic. In October 2022, [executives were boasting](https://www.cnbc.com/2022/10/27/mcdonalds-mcd-earnings-q3-2022.html) that they’d been raising prices without crimping traffic, even as competitors began to warn that some customers were closing their wallets after inflation peaked above 9% that summer. Still, the U.S. had repeatedly dodged a much-forecast recession, and [Americans kept spending on nonessentials](https://www.nbcnews.com/business/economy/year-peak-inflation-travel-leisure-mostly-cost-less-rcna92760) like travel and dining out — despite regularly relaying to pollsters their dismal views of an otherwise solid economy. Even so, 64% of consumers said they noticed price increases at quick-service restaurants in September, more than at any other type of venue, according to a survey by Datassential, a food and beverage market researcher. Politicians are still drawing attention to fast-food costs, too, as the election season barrels toward a tumultuous finish. A group of Democratic senators this month [denounced McDonald’s for menu prices](https://www.nbcnews.com/news/us-news/democratic-senators-slam-mcdonalds-menu-price-hikes-rcna176380) that they said outstripped inflation, accusing the company of looking to profit “at the expense of people’s ability to put food on the table.” The financial results come toward the end of a humbling year for the nearly $213 billion restaurant chain, whose shares remained steady on the heels of its latest earnings. Kempczinski [sought to reassure investors](https://www.cnbc.com/2024/10/29/mcdonalds-e-coli-outbreak-ceo-comments.html) that [the E. coli outbreak](https://www.nbcnews.com/health/health-news/illnesses-linked-mcdonalds-e-coli-outbreak-rise-75-cdc-says-rcna177260), linked to Quarter Pounder burgers, was under control after the health crisis temporarily dented the company’s stock and caused U.S. foot traffic to drop nearly 10% in the days afterward, according to estimates by Gordon Haskett financial researchers. The fast-food giant [reported Tuesday](https://www.cnbc.com/2024/10/29/mcdonalds-mcd-earnings-q3-2024.html) that it had reversed its recent U.S. sales drop, posting a 0.3% uptick in the third quarter. Foot traffic was still down slightly, but the company said its summer of discounts was paying off. But by early this year, [photos of eye-watering menu prices](https://x.com/sam_learner/status/1681367351143301129) at some McDonald’s locations — including an $18 Big Mac combo at a Connecticut rest stop from July 2023 — went viral, bringing diners’ long-simmering frustrations to a boiling point that the company couldn’t ignore. On an earnings call in April, Kempczinski acknowledged that foot traffic had fallen. “We will stay laser-focused on providing an unparalleled experience with simple, everyday value and affordability that our consumers can count on as they continue to be mindful about their spending,” CEO Chris Kempczinski [said in a statement](https://www.prnewswire.com/news-releases/mcdonalds-reports-third-quarter-2024-results-302289216.html?Fds-Load-Behavior=force-external) alongside the earnings report.'}, {'index': 3, 'tags': ['food_&_dining', 'news_&_social_concern'], 'content': '![mcdonalds drive-thru economy fast food](https://media-cldnry.s-nbcnews.com/image/upload/t_fit-760w,f_auto,q_auto:best/rockcms/2024-10/241024-los-angeles-mcdonalds-drive-thru-ac-1059p-cfc311.jpg)McDonald’s has had some success leaning into discounts this year. Eric Thayer / Bloomberg via Getty Images file'}, {'index': 4, 'tags': ['business_&_entrepreneurs', 'food_&_dining', 'news_&_social_concern'], 'content': 'McDonald’s has faced a customer revolt over pricey Big Macs, an unsolicited cameo in election-season crossfire, and now an E. coli outbreak — just as the company had been luring customers back with more affordable burgers. Despite a difficult quarter, McDonald’s looks resilient in the face of various pressures, analysts say — something the company shares with U.S. consumers overall. “Consumers continue to be even more discriminating with every dollar that they spend,” he said at the time. Going forward, McDonald’s would be “laser-focused” on affordability.  “McDonald’s has also done a good job of embedding the brand in popular culture to enhance its relevance and meaning around fun and family. But it also needed to modify the product line to meet the expectations of a consumer who is on a tight budget,” he said. “The thing that McDonald’s had struggled with, and why I think we’re seeing kind of an inflection point, is a value proposition,” Senatore said. “McDonald’s menu price increases had run ahead of a lot of its restaurant peers. … Consumers are savvy enough to know that.” For many consumers, the fast-food giant’s menus serve as an informal gauge of the economy overall, said Sara Senatore, a Bank of America analyst covering restaurants. “The spotlight is always on McDonald’s because it’s so big” and something of a “bellwether,” she said. McDonald’s didn’t respond to requests for comment.'}, {'index': 5, 'tags': ['business_&_entrepreneurs', 'food_&_dining'], 'content': 'Mickey D’s’ $5 meal deal, which it launched in late June to jumpstart slumping sales, has given the company an appealing price point to advertise nationwide, Senatore said, speculating that it could open the door to a new permanent value offering. But before that promotion rolled out, the company’s reputation as a low-cost option had taken a bruising hit.'}]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from crawl4ai import CosineStrategy\n",
+    "\n",
+    "async def cosine_similarity_extraction():\n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        strategy = CosineStrategy(\n",
+    "            word_count_threshold=10,\n",
+    "            max_dist=0.2, # Maximum distance between two words\n",
+    "            linkage_method=\"ward\", # Linkage method for hierarchical clustering (ward, complete, average, single)\n",
+    "            top_k=3, # Number of top keywords to extract\n",
+    "            sim_threshold=0.3, # Similarity threshold for clustering\n",
+    "            semantic_filter=\"McDonald's economic impact, American consumer trends\", # Keywords to filter the content semantically using embeddings\n",
+    "            verbose=True\n",
+    "        )\n",
+    "        \n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://www.nbcnews.com/business/consumer/how-mcdonalds-e-coli-crisis-inflation-politics-reflect-american-story-rcna177156\",\n",
+    "            extraction_strategy=strategy\n",
+    "        )\n",
+    "        print(json.loads(result.extracted_content)[:5])\n",
+    "\n",
+    "asyncio.run(cosine_similarity_extraction())\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ff423629",
+   "metadata": {},
+   "source": [
+    "#### 10. **Conclusion and Next Steps**\n",
+    "\n",
+    "You’ve explored core features of Crawl4AI, including dynamic content handling, link analysis, and advanced extraction strategies. Visit our documentation for further details on using Crawl4AI’s extensive features.\n",
+    "\n",
+    "- GitHub Repository: [https://github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)\n",
+    "- Twitter: [@unclecode](https://twitter.com/unclecode)\n",
+    "- Website: [https://crawl4ai.com](https://crawl4ai.com)\n",
+    "\n",
+    "Happy Crawling with Crawl4AI! 🕷️🤖\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d34c1d35",
+   "metadata": {},
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/docs/examples/quickstart.py b/docs/examples/quickstart.py
new file mode 100644
index 0000000..9992a6b
--- /dev/null
+++ b/docs/examples/quickstart.py
@@ -0,0 +1,562 @@
+import os, sys
+
+from crawl4ai import LLMConfig
+
+sys.path.append(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+)
+
+import asyncio
+import time
+import json
+import re
+from typing import Dict
+from bs4 import BeautifulSoup
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai import (
+    JsonCssExtractionStrategy,
+    LLMExtractionStrategy,
+)
+
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+print("Crawl4AI: Advanced Web Crawling and Data Extraction")
+print("GitHub Repository: https://github.com/unclecode/crawl4ai")
+print("Twitter: @unclecode")
+print("Website: https://crawl4ai.com")
+
+
+# Basic Example - Simple Crawl
+async def simple_crawl():
+    print("\n--- Basic Usage ---")
+    browser_config = BrowserConfig(headless=True)
+    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        print(result.markdown[:500])
+
+
+async def clean_content():
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        excluded_tags=["nav", "footer", "aside"],
+        remove_overlay_elements=True,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(
+                threshold=0.48, threshold_type="fixed", min_word_threshold=0
+            ),
+            options={"ignore_links": True},
+        ),
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://en.wikipedia.org/wiki/Apple",
+            config=crawler_config,
+        )
+        full_markdown_length = len(result.markdown.raw_markdown)
+        fit_markdown_length = len(result.markdown.fit_markdown)
+        print(f"Full Markdown Length: {full_markdown_length}")
+        print(f"Fit Markdown Length: {fit_markdown_length}")
+
+
+async def link_analysis():
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.ENABLED,
+        exclude_external_links=True,
+        exclude_social_media_links=True,
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            config=crawler_config,
+        )
+        print(f"Found {len(result.links['internal'])} internal links")
+        print(f"Found {len(result.links['external'])} external links")
+
+        for link in result.links["internal"][:5]:
+            print(f"Href: {link['href']}\nText: {link['text']}\n")
+
+
+# JavaScript Execution Example
+async def simple_example_with_running_js_code():
+    print("\n--- Executing JavaScript and Using CSS Selectors ---")
+
+    browser_config = BrowserConfig(headless=True, java_script_enabled=True)
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        js_code="const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();",
+        # wait_for="() => { return Array.from(document.querySelectorAll('article.tease-card')).length > 10; }"
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        print(result.markdown[:500])
+
+
+# CSS Selector Example
+async def simple_example_with_css_selector():
+    print("\n--- Using CSS Selectors ---")
+    browser_config = BrowserConfig(headless=True)
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, css_selector=".wide-tease-item__description"
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        print(result.markdown[:500])
+
+
+async def media_handling():
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, exclude_external_images=True, screenshot=True
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        for img in result.media["images"][:5]:
+            print(f"Image URL: {img['src']}, Alt: {img['alt']}, Score: {img['score']}")
+
+
+async def custom_hook_workflow(verbose=True):
+    async with AsyncWebCrawler() as crawler:
+        # Set a 'before_goto' hook to run custom code just before navigation
+        crawler.crawler_strategy.set_hook(
+            "before_goto",
+            lambda page, context: print("[Hook] Preparing to navigate..."),
+        )
+
+        # Perform the crawl operation
+        result = await crawler.arun(url="https://crawl4ai.com")
+        print(result.markdown.raw_markdown[:500].replace("\n", " -- "))
+
+
+# Proxy Example
+async def use_proxy():
+    print("\n--- Using a Proxy ---")
+    browser_config = BrowserConfig(
+        headless=True,
+        proxy_config={
+            "server": "http://proxy.example.com:8080",
+            "username": "username",
+            "password": "password",
+        },
+    )
+    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        if result.success:
+            print(result.markdown[:500])
+
+
+# Screenshot Example
+async def capture_and_save_screenshot(url: str, output_path: str):
+    browser_config = BrowserConfig(headless=True)
+    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, screenshot=True)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url=url, config=crawler_config)
+
+        if result.success and result.screenshot:
+            import base64
+
+            screenshot_data = base64.b64decode(result.screenshot)
+            with open(output_path, "wb") as f:
+                f.write(screenshot_data)
+            print(f"Screenshot saved successfully to {output_path}")
+        else:
+            print("Failed to capture screenshot")
+
+
+# LLM Extraction Example
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(
+        ..., description="Fee for output token for the OpenAI model."
+    )
+
+
+async def extract_structured_data_using_llm(
+    provider: str, api_token: str = None, extra_headers: Dict[str, str] = None
+):
+    print(f"\n--- Extracting Structured Data with {provider} ---")
+
+    if api_token is None and provider != "ollama":
+        print(f"API token is required for {provider}. Skipping this example.")
+        return
+
+    browser_config = BrowserConfig(headless=True)
+
+    extra_args = {"temperature": 0, "top_p": 0.9, "max_tokens": 2000}
+    if extra_headers:
+        extra_args["extra_headers"] = extra_headers
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        word_count_threshold=1,
+        page_timeout=80000,
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider=provider,api_token=api_token),
+            schema=OpenAIModelFee.model_json_schema(),
+            extraction_type="schema",
+            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+            Do not miss any models in the entire content.""",
+            extra_args=extra_args,
+        ),
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://openai.com/api/pricing/", config=crawler_config
+        )
+        print(result.extracted_content)
+
+
+# CSS Extraction Example
+async def extract_structured_data_using_css_extractor():
+    print("\n--- Using JsonCssExtractionStrategy for Fast Structured Output ---")
+    schema = {
+        "name": "KidoCode Courses",
+        "baseSelector": "section.charge-methodology .framework-collection-item.w-dyn-item",
+        "fields": [
+            {
+                "name": "section_title",
+                "selector": "h3.heading-50",
+                "type": "text",
+            },
+            {
+                "name": "section_description",
+                "selector": ".charge-content",
+                "type": "text",
+            },
+            {
+                "name": "course_name",
+                "selector": ".text-block-93",
+                "type": "text",
+            },
+            {
+                "name": "course_description",
+                "selector": ".course-content-text",
+                "type": "text",
+            },
+            {
+                "name": "course_icon",
+                "selector": ".image-92",
+                "type": "attribute",
+                "attribute": "src",
+            },
+        ],
+    }
+
+    browser_config = BrowserConfig(headless=True, java_script_enabled=True)
+
+    js_click_tabs = """
+    (async () => {
+        const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");
+        for(let tab of tabs) {
+            tab.scrollIntoView();
+            tab.click();
+            await new Promise(r => setTimeout(r, 500));
+        }
+    })();
+    """
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=JsonCssExtractionStrategy(schema),
+        js_code=[js_click_tabs],
+        delay_before_return_html=1
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology", config=crawler_config
+        )
+
+        companies = json.loads(result.extracted_content)
+        print(f"Successfully extracted {len(companies)} companies")
+        print(json.dumps(companies[0], indent=2))
+
+
+# Dynamic Content Examples - Method 1
+async def crawl_dynamic_content_pages_method_1():
+    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
+    first_commit = ""
+
+    async def on_execution_started(page, **kwargs):
+        nonlocal first_commit
+        try:
+            while True:
+                await page.wait_for_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await page.query_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await commit.evaluate("(element) => element.textContent")
+                commit = re.sub(r"\s+", "", commit)
+                if commit and commit != first_commit:
+                    first_commit = commit
+                    break
+                await asyncio.sleep(0.5)
+        except Exception as e:
+            print(f"Warning: New content didn't appear after JavaScript execution: {e}")
+
+    browser_config = BrowserConfig(headless=False, java_script_enabled=True)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        js_next_page = """
+        const button = document.querySelector('a[data-testid="pagination-next-button"]');
+        if (button) button.click();
+        """
+
+        for page in range(3):
+            crawler_config = CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS,
+                css_selector="li.Box-sc-g0xbh4-0",
+                js_code=js_next_page if page > 0 else None,
+                js_only=page > 0,
+                session_id=session_id,
+            )
+
+            result = await crawler.arun(url=url, config=crawler_config)
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            soup = BeautifulSoup(result.cleaned_html, "html.parser")
+            commits = soup.select("li")
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+# Dynamic Content Examples - Method 2
+async def crawl_dynamic_content_pages_method_2():
+    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
+
+    browser_config = BrowserConfig(headless=False, java_script_enabled=True)
+
+    js_next_page_and_wait = """
+    (async () => {
+        const getCurrentCommit = () => {
+            const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+            return commits.length > 0 ? commits[0].textContent.trim() : null;
+        };
+
+        const initialCommit = getCurrentCommit();
+        const button = document.querySelector('a[data-testid="pagination-next-button"]');
+        if (button) button.click();
+
+        while (true) {
+            await new Promise(resolve => setTimeout(resolve, 100));
+            const newCommit = getCurrentCommit();
+            if (newCommit && newCommit !== initialCommit) {
+                break;
+            }
+        }
+    })();
+    """
+
+    schema = {
+        "name": "Commit Extractor",
+        "baseSelector": "li.Box-sc-g0xbh4-0",
+        "fields": [
+            {
+                "name": "title",
+                "selector": "h4.markdown-title",
+                "type": "text",
+                "transform": "strip",
+            },
+        ],
+    }
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        extraction_strategy = JsonCssExtractionStrategy(schema)
+
+        for page in range(3):
+            crawler_config = CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS,
+                css_selector="li.Box-sc-g0xbh4-0",
+                extraction_strategy=extraction_strategy,
+                js_code=js_next_page_and_wait if page > 0 else None,
+                js_only=page > 0,
+                session_id=session_id,
+            )
+
+            result = await crawler.arun(url=url, config=crawler_config)
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            commits = json.loads(result.extracted_content)
+            all_commits.extend(commits)
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+async def cosine_similarity_extraction():
+    from crawl4ai import CosineStrategy
+    crawl_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=CosineStrategy(
+            word_count_threshold=10,
+            max_dist=0.2,  # Maximum distance between two words
+            linkage_method="ward",  # Linkage method for hierarchical clustering (ward, complete, average, single)
+            top_k=3,  # Number of top keywords to extract
+            sim_threshold=0.3,  # Similarity threshold for clustering
+            semantic_filter="McDonald's economic impact, American consumer trends",  # Keywords to filter the content semantically using embeddings
+            verbose=True,
+        ),
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business/consumer/how-mcdonalds-e-coli-crisis-inflation-politics-reflect-american-story-rcna177156",
+            config=crawl_config,
+        )
+        print(json.loads(result.extracted_content)[:5])
+
+
+# Browser Comparison
+async def crawl_custom_browser_type():
+    print("\n--- Browser Comparison ---")
+
+    # Firefox
+    browser_config_firefox = BrowserConfig(browser_type="firefox", headless=True)
+    start = time.time()
+    async with AsyncWebCrawler(config=browser_config_firefox) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com",
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+        )
+        print("Firefox:", time.time() - start)
+        print(result.markdown[:500])
+
+    # WebKit
+    browser_config_webkit = BrowserConfig(browser_type="webkit", headless=True)
+    start = time.time()
+    async with AsyncWebCrawler(config=browser_config_webkit) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com",
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+        )
+        print("WebKit:", time.time() - start)
+        print(result.markdown[:500])
+
+    # Chromium (default)
+    browser_config_chromium = BrowserConfig(browser_type="chromium", headless=True)
+    start = time.time()
+    async with AsyncWebCrawler(config=browser_config_chromium) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com",
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+        )
+        print("Chromium:", time.time() - start)
+        print(result.markdown[:500])
+
+
+# Anti-Bot and User Simulation
+async def crawl_with_user_simulation():
+    browser_config = BrowserConfig(
+        headless=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "mobile", "os_type": "android"},
+    )
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        magic=True,
+        simulate_user=True,
+        override_navigator=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="YOUR-URL-HERE", config=crawler_config)
+        print(result.markdown)
+
+
+async def ssl_certification():
+    # Configure crawler to fetch SSL certificate
+    config = CrawlerRunConfig(
+        fetch_ssl_certificate=True,
+        cache_mode=CacheMode.BYPASS,  # Bypass cache to always get fresh certificates
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url="https://example.com", config=config)
+
+        if result.success and result.ssl_certificate:
+            cert = result.ssl_certificate
+
+            tmp_dir = os.path.join(__location__, "tmp")
+            os.makedirs(tmp_dir, exist_ok=True)
+
+            # 1. Access certificate properties directly
+            print("\nCertificate Information:")
+            print(f"Issuer: {cert.issuer.get('CN', '')}")
+            print(f"Valid until: {cert.valid_until}")
+            print(f"Fingerprint: {cert.fingerprint}")
+
+            # 2. Export certificate in different formats
+            cert.to_json(os.path.join(tmp_dir, "certificate.json"))  # For analysis
+            print("\nCertificate exported to:")
+            print(f"- JSON: {os.path.join(tmp_dir, 'certificate.json')}")
+
+            pem_data = cert.to_pem(
+                os.path.join(tmp_dir, "certificate.pem")
+            )  # For web servers
+            print(f"- PEM: {os.path.join(tmp_dir, 'certificate.pem')}")
+
+            der_data = cert.to_der(
+                os.path.join(tmp_dir, "certificate.der")
+            )  # For Java apps
+            print(f"- DER: {os.path.join(tmp_dir, 'certificate.der')}")
+
+
+# Main execution
+async def main():
+    # Basic examples
+    await simple_crawl()
+    await simple_example_with_running_js_code()
+    await simple_example_with_css_selector()
+
+    # Advanced examples
+    await extract_structured_data_using_css_extractor()
+    await extract_structured_data_using_llm(
+        "openai/gpt-4o", os.getenv("OPENAI_API_KEY")
+    )
+    await crawl_dynamic_content_pages_method_1()
+    await crawl_dynamic_content_pages_method_2()
+
+    # Browser comparisons
+    await crawl_custom_browser_type()
+
+    # Screenshot example
+    await capture_and_save_screenshot(
+        "https://www.example.com",
+        os.path.join(__location__, "tmp/example_screenshot.jpg")
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/quickstart_examples_set_1.py b/docs/examples/quickstart_examples_set_1.py
new file mode 100644
index 0000000..078d1c4
--- /dev/null
+++ b/docs/examples/quickstart_examples_set_1.py
@@ -0,0 +1,412 @@
+import asyncio
+import os
+import json
+import base64
+from pathlib import Path
+from typing import List
+from crawl4ai import ProxyConfig
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, CrawlResult
+from crawl4ai import RoundRobinProxyStrategy
+from crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategy
+from crawl4ai import LLMConfig
+from crawl4ai import PruningContentFilter, BM25ContentFilter
+from crawl4ai import DefaultMarkdownGenerator
+from crawl4ai import BFSDeepCrawlStrategy, DomainFilter, FilterChain
+from crawl4ai import BrowserConfig
+
+__cur_dir__ = Path(__file__).parent
+
+async def demo_basic_crawl():
+    """Basic web crawling with markdown generation"""
+    print("\n=== 1. Basic Web Crawling ===")
+    async with AsyncWebCrawler(config = BrowserConfig(
+        viewport_height=800,
+        viewport_width=1200,
+        headless=True,
+        verbose=True,
+    )) as crawler:
+        results: List[CrawlResult] = await crawler.arun(
+            url="https://news.ycombinator.com/"
+        )
+
+        for i, result in enumerate(results):
+            print(f"Result {i + 1}:")
+            print(f"Success: {result.success}")
+            if result.success:
+                print(f"Markdown length: {len(result.markdown.raw_markdown)} chars")
+                print(f"First 100 chars: {result.markdown.raw_markdown[:100]}...")
+            else:
+                print("Failed to crawl the URL")
+
+async def demo_parallel_crawl():
+    """Crawl multiple URLs in parallel"""
+    print("\n=== 2. Parallel Crawling ===")
+
+    urls = [
+        "https://news.ycombinator.com/",
+        "https://example.com/",
+        "https://httpbin.org/html",
+    ]
+
+    async with AsyncWebCrawler() as crawler:
+        results: List[CrawlResult] = await crawler.arun_many(
+            urls=urls,
+        )
+
+        print(f"Crawled {len(results)} URLs in parallel:")
+        for i, result in enumerate(results):
+            print(
+                f"  {i + 1}. {result.url} - {'Success' if result.success else 'Failed'}"
+            )
+
+async def demo_fit_markdown():
+    """Generate focused markdown with LLM content filter"""
+    print("\n=== 3. Fit Markdown with LLM Content Filter ===")
+
+    async with AsyncWebCrawler() as crawler:
+        result: CrawlResult = await crawler.arun(
+            url = "https://en.wikipedia.org/wiki/Python_(programming_language)",
+            config=CrawlerRunConfig(
+                markdown_generator=DefaultMarkdownGenerator(
+                    content_filter=PruningContentFilter()
+                )
+            ),
+        )
+
+        # Print stats and save the fit markdown
+        print(f"Raw: {len(result.markdown.raw_markdown)} chars")
+        print(f"Fit: {len(result.markdown.fit_markdown)} chars")
+
+async def demo_llm_structured_extraction_no_schema():
+    # Create a simple LLM extraction strategy (no schema required)
+    extraction_strategy = LLMExtractionStrategy(
+        llm_config=LLMConfig(
+            provider="groq/qwen-2.5-32b",
+            api_token="env:GROQ_API_KEY",
+        ),
+        instruction="This is news.ycombinator.com, extract all news, and for each, I want title, source url, number of comments.",
+        extract_type="schema",
+        schema="{title: string, url: string, comments: int}",
+        extra_args={
+            "temperature": 0.0,
+            "max_tokens": 4096,
+        },
+        verbose=True,
+    )
+
+    config = CrawlerRunConfig(extraction_strategy=extraction_strategy)
+
+    async with AsyncWebCrawler() as crawler:
+        results: List[CrawlResult] = await crawler.arun(
+            "https://news.ycombinator.com/", config=config
+        )
+
+        for result in results:
+            print(f"URL: {result.url}")
+            print(f"Success: {result.success}")
+            if result.success:
+                data = json.loads(result.extracted_content)
+                print(json.dumps(data, indent=2))
+            else:
+                print("Failed to extract structured data")
+
+async def demo_css_structured_extraction_no_schema():
+    """Extract structured data using CSS selectors"""
+    print("\n=== 5. CSS-Based Structured Extraction ===")
+    # Sample HTML for schema generation (one-time cost)
+    sample_html = """
+

+    
+        

+            

+                

+                    ...
+                

+            

+            

+                
Malicious Python Packages on PyPI Downloaded 39,000+ Times, Steal Sensitive Data

+                

+                    Apr 05, 2025
+                    Malware / Supply Chain Attack
+                

+                
 Cybersecurity researchers have...

+            

+        

+    
+

+    """
+
+    # Check if schema file exists
+    schema_file_path = f"{__cur_dir__}/tmp/schema.json"
+    if os.path.exists(schema_file_path):
+        with open(schema_file_path, "r") as f:
+            schema = json.load(f)
+    else:
+        # Generate schema using LLM (one-time setup)
+        schema = JsonCssExtractionStrategy.generate_schema(
+            html=sample_html,
+            llm_config=LLMConfig(
+                provider="groq/qwen-2.5-32b",
+                api_token="env:GROQ_API_KEY",
+            ),
+            query="From https://thehackernews.com/, I have shared a sample of one news div with a title, date, and description. Please generate a schema for this news div.",
+        )
+
+    print(f"Generated schema: {json.dumps(schema, indent=2)}")
+    # Save the schema to a file , and use it for future extractions, in result for such extraction you will call LLM once
+    with open(f"{__cur_dir__}/tmp/schema.json", "w") as f:
+        json.dump(schema, f, indent=2)
+
+    # Create no-LLM extraction strategy with the generated schema
+    extraction_strategy = JsonCssExtractionStrategy(schema)
+    config = CrawlerRunConfig(extraction_strategy=extraction_strategy)
+
+    # Use the fast CSS extraction (no LLM calls during extraction)
+    async with AsyncWebCrawler() as crawler:
+        results: List[CrawlResult] = await crawler.arun(
+            "https://thehackernews.com", config=config
+        )
+
+        for result in results:
+            print(f"URL: {result.url}")
+            print(f"Success: {result.success}")
+            if result.success:
+                data = json.loads(result.extracted_content)
+                print(json.dumps(data, indent=2))
+            else:
+                print("Failed to extract structured data")
+
+async def demo_deep_crawl():
+    """Deep crawling with BFS strategy"""
+    print("\n=== 6. Deep Crawling ===")
+
+    filter_chain = FilterChain([DomainFilter(allowed_domains=["crawl4ai.com"])])
+
+    deep_crawl_strategy = BFSDeepCrawlStrategy(
+        max_depth=1, max_pages=5, filter_chain=filter_chain
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        results: List[CrawlResult] = await crawler.arun(
+            url="https://docs.crawl4ai.com",
+            config=CrawlerRunConfig(deep_crawl_strategy=deep_crawl_strategy),
+        )
+
+        print(f"Deep crawl returned {len(results)} pages:")
+        for i, result in enumerate(results):
+            depth = result.metadata.get("depth", "unknown")
+            print(f"  {i + 1}. {result.url} (Depth: {depth})")
+
+async def demo_js_interaction():
+    """Execute JavaScript to load more content"""
+    print("\n=== 7. JavaScript Interaction ===")
+
+    # A simple page that needs JS to reveal content
+    async with AsyncWebCrawler(config=BrowserConfig(headless=False)) as crawler:
+        # Initial load
+
+        news_schema = {
+            "name": "news",
+            "baseSelector": "tr.athing",
+            "fields": [
+                {
+                    "name": "title",
+                    "selector": "span.titleline",
+                    "type": "text",
+                }
+            ],
+        }
+        results: List[CrawlResult] = await crawler.arun(
+            url="https://news.ycombinator.com",
+            config=CrawlerRunConfig(
+                session_id="hn_session",  # Keep session
+                extraction_strategy=JsonCssExtractionStrategy(schema=news_schema),
+            ),
+        )
+
+        news = []
+        for result in results:
+            if result.success:
+                data = json.loads(result.extracted_content)
+                news.extend(data)
+                print(json.dumps(data, indent=2))
+            else:
+                print("Failed to extract structured data")
+
+        print(f"Initial items: {len(news)}")
+
+        # Click "More" link
+        more_config = CrawlerRunConfig(
+            js_code="document.querySelector('a.morelink').click();",
+            js_only=True,  # Continue in same page
+            session_id="hn_session",  # Keep session
+            extraction_strategy=JsonCssExtractionStrategy(
+                schema=news_schema,
+            ),
+        )
+
+        result: List[CrawlResult] = await crawler.arun(
+            url="https://news.ycombinator.com", config=more_config
+        )
+
+        # Extract new items
+        for result in results:
+            if result.success:
+                data = json.loads(result.extracted_content)
+                news.extend(data)
+                print(json.dumps(data, indent=2))
+            else:
+                print("Failed to extract structured data")
+        print(f"Total items: {len(news)}")
+
+async def demo_media_and_links():
+    """Extract media and links from a page"""
+    print("\n=== 8. Media and Links Extraction ===")
+
+    async with AsyncWebCrawler() as crawler:
+        result: List[CrawlResult] = await crawler.arun("https://en.wikipedia.org/wiki/Main_Page")
+
+        for i, result in enumerate(result):
+            # Extract and save all images
+            images = result.media.get("images", [])
+            print(f"Found {len(images)} images")
+
+            # Extract and save all links (internal and external)
+            internal_links = result.links.get("internal", [])
+            external_links = result.links.get("external", [])
+            print(f"Found {len(internal_links)} internal links")
+            print(f"Found {len(external_links)} external links")
+
+            # Print some of the images and links
+            for image in images[:3]:
+                print(f"Image: {image['src']}")
+            for link in internal_links[:3]:
+                print(f"Internal link: {link['href']}")
+            for link in external_links[:3]:
+                print(f"External link: {link['href']}")
+
+            # # Save everything to files
+            with open(f"{__cur_dir__}/tmp/images.json", "w") as f:
+                json.dump(images, f, indent=2)
+
+            with open(f"{__cur_dir__}/tmp/links.json", "w") as f:
+                json.dump(
+                    {"internal": internal_links, "external": external_links},
+                    f,
+                    indent=2,
+                )
+
+async def demo_screenshot_and_pdf():
+    """Capture screenshot and PDF of a page"""
+    print("\n=== 9. Screenshot and PDF Capture ===")
+
+    async with AsyncWebCrawler() as crawler:
+        result: List[CrawlResult] = await crawler.arun(
+            # url="https://example.com",
+            url="https://en.wikipedia.org/wiki/Giant_anteater",
+            config=CrawlerRunConfig(screenshot=True, pdf=True),
+        )
+
+        for i, result in enumerate(result):
+            # if result.screenshot_data:
+            if result.screenshot:
+                # Save screenshot
+                screenshot_path = f"{__cur_dir__}/tmp/example_screenshot.png"
+                with open(screenshot_path, "wb") as f:
+                    f.write(base64.b64decode(result.screenshot))
+                print(f"Screenshot saved to {screenshot_path}")
+
+            # if result.pdf_data:
+            if result.pdf:
+                # Save PDF
+                pdf_path = f"{__cur_dir__}/tmp/example.pdf"
+                with open(pdf_path, "wb") as f:
+                    f.write(result.pdf)
+                print(f"PDF saved to {pdf_path}")
+
+async def demo_proxy_rotation():
+    """Proxy rotation for multiple requests"""
+    print("\n=== 10. Proxy Rotation ===")
+
+    # Example proxies (replace with real ones)
+    proxies = [
+        ProxyConfig(server="http://proxy1.example.com:8080"),
+        ProxyConfig(server="http://proxy2.example.com:8080"),
+    ]
+
+    proxy_strategy = RoundRobinProxyStrategy(proxies)
+
+    print(f"Using {len(proxies)} proxies in rotation")
+    print(
+        "Note: This example uses placeholder proxies - replace with real ones to test"
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            proxy_rotation_strategy=proxy_strategy
+        )
+
+        # In a real scenario, these would be run and the proxies would rotate
+        print("In a real scenario, requests would rotate through the available proxies")
+
+async def demo_raw_html_and_file():
+    """Process raw HTML and local files"""
+    print("\n=== 11. Raw HTML and Local Files ===")
+
+    raw_html = """
+    
+        
Sample Article

+        
This is sample content for testing Crawl4AI's raw HTML processing.

+    
+    """
+
+    # Save to file
+    file_path = Path("docs/examples/tmp/sample.html").absolute()
+    with open(file_path, "w") as f:
+        f.write(raw_html)
+
+    async with AsyncWebCrawler() as crawler:
+        # Crawl raw HTML
+        raw_result = await crawler.arun(
+            url="raw:" + raw_html, config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+        )
+        print("Raw HTML processing:")
+        print(f"  Markdown: {raw_result.markdown.raw_markdown[:50]}...")
+
+        # Crawl local file
+        file_result = await crawler.arun(
+            url=f"file://{file_path}",
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+        )
+        print("\nLocal file processing:")
+        print(f"  Markdown: {file_result.markdown.raw_markdown[:50]}...")
+
+    # Clean up
+    os.remove(file_path)
+    print(f"Processed both raw HTML and local file ({file_path})")
+
+async def main():
+    """Run all demo functions sequentially"""
+    print("=== Comprehensive Crawl4AI Demo ===")
+    print("Note: Some examples require API keys or other configurations")
+
+    # Run all demos
+    await demo_basic_crawl()
+    await demo_parallel_crawl()
+    await demo_fit_markdown()
+    await demo_llm_structured_extraction_no_schema()
+    await demo_css_structured_extraction_no_schema()
+    await demo_deep_crawl()
+    await demo_js_interaction()
+    await demo_media_and_links()
+    await demo_screenshot_and_pdf()
+    # # await demo_proxy_rotation()
+    await demo_raw_html_and_file()
+
+    # Clean up any temp files that may have been created
+    print("\n=== Demo Complete ===")
+    print("Check for any generated files (screenshots, PDFs) in the current directory")
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/quickstart_examples_set_2.py b/docs/examples/quickstart_examples_set_2.py
new file mode 100644
index 0000000..b12b084
--- /dev/null
+++ b/docs/examples/quickstart_examples_set_2.py
@@ -0,0 +1,562 @@
+import os, sys
+
+from crawl4ai.types import LLMConfig
+
+sys.path.append(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+)
+
+import asyncio
+import time
+import json
+import re
+from typing import Dict
+from bs4 import BeautifulSoup
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai import (
+    JsonCssExtractionStrategy,
+    LLMExtractionStrategy,
+)
+
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+print("Crawl4AI: Advanced Web Crawling and Data Extraction")
+print("GitHub Repository: https://github.com/unclecode/crawl4ai")
+print("Twitter: @unclecode")
+print("Website: https://crawl4ai.com")
+
+
+# Basic Example - Simple Crawl
+async def simple_crawl():
+    print("\n--- Basic Usage ---")
+    browser_config = BrowserConfig(headless=True)
+    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        print(result.markdown[:500])
+
+
+async def clean_content():
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        excluded_tags=["nav", "footer", "aside"],
+        remove_overlay_elements=True,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(
+                threshold=0.48, threshold_type="fixed", min_word_threshold=0
+            ),
+            options={"ignore_links": True},
+        ),
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://en.wikipedia.org/wiki/Apple",
+            config=crawler_config,
+        )
+        full_markdown_length = len(result.markdown.raw_markdown)
+        fit_markdown_length = len(result.markdown.fit_markdown)
+        print(f"Full Markdown Length: {full_markdown_length}")
+        print(f"Fit Markdown Length: {fit_markdown_length}")
+
+
+async def link_analysis():
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.ENABLED,
+        exclude_external_links=True,
+        exclude_social_media_links=True,
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            config=crawler_config,
+        )
+        print(f"Found {len(result.links['internal'])} internal links")
+        print(f"Found {len(result.links['external'])} external links")
+
+        for link in result.links["internal"][:5]:
+            print(f"Href: {link['href']}\nText: {link['text']}\n")
+
+
+# JavaScript Execution Example
+async def simple_example_with_running_js_code():
+    print("\n--- Executing JavaScript and Using CSS Selectors ---")
+
+    browser_config = BrowserConfig(headless=True, java_script_enabled=True)
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        js_code="const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();",
+        # wait_for="() => { return Array.from(document.querySelectorAll('article.tease-card')).length > 10; }"
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        print(result.markdown[:500])
+
+
+# CSS Selector Example
+async def simple_example_with_css_selector():
+    print("\n--- Using CSS Selectors ---")
+    browser_config = BrowserConfig(headless=True)
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, css_selector=".wide-tease-item__description"
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        print(result.markdown[:500])
+
+
+async def media_handling():
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, exclude_external_images=True, screenshot=True
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        for img in result.media["images"][:5]:
+            print(f"Image URL: {img['src']}, Alt: {img['alt']}, Score: {img['score']}")
+
+
+async def custom_hook_workflow(verbose=True):
+    async with AsyncWebCrawler() as crawler:
+        # Set a 'before_goto' hook to run custom code just before navigation
+        crawler.crawler_strategy.set_hook(
+            "before_goto",
+            lambda page, context: print("[Hook] Preparing to navigate..."),
+        )
+
+        # Perform the crawl operation
+        result = await crawler.arun(url="https://crawl4ai.com")
+        print(result.markdown.raw_markdown[:500].replace("\n", " -- "))
+
+
+# Proxy Example
+async def use_proxy():
+    print("\n--- Using a Proxy ---")
+    browser_config = BrowserConfig(
+        headless=True,
+        proxy_config={
+            "server": "http://proxy.example.com:8080",
+            "username": "username",
+            "password": "password",
+        },
+    )
+    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", config=crawler_config
+        )
+        if result.success:
+            print(result.markdown[:500])
+
+
+# Screenshot Example
+async def capture_and_save_screenshot(url: str, output_path: str):
+    browser_config = BrowserConfig(headless=True)
+    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, screenshot=True)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url=url, config=crawler_config)
+
+        if result.success and result.screenshot:
+            import base64
+
+            screenshot_data = base64.b64decode(result.screenshot)
+            with open(output_path, "wb") as f:
+                f.write(screenshot_data)
+            print(f"Screenshot saved successfully to {output_path}")
+        else:
+            print("Failed to capture screenshot")
+
+
+# LLM Extraction Example
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(
+        ..., description="Fee for output token for the OpenAI model."
+    )
+
+
+async def extract_structured_data_using_llm(
+    provider: str, api_token: str = None, extra_headers: Dict[str, str] = None
+):
+    print(f"\n--- Extracting Structured Data with {provider} ---")
+
+    if api_token is None and provider != "ollama":
+        print(f"API token is required for {provider}. Skipping this example.")
+        return
+
+    browser_config = BrowserConfig(headless=True)
+
+    extra_args = {"temperature": 0, "top_p": 0.9, "max_tokens": 2000}
+    if extra_headers:
+        extra_args["extra_headers"] = extra_headers
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        word_count_threshold=1,
+        page_timeout=80000,
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider=provider,api_token=api_token),
+            schema=OpenAIModelFee.model_json_schema(),
+            extraction_type="schema",
+            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+            Do not miss any models in the entire content.""",
+            extra_args=extra_args,
+        ),
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://openai.com/api/pricing/", config=crawler_config
+        )
+        print(result.extracted_content)
+
+
+# CSS Extraction Example
+async def extract_structured_data_using_css_extractor():
+    print("\n--- Using JsonCssExtractionStrategy for Fast Structured Output ---")
+    schema = {
+        "name": "KidoCode Courses",
+        "baseSelector": "section.charge-methodology .framework-collection-item.w-dyn-item",
+        "fields": [
+            {
+                "name": "section_title",
+                "selector": "h3.heading-50",
+                "type": "text",
+            },
+            {
+                "name": "section_description",
+                "selector": ".charge-content",
+                "type": "text",
+            },
+            {
+                "name": "course_name",
+                "selector": ".text-block-93",
+                "type": "text",
+            },
+            {
+                "name": "course_description",
+                "selector": ".course-content-text",
+                "type": "text",
+            },
+            {
+                "name": "course_icon",
+                "selector": ".image-92",
+                "type": "attribute",
+                "attribute": "src",
+            },
+        ],
+    }
+
+    browser_config = BrowserConfig(headless=True, java_script_enabled=True)
+
+    js_click_tabs = """
+    (async () => {
+        const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");
+        for(let tab of tabs) {
+            tab.scrollIntoView();
+            tab.click();
+            await new Promise(r => setTimeout(r, 500));
+        }
+    })();
+    """
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=JsonCssExtractionStrategy(schema),
+        js_code=[js_click_tabs],
+        delay_before_return_html=1
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology", config=crawler_config
+        )
+
+        companies = json.loads(result.extracted_content)
+        print(f"Successfully extracted {len(companies)} companies")
+        print(json.dumps(companies[0], indent=2))
+
+
+# Dynamic Content Examples - Method 1
+async def crawl_dynamic_content_pages_method_1():
+    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
+    first_commit = ""
+
+    async def on_execution_started(page, **kwargs):
+        nonlocal first_commit
+        try:
+            while True:
+                await page.wait_for_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await page.query_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await commit.evaluate("(element) => element.textContent")
+                commit = re.sub(r"\s+", "", commit)
+                if commit and commit != first_commit:
+                    first_commit = commit
+                    break
+                await asyncio.sleep(0.5)
+        except Exception as e:
+            print(f"Warning: New content didn't appear after JavaScript execution: {e}")
+
+    browser_config = BrowserConfig(headless=False, java_script_enabled=True)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        js_next_page = """
+        const button = document.querySelector('a[data-testid="pagination-next-button"]');
+        if (button) button.click();
+        """
+
+        for page in range(3):
+            crawler_config = CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS,
+                css_selector="li.Box-sc-g0xbh4-0",
+                js_code=js_next_page if page > 0 else None,
+                js_only=page > 0,
+                session_id=session_id,
+            )
+
+            result = await crawler.arun(url=url, config=crawler_config)
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            soup = BeautifulSoup(result.cleaned_html, "html.parser")
+            commits = soup.select("li")
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+# Dynamic Content Examples - Method 2
+async def crawl_dynamic_content_pages_method_2():
+    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
+
+    browser_config = BrowserConfig(headless=False, java_script_enabled=True)
+
+    js_next_page_and_wait = """
+    (async () => {
+        const getCurrentCommit = () => {
+            const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+            return commits.length > 0 ? commits[0].textContent.trim() : null;
+        };
+
+        const initialCommit = getCurrentCommit();
+        const button = document.querySelector('a[data-testid="pagination-next-button"]');
+        if (button) button.click();
+
+        while (true) {
+            await new Promise(resolve => setTimeout(resolve, 100));
+            const newCommit = getCurrentCommit();
+            if (newCommit && newCommit !== initialCommit) {
+                break;
+            }
+        }
+    })();
+    """
+
+    schema = {
+        "name": "Commit Extractor",
+        "baseSelector": "li.Box-sc-g0xbh4-0",
+        "fields": [
+            {
+                "name": "title",
+                "selector": "h4.markdown-title",
+                "type": "text",
+                "transform": "strip",
+            },
+        ],
+    }
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        extraction_strategy = JsonCssExtractionStrategy(schema)
+
+        for page in range(3):
+            crawler_config = CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS,
+                css_selector="li.Box-sc-g0xbh4-0",
+                extraction_strategy=extraction_strategy,
+                js_code=js_next_page_and_wait if page > 0 else None,
+                js_only=page > 0,
+                session_id=session_id,
+            )
+
+            result = await crawler.arun(url=url, config=crawler_config)
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            commits = json.loads(result.extracted_content)
+            all_commits.extend(commits)
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+async def cosine_similarity_extraction():
+    from crawl4ai import CosineStrategy
+    crawl_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=CosineStrategy(
+            word_count_threshold=10,
+            max_dist=0.2,  # Maximum distance between two words
+            linkage_method="ward",  # Linkage method for hierarchical clustering (ward, complete, average, single)
+            top_k=3,  # Number of top keywords to extract
+            sim_threshold=0.3,  # Similarity threshold for clustering
+            semantic_filter="McDonald's economic impact, American consumer trends",  # Keywords to filter the content semantically using embeddings
+            verbose=True,
+        ),
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business/consumer/how-mcdonalds-e-coli-crisis-inflation-politics-reflect-american-story-rcna177156",
+            config=crawl_config,
+        )
+        print(json.loads(result.extracted_content)[:5])
+
+
+# Browser Comparison
+async def crawl_custom_browser_type():
+    print("\n--- Browser Comparison ---")
+
+    # Firefox
+    browser_config_firefox = BrowserConfig(browser_type="firefox", headless=True)
+    start = time.time()
+    async with AsyncWebCrawler(config=browser_config_firefox) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com",
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+        )
+        print("Firefox:", time.time() - start)
+        print(result.markdown[:500])
+
+    # WebKit
+    browser_config_webkit = BrowserConfig(browser_type="webkit", headless=True)
+    start = time.time()
+    async with AsyncWebCrawler(config=browser_config_webkit) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com",
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+        )
+        print("WebKit:", time.time() - start)
+        print(result.markdown[:500])
+
+    # Chromium (default)
+    browser_config_chromium = BrowserConfig(browser_type="chromium", headless=True)
+    start = time.time()
+    async with AsyncWebCrawler(config=browser_config_chromium) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com",
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+        )
+        print("Chromium:", time.time() - start)
+        print(result.markdown[:500])
+
+
+# Anti-Bot and User Simulation
+async def crawl_with_user_simulation():
+    browser_config = BrowserConfig(
+        headless=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "mobile", "os_type": "android"},
+    )
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        magic=True,
+        simulate_user=True,
+        override_navigator=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="YOUR-URL-HERE", config=crawler_config)
+        print(result.markdown)
+
+
+async def ssl_certification():
+    # Configure crawler to fetch SSL certificate
+    config = CrawlerRunConfig(
+        fetch_ssl_certificate=True,
+        cache_mode=CacheMode.BYPASS,  # Bypass cache to always get fresh certificates
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url="https://example.com", config=config)
+
+        if result.success and result.ssl_certificate:
+            cert = result.ssl_certificate
+
+            tmp_dir = os.path.join(__location__, "tmp")
+            os.makedirs(tmp_dir, exist_ok=True)
+
+            # 1. Access certificate properties directly
+            print("\nCertificate Information:")
+            print(f"Issuer: {cert.issuer.get('CN', '')}")
+            print(f"Valid until: {cert.valid_until}")
+            print(f"Fingerprint: {cert.fingerprint}")
+
+            # 2. Export certificate in different formats
+            cert.to_json(os.path.join(tmp_dir, "certificate.json"))  # For analysis
+            print("\nCertificate exported to:")
+            print(f"- JSON: {os.path.join(tmp_dir, 'certificate.json')}")
+
+            pem_data = cert.to_pem(
+                os.path.join(tmp_dir, "certificate.pem")
+            )  # For web servers
+            print(f"- PEM: {os.path.join(tmp_dir, 'certificate.pem')}")
+
+            der_data = cert.to_der(
+                os.path.join(tmp_dir, "certificate.der")
+            )  # For Java apps
+            print(f"- DER: {os.path.join(tmp_dir, 'certificate.der')}")
+
+
+# Main execution
+async def main():
+    # Basic examples
+    await simple_crawl()
+    await simple_example_with_running_js_code()
+    await simple_example_with_css_selector()
+
+    # Advanced examples
+    await extract_structured_data_using_css_extractor()
+    await extract_structured_data_using_llm(
+        "openai/gpt-4o", os.getenv("OPENAI_API_KEY")
+    )
+    await crawl_dynamic_content_pages_method_1()
+    await crawl_dynamic_content_pages_method_2()
+
+    # Browser comparisons
+    await crawl_custom_browser_type()
+
+    # Screenshot example
+    await capture_and_save_screenshot(
+        "https://www.example.com",
+        os.path.join(__location__, "tmp/example_screenshot.jpg")
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/regex_extraction_quickstart.py b/docs/examples/regex_extraction_quickstart.py
new file mode 100644
index 0000000..54b9c38
--- /dev/null
+++ b/docs/examples/regex_extraction_quickstart.py
@@ -0,0 +1,143 @@
+# == File: regex_extraction_quickstart.py ==
+"""
+Mini–quick-start for RegexExtractionStrategy
+────────────────────────────────────────────
+3 bite-sized demos that parallel the style of *quickstart_examples_set_1.py*:
+
+1.  **Default catalog** – scrape a page and pull out e-mails / phones / URLs, etc.
+2.  **Custom pattern**  – add your own regex at instantiation time.
+3.  **LLM-assisted schema** – ask the model to write a pattern, cache it, then
+    run extraction _without_ further LLM calls.
+
+Run the whole thing with::
+
+    python regex_extraction_quickstart.py
+"""
+
+import os, json, asyncio
+from pathlib import Path
+from typing import List
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    CrawlResult,
+    RegexExtractionStrategy,
+    LLMConfig,
+)
+
+# ────────────────────────────────────────────────────────────────────────────
+# 1. Default-catalog extraction
+# ────────────────────────────────────────────────────────────────────────────
+async def demo_regex_default() -> None:
+    print("\n=== 1. Regex extraction – default patterns ===")
+
+    url = "https://www.iana.org/domains/example"      # has e-mail + URLs
+    strategy = RegexExtractionStrategy(
+        pattern = RegexExtractionStrategy.Url | RegexExtractionStrategy.Currency
+    )               
+    config   = CrawlerRunConfig(extraction_strategy=strategy)
+
+    async with AsyncWebCrawler() as crawler:
+        result: CrawlResult = await crawler.arun(url, config=config)
+
+    print(f"Fetched {url} - success={result.success}")
+    if result.success:
+        data = json.loads(result.extracted_content)
+        for d in data[:10]:
+            print(f"  {d['label']:<12} {d['value']}")
+        print(f"... total matches: {len(data)}")
+    else:
+        print("  !!! crawl failed")
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# 2. Custom pattern override / extension
+# ────────────────────────────────────────────────────────────────────────────
+async def demo_regex_custom() -> None:
+    print("\n=== 2. Regex extraction – custom price pattern ===")
+
+    url = "https://www.apple.com/shop/buy-mac/macbook-pro"
+    price_pattern = {"usd_price": r"\$\s?\d{1,3}(?:,\d{3})*(?:\.\d{2})?"}
+
+    strategy = RegexExtractionStrategy(custom = price_pattern)
+    config   = CrawlerRunConfig(extraction_strategy=strategy)
+
+    async with AsyncWebCrawler() as crawler:
+        result: CrawlResult = await crawler.arun(url, config=config)
+
+    if result.success:
+        data = json.loads(result.extracted_content)
+        for d in data:
+            print(f"  {d['value']}")
+        if not data:
+            print("  (No prices found - page layout may have changed)")
+    else:
+        print("  !!! crawl failed")
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# 3. One-shot LLM pattern generation, then fast extraction
+# ────────────────────────────────────────────────────────────────────────────
+async def demo_regex_generate_pattern() -> None:
+    print("\n=== 3. generate_pattern → regex extraction ===")
+
+    cache_dir   = Path(__file__).parent / "tmp"
+    cache_dir.mkdir(exist_ok=True)
+    pattern_file = cache_dir / "price_pattern.json"
+
+    url = "https://www.lazada.sg/tag/smartphone/"
+
+    # ── 3-A. build or load the cached pattern
+    if pattern_file.exists():
+        pattern = json.load(pattern_file.open(encoding="utf-8"))
+        print("Loaded cached pattern:", pattern)
+    else:
+        print("Generating pattern via LLM…")
+
+        llm_cfg = LLMConfig(
+            provider="openai/gpt-4o-mini",
+            api_token="env:OPENAI_API_KEY",
+        )
+
+        # pull one sample page as HTML context
+        async with AsyncWebCrawler() as crawler:
+            html = (await crawler.arun(url)).fit_html 
+
+        pattern = RegexExtractionStrategy.generate_pattern(
+            label="price",
+            html=html,
+            query="Prices in Malaysian Ringgit (e.g. RM1,299.00 or RM200)",
+            llm_config=llm_cfg,
+        )
+
+        json.dump(pattern, pattern_file.open("w", encoding="utf-8"), indent=2)
+        print("Saved pattern:", pattern_file)
+
+    # ── 3-B. extraction pass – zero LLM calls
+    strategy = RegexExtractionStrategy(custom=pattern)
+    config   = CrawlerRunConfig(extraction_strategy=strategy, delay_before_return_html=3)
+
+    async with AsyncWebCrawler() as crawler:
+        result: CrawlResult = await crawler.arun(url, config=config)
+
+    if result.success:
+        data = json.loads(result.extracted_content)
+        for d in data[:15]:
+            print(f"  {d['value']}")
+        print(f"... total matches: {len(data)}")
+    else:
+        print("  !!! crawl failed")
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Entrypoint
+# ────────────────────────────────────────────────────────────────────────────
+async def main() -> None:
+    # await demo_regex_default()
+    # await demo_regex_custom()
+    await demo_regex_generate_pattern()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/research_assistant.py b/docs/examples/research_assistant.py
new file mode 100644
index 0000000..84ba3c7
--- /dev/null
+++ b/docs/examples/research_assistant.py
@@ -0,0 +1,172 @@
+# Make sure to install the required packageschainlit and groq
+import os, time
+from openai import AsyncOpenAI
+import chainlit as cl
+import re
+import requests
+from io import BytesIO
+from chainlit.element import ElementBased
+from groq import Groq
+
+# Import threadpools to run the crawl_url function in a separate thread
+from concurrent.futures import ThreadPoolExecutor
+
+client = AsyncOpenAI(
+    base_url="https://api.groq.com/openai/v1", api_key=os.getenv("GROQ_API_KEY")
+)
+
+# Instrument the OpenAI client
+cl.instrument_openai()
+
+settings = {
+    "model": "llama3-8b-8192",
+    "temperature": 0.5,
+    "max_tokens": 500,
+    "top_p": 1,
+    "frequency_penalty": 0,
+    "presence_penalty": 0,
+}
+
+
+def extract_urls(text):
+    url_pattern = re.compile(r"(https?://\S+)")
+    return url_pattern.findall(text)
+
+
+def crawl_url(url):
+    data = {
+        "urls": [url],
+        "include_raw_html": True,
+        "word_count_threshold": 10,
+        "extraction_strategy": "NoExtractionStrategy",
+        "chunking_strategy": "RegexChunking",
+    }
+    response = requests.post("https://crawl4ai.com/crawl", json=data)
+    response_data = response.json()
+    response_data = response_data["results"][0]
+    return response_data["markdown"]
+
+
+@cl.on_chat_start
+async def on_chat_start():
+    cl.user_session.set("session", {"history": [], "context": {}})
+    await cl.Message(content="Welcome to the chat! How can I assist you today?").send()
+
+
+@cl.on_message
+async def on_message(message: cl.Message):
+    user_session = cl.user_session.get("session")
+
+    # Extract URLs from the user's message
+    urls = extract_urls(message.content)
+
+    futures = []
+    with ThreadPoolExecutor() as executor:
+        for url in urls:
+            futures.append(executor.submit(crawl_url, url))
+
+    results = [future.result() for future in futures]
+
+    for url, result in zip(urls, results):
+        ref_number = f"REF_{len(user_session['context']) + 1}"
+        user_session["context"][ref_number] = {"url": url, "content": result}
+
+    user_session["history"].append({"role": "user", "content": message.content})
+
+    # Create a system message that includes the context
+    context_messages = [
+        f'\n{data["content"]}\n'
+        for ref, data in user_session["context"].items()
+    ]
+    if context_messages:
+        system_message = {
+            "role": "system",
+            "content": (
+                "You are a helpful bot. Use the following context for answering questions. "
+                "Refer to the sources using the REF number in square brackets, e.g., [1], only if the source is given in the appendices below.\n\n"
+                "If the question requires any information from the provided appendices or context, refer to the sources. "
+                "If not, there is no need to add a references section. "
+                "At the end of your response, provide a reference section listing the URLs and their REF numbers only if sources from the appendices were used.\n\n"
+                "\n\n".join(context_messages)
+            ),
+        }
+    else:
+        system_message = {"role": "system", "content": "You are a helpful assistant."}
+
+    msg = cl.Message(content="")
+    await msg.send()
+
+    # Get response from the LLM
+    stream = await client.chat.completions.create(
+        messages=[system_message, *user_session["history"]], stream=True, **settings
+    )
+
+    assistant_response = ""
+    async for part in stream:
+        if token := part.choices[0].delta.content:
+            assistant_response += token
+            await msg.stream_token(token)
+
+    # Add assistant message to the history
+    user_session["history"].append({"role": "assistant", "content": assistant_response})
+    await msg.update()
+
+    # Append the reference section to the assistant's response
+    reference_section = "\n\nReferences:\n"
+    for ref, data in user_session["context"].items():
+        reference_section += f"[{ref.split('_')[1]}]: {data['url']}\n"
+
+    msg.content += reference_section
+    await msg.update()
+
+
+@cl.on_audio_chunk
+async def on_audio_chunk(chunk: cl.AudioChunk):
+    if chunk.isStart:
+        buffer = BytesIO()
+        # This is required for whisper to recognize the file type
+        buffer.name = f"input_audio.{chunk.mimeType.split('/')[1]}"
+        # Initialize the session for a new audio stream
+        cl.user_session.set("audio_buffer", buffer)
+        cl.user_session.set("audio_mime_type", chunk.mimeType)
+
+    # Write the chunks to a buffer and transcribe the whole audio at the end
+    cl.user_session.get("audio_buffer").write(chunk.data)
+
+    pass
+
+
+@cl.step(type="tool")
+async def speech_to_text(audio_file):
+    cli = Groq()
+
+    response = await client.audio.transcriptions.create(
+        model="whisper-large-v3", file=audio_file
+    )
+
+    return response.text
+
+
+@cl.on_audio_end
+async def on_audio_end(elements: list[ElementBased]):
+    # Get the audio buffer from the session
+    audio_buffer: BytesIO = cl.user_session.get("audio_buffer")
+    audio_buffer.seek(0)  # Move the file pointer to the beginning
+    audio_file = audio_buffer.read()
+    audio_mime_type: str = cl.user_session.get("audio_mime_type")
+
+    start_time = time.time()
+    whisper_input = (audio_buffer.name, audio_file, audio_mime_type)
+    transcription = await speech_to_text(whisper_input)
+    end_time = time.time()
+    print(f"Transcription took {end_time - start_time} seconds")
+
+    user_msg = cl.Message(author="You", type="user_message", content=transcription)
+    await user_msg.send()
+    await on_message(user_msg)
+
+
+if __name__ == "__main__":
+    from chainlit.cli import run_chainlit
+
+    run_chainlit(__file__)
diff --git a/docs/examples/rest_call.py b/docs/examples/rest_call.py
new file mode 100644
index 0000000..47c0943
--- /dev/null
+++ b/docs/examples/rest_call.py
@@ -0,0 +1,54 @@
+import requests, base64, os
+
+data = {
+    "urls": ["https://www.nbcnews.com/business"],
+    "screenshot": True,
+}
+
+response = requests.post("https://crawl4ai.com/crawl", json=data)
+result = response.json()["results"][0]
+print(result.keys())
+# dict_keys(['url', 'html', 'success', 'cleaned_html', 'media',
+# 'links', 'screenshot', 'markdown', 'extracted_content',
+# 'metadata', 'error_message'])
+with open("screenshot.png", "wb") as f:
+    f.write(base64.b64decode(result["screenshot"]))
+
+# Example of filtering the content using CSS selectors
+data = {
+    "urls": ["https://www.nbcnews.com/business"],
+    "css_selector": "article",
+    "screenshot": True,
+}
+
+# Example of executing a JS script on the page before extracting the content
+data = {
+    "urls": ["https://www.nbcnews.com/business"],
+    "screenshot": True,
+    "js": [
+        """
+    const loadMoreButton = Array.from(document.querySelectorAll('button')).
+    find(button => button.textContent.includes('Load More'));
+    loadMoreButton && loadMoreButton.click();
+    """
+    ],
+}
+
+# Example of using a custom extraction strategy
+data = {
+    "urls": ["https://www.nbcnews.com/business"],
+    "extraction_strategy": "CosineStrategy",
+    "extraction_strategy_args": {"semantic_filter": "inflation rent prices"},
+}
+
+# Example of using LLM to extract content
+data = {
+    "urls": ["https://www.nbcnews.com/business"],
+    "extraction_strategy": "LLMExtractionStrategy",
+    "extraction_strategy_args": {
+        "provider": "groq/llama3-8b-8192",
+        "api_token": os.environ.get("GROQ_API_KEY"),
+        "instruction": """I am interested in only financial news, 
+        and translate them in French.""",
+    },
+}
diff --git a/docs/examples/sample_ecommerce.html b/docs/examples/sample_ecommerce.html
new file mode 100644
index 0000000..4698d9c
--- /dev/null
+++ b/docs/examples/sample_ecommerce.html
@@ -0,0 +1,106 @@
+
+
+
+    
+    
+    Sample E-commerce Page for JsonCssExtractionStrategy Testing
+    
+
+
+    
Sample E-commerce Product Catalog

+    

+
+    
+
+
\ No newline at end of file
diff --git a/docs/examples/scraping_strategies_performance.py b/docs/examples/scraping_strategies_performance.py
new file mode 100644
index 0000000..72e1915
--- /dev/null
+++ b/docs/examples/scraping_strategies_performance.py
@@ -0,0 +1,136 @@
+import time, re
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+# WebScrapingStrategy is now an alias for LXMLWebScrapingStrategy
+import time
+import functools
+from collections import defaultdict
+
+class TimingStats:
+    def __init__(self):
+        self.stats = defaultdict(lambda: defaultdict(lambda: {"calls": 0, "total_time": 0}))
+        
+    def add(self, strategy_name, func_name, elapsed):
+        self.stats[strategy_name][func_name]["calls"] += 1
+        self.stats[strategy_name][func_name]["total_time"] += elapsed
+        
+    def report(self):
+        for strategy_name, funcs in self.stats.items():
+            print(f"\n{strategy_name} Timing Breakdown:")
+            print("-" * 60)
+            print(f"{'Function':<30} {'Calls':<10} {'Total(s)':<10} {'Avg(ms)':<10}")
+            print("-" * 60)
+            
+            for func, data in sorted(funcs.items(), key=lambda x: x[1]["total_time"], reverse=True):
+                avg_ms = (data["total_time"] / data["calls"]) * 1000
+                print(f"{func:<30} {data['calls']:<10} {data['total_time']:<10.3f} {avg_ms:<10.2f}")
+
+timing_stats = TimingStats()
+
+# Modify timing decorator
+def timing_decorator(strategy_name):
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            start = time.time()
+            result = func(*args, **kwargs)
+            elapsed = time.time() - start
+            timing_stats.add(strategy_name, func.__name__, elapsed)
+            return result
+        return wrapper
+    return decorator
+
+# Modified decorator application
+def apply_decorators(cls, method_name, strategy_name):
+    try:
+        original_method = getattr(cls, method_name)
+        decorated_method = timing_decorator(strategy_name)(original_method)
+        setattr(cls, method_name, decorated_method)
+    except AttributeError:
+        print(f"Method {method_name} not found in class {cls.__name__}.")
+
+# Apply to key methods
+methods_to_profile = [
+    '_scrap',
+    # 'process_element', 
+    '_process_element', 
+    'process_image',
+]
+
+
+# Apply decorators to both strategies
+for strategy, name in [(LXMLWebScrapingStrategy, "LXML")]:
+    for method in methods_to_profile:
+        apply_decorators(strategy, method, name)
+
+
+def generate_large_html(n_elements=1000):
+    html = ['']
+    for i in range(n_elements):
+        html.append(f'''
+            

+                
Heading {i}

+                

+                    

+                        
This is paragraph {i} with some content and a link

+                    

+                

+                Image {i}
+                
    
+                    
  • List item {i}.1
    • 
+                    
  • List item {i}.2
    • 
+                

+            

+        ''')
+    html.append('')
+    return ''.join(html)
+
+def test_scraping():
+    # Initialize both scrapers
+    original_scraper = LXMLWebScrapingStrategy()
+    selected_scraper = LXMLWebScrapingStrategy()
+    
+    # Generate test HTML
+    print("Generating HTML...")
+    html = generate_large_html(5000)
+    print(f"HTML Size: {len(html)/1024:.2f} KB")
+    
+    # Time the scraping
+    print("\nStarting scrape...")
+    start_time = time.time()
+    
+    kwargs = {
+        "url": "http://example.com",
+        "html": html,
+        "word_count_threshold": 5,
+        "keep_data_attributes": True
+    }
+    
+    t1 = time.perf_counter()
+    result_selected = selected_scraper.scrap(**kwargs)
+    t2 = time.perf_counter()
+    
+    result_original = original_scraper.scrap(**kwargs)
+    t3 = time.perf_counter()
+    
+    elapsed = t3 - start_time
+    print(f"\nScraping completed in {elapsed:.2f} seconds")
+    
+    timing_stats.report()
+    
+    # Print stats of LXML output
+    print("\Turbo Output:")
+    print(f"\nExtracted links: {len(result_selected.links.internal) + len(result_selected.links.external)}")
+    print(f"Extracted images: {len(result_selected.media.images)}")
+    print(f"Clean HTML size: {len(result_selected.cleaned_html)/1024:.2f} KB")
+    print(f"Scraping time: {t2 - t1:.2f} seconds")
+
+    # Print stats of original output
+    print("\nOriginal Output:")
+    print(f"\nExtracted links: {len(result_original.links.internal) + len(result_original.links.external)}")
+    print(f"Extracted images: {len(result_original.media.images)}")
+    print(f"Clean HTML size: {len(result_original.cleaned_html)/1024:.2f} KB")
+    print(f"Scraping time: {t3 - t1:.2f} seconds")
+        
+        
+if __name__ == "__main__":
+    test_scraping()
\ No newline at end of file
diff --git a/docs/examples/serp_api_project_11_feb.py b/docs/examples/serp_api_project_11_feb.py
new file mode 100644
index 0000000..8f9312f
--- /dev/null
+++ b/docs/examples/serp_api_project_11_feb.py
@@ -0,0 +1,308 @@
+import asyncio
+import json
+from typing import Any, Dict, List, Optional
+
+from regex import P
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    LLMConfig,
+    LLMExtractionStrategy,
+    JsonCssExtractionStrategy,
+    CrawlerHub,
+    CrawlResult,
+    DefaultMarkdownGenerator,
+    PruningContentFilter,
+)
+from pathlib import Path
+from pydantic import BaseModel
+
+__current_dir = Path(__file__).parent
+
+# Crawl4ai Hello Web
+async def little_hello_web():
+    async with AsyncWebCrawler() as crawler:
+        result : CrawlResult = await crawler.arun(
+            url="https://www.helloworld.org"
+        )
+        print(result.markdown.raw_markdown[:500])
+
+async def hello_web():
+    browser_config = BrowserConfig(headless=True, verbose=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        crawler_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48, threshold_type="fixed", min_word_threshold=0
+                )
+            ),        
+        )
+        result : CrawlResult = await crawler.arun(
+            url="https://www.helloworld.org", config=crawler_config
+        )
+        print(result.markdown.fit_markdown[:500])
+
+# Naive Approach Using Large Language Models
+async def extract_using_llm():
+    print("Extracting using Large Language Models")
+
+    browser_config = BrowserConfig(headless=True, verbose=True)
+    crawler = AsyncWebCrawler(config=browser_config) 
+
+    await crawler.start()
+    try:
+        class Sitelink(BaseModel):
+            title: str
+            link: str
+
+        class GoogleSearchResult(BaseModel):
+            title: str
+            link: str
+            snippet: str
+            sitelinks: Optional[List[Sitelink]] = None        
+
+    llm_extraction_strategy = LLMExtractionStrategy(
+            llm_config = LLMConfig(
+                provider = "openai/gpt-4o",
+            ),
+            schema = GoogleSearchResult.model_json_schema(),
+            instruction="""I want to extract the title, link, snippet, and sitelinks from a Google search result. I shared here the content of div#search from the search result page. We are just interested in organic search results.
+            Example: 
+            {
+                "title": "Google",
+                "link": "https://www.google.com",
+                "snippet": "Google is a search engine.",
+                "sitelinks": [
+                    {
+                        "title": "Gmail",
+                        "link": "https://mail.google.com"
+                    },
+                    {
+                        "title": "Google Drive",
+                        "link": "https://drive.google.com"
+                    }
+                ]
+            }""",
+            # apply_chunking=False,
+            chunk_token_threshold=2 ** 12, # 2^12 = 4096
+            verbose=True,
+            # input_format="html", # html, markdown, cleaned_html
+            input_format="cleaned_html"
+        )
+
+
+        crawl_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            keep_attrs=["id", "class"],
+            keep_data_attributes=True,
+            delay_before_return_html=2,
+            extraction_strategy=llm_extraction_strategy,
+            css_selector="div#search",
+        )
+
+        result : CrawlResult = await crawler.arun(
+            url="https://www.google.com/search?q=apple%20inc&start=0&num=10",
+            config=crawl_config,
+        )
+    
+        search_result = {}
+        if result.success:
+            search_result = json.loads(result.extracted_content)
+
+            # save search result to file
+            with open(__current_dir / "search_result_using_llm.json", "w") as f:
+                f.write(json.dumps(search_result, indent=4))
+            print(json.dumps(search_result, indent=4)) 
+
+    finally:
+        await crawler.close()
+
+# Example of using CrawlerHub
+async def schema_generator():
+    print("Generating schema")
+    html = ""
+
+    # Load html from file
+    with open(__current_dir / "google_search_item.html", "r") as f:
+        html = f.read()
+    
+    organic_schema = JsonCssExtractionStrategy.generate_schema(
+            html=html,
+            target_json_example="""{
+                "title": "...",
+                "link": "...",
+                "snippet": "...",
+                "date": "1 hour ago",
+                "sitelinks": [
+                    {
+                        "title": "...",
+                        "link": "..."
+                    }
+                ]
+            }""",
+            query="""The given HTML is the crawled HTML from the Google search result, which refers to one HTML element representing one organic Google search result. Please find the schema for the organic search item based on the given HTML. I am interested in the title, link, snippet text, sitelinks, and date.""",
+        )
+    
+    print(json.dumps(organic_schema, indent=4))    
+    pass
+
+# Golden Standard
+async def build_schema(html:str, force: bool = False) -> Dict[str, Any]:
+    print("Building schema")
+    schemas = {}
+    if (__current_dir / "organic_schema.json").exists() and not force:
+        with open(__current_dir / "organic_schema.json", "r") as f:
+            schemas["organic"] = json.loads(f.read())
+    else:        
+        # Extract schema from html
+        organic_schema = JsonCssExtractionStrategy.generate_schema(
+            html=html,
+            target_json_example="""{
+                "title": "...",
+                "link": "...",
+                "snippet": "...",
+                "date": "1 hour ago",
+                "sitelinks": [
+                    {
+                        "title": "...",
+                        "link": "..."
+                    }
+                ]
+            }""",
+            query="""The given html is the crawled html from Google search result. Please find the schema for organic search item in the given html, I am interested in title, link, snippet text, sitelinks and date. Usually they are all inside a div#search.""",
+        )
+
+        # Save schema to file current_dir/organic_schema.json
+        with open(__current_dir / "organic_schema.json", "w") as f:
+            f.write(json.dumps(organic_schema, indent=4))
+        
+        schemas["organic"] = organic_schema    
+
+    # Repeat the same for top_stories_schema
+    if (__current_dir / "top_stories_schema.json").exists():
+        with open(__current_dir / "top_stories_schema.json", "r") as f:
+            schemas["top_stories"] = json.loads(f.read())
+    else:
+        top_stories_schema = JsonCssExtractionStrategy.generate_schema(
+            html=html,
+            target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "source": "Insider Monkey",
+            "date": "1 hour ago",
+        }""",
+            query="""The given HTML is the crawled HTML from the Google search result. Please find the schema for the Top Stories item in the given HTML. I am interested in the title, link, source, and date.""",
+        )
+
+        with open(__current_dir / "top_stories_schema.json", "w") as f:
+            f.write(json.dumps(top_stories_schema, indent=4))
+        
+        schemas["top_stories"] = top_stories_schema
+
+    # Repeat the same for suggested_queries_schema
+    if (__current_dir / "suggested_queries_schema.json").exists():
+        with open(__current_dir / "suggested_queries_schema.json", "r") as f:
+            schemas["suggested_queries"] = json.loads(f.read())
+    else:
+        suggested_queries_schema = JsonCssExtractionStrategy.generate_schema(
+            html=html,
+            target_json_example="""{
+            "query": "A for Apple",
+        }""",
+            query="""The given HTML contains the crawled HTML from Google search results. Please find the schema for each suggested query in the section "relatedSearches" at the bottom of the page. I am interested in the queries only.""",
+        )
+
+        with open(__current_dir / "suggested_queries_schema.json", "w") as f:
+            f.write(json.dumps(suggested_queries_schema, indent=4))
+        
+        schemas["suggested_queries"] = suggested_queries_schema
+    
+    return schemas
+
+async def search(q: str = "apple inc") -> Dict[str, Any]:
+    print("Searching for:", q)
+
+    browser_config = BrowserConfig(headless=True, verbose=True)
+    crawler = AsyncWebCrawler(config=browser_config)
+    search_result: Dict[str, List[Dict[str, Any]]] = {} 
+
+    await crawler.start()
+    try:
+        crawl_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            keep_attrs=["id", "class"],
+            keep_data_attributes=True,
+            delay_before_return_html=2,
+        )
+        from urllib.parse import quote
+        result: CrawlResult = await crawler.arun(
+            f"https://www.google.com/search?q={quote(q)}&start=0&num=10",
+            config=crawl_config
+        )
+
+        if result.success:
+            schemas : Dict[str, Any] = await build_schema(result.html)
+
+            for schema in schemas.values():
+                schema_key = schema["name"].lower().replace(' ', '_')
+                search_result[schema_key] = JsonCssExtractionStrategy(
+                    schema=schema
+                ).run(
+                    url="",
+                    sections=[result.html],
+                )
+
+            # save search result to file
+            with open(__current_dir / "search_result.json", "w") as f:
+                f.write(json.dumps(search_result, indent=4))
+            print(json.dumps(search_result, indent=4))        
+
+    finally:
+        await crawler.close()
+
+    return search_result
+
+# Example of using CrawlerHub
+async def hub_example(query: str = "apple inc"):
+    print("Using CrawlerHub")
+    crawler_cls = CrawlerHub.get("google_search")
+    crawler = crawler_cls()
+
+    # Text search
+    text_results = await crawler.run(
+        query=query,
+        search_type="text",  
+        schema_cache_path="/Users/unclecode/.crawl4ai"
+    )
+    # Save search result to file
+    with open(__current_dir / "search_result_using_hub.json", "w") as f:
+        f.write(json.dumps(json.loads(text_results), indent=4))
+
+    print(json.dumps(json.loads(text_results), indent=4))
+
+
+async def demo():
+    # Step 1: Introduction & Overview 
+    # await little_hello_web()
+    # await hello_web()
+
+    # Step 2: Demo end result, using hub
+    # await hub_example()
+
+    # Step 3: Using LLm for extraction
+    # await extract_using_llm()
+
+    # Step 4: GEt familiar with schema generation
+    # await schema_generator()
+
+    # Step 5: Golden Standard
+    # await search()
+
+    # Step 6: Introduction to CrawlerHub
+    await hub_example()
+
+if __name__ == "__main__":
+    asyncio.run(demo())
diff --git a/docs/examples/session_id_example.py b/docs/examples/session_id_example.py
new file mode 100644
index 0000000..e49b781
--- /dev/null
+++ b/docs/examples/session_id_example.py
@@ -0,0 +1,38 @@
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    DefaultMarkdownGenerator,
+    PruningContentFilter,
+    CrawlResult
+)
+
+                   
+
+async def main():    
+    browser_config = BrowserConfig(
+        headless=False, 
+        verbose=True,
+    )
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        crawler_config = CrawlerRunConfig(
+            session_id= "hello_world", # This help us to use the same page 
+        )
+        result : CrawlResult = await crawler.arun(
+            url="https://www.helloworld.org", config=crawler_config
+        )
+        # Add a breakpoint here, then you will the page is open and browser is not closed
+        print(result.markdown.raw_markdown[:500])
+        
+        new_config = crawler_config.clone(js_code=["(() => ({'data':'hello'}))()"], js_only=True)
+        result : CrawlResult = await crawler.arun( # This time there is no fetch and this only executes JS in the same opened page
+            url="https://www.helloworld.org", config= new_config
+        )
+        print(result.js_execution_result) # You should see {'data':'hello'} in the console
+        
+        # Get direct access to Playwright paege object. This works only if you use the same session_id and pass same config 
+        page, context = crawler.crawler_strategy.get_page(new_config)
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/shadow_dom_crawling.py b/docs/examples/shadow_dom_crawling.py
new file mode 100644
index 0000000..49b098b
--- /dev/null
+++ b/docs/examples/shadow_dom_crawling.py
@@ -0,0 +1,77 @@
+"""
+Shadow DOM Crawling Example
+============================
+
+Demonstrates how to use `flatten_shadow_dom=True` to extract content
+hidden inside Shadow DOM trees on sites built with Web Components
+(Stencil, Lit, Shoelace, Angular Elements, etc.).
+
+Shadow DOM creates encapsulated sub-trees that are invisible to the
+normal page serialization (page.content() / outerHTML). The
+`flatten_shadow_dom` option walks these trees and produces a single
+flat HTML document that includes all shadow content.
+
+This example crawls a Bosch Rexroth product page where the product
+description, technical specs, and downloads are rendered entirely
+inside Shadow DOM by Stencil.js web components.
+"""
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+URL = "https://store.boschrexroth.com/en/us/p/hydraulic-cylinder-r900999011"
+
+
+async def main():
+    browser_config = BrowserConfig(headless=True)
+
+    # ── 1. Baseline: without shadow DOM flattening ──────────────────
+    print("=" * 60)
+    print("Without flatten_shadow_dom (baseline)")
+    print("=" * 60)
+
+    config = CrawlerRunConfig(
+        wait_until="load",
+        delay_before_return_html=3.0,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(URL, config=config)
+
+    md = result.markdown.raw_markdown if result.markdown else ""
+    print(f"Markdown length: {len(md)} chars")
+    print(f"Has product description: {'mill type design' in md.lower()}")
+    print(f"Has technical specs:     {'CDH1' in md}")
+    print(f"Has downloads section:   {'Downloads' in md}")
+    print()
+
+    # ── 2. With shadow DOM flattening ───────────────────────────────
+    print("=" * 60)
+    print("With flatten_shadow_dom=True")
+    print("=" * 60)
+
+    config = CrawlerRunConfig(
+        wait_until="load",
+        delay_before_return_html=3.0,
+        flatten_shadow_dom=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(URL, config=config)
+
+    md = result.markdown.raw_markdown if result.markdown else ""
+    print(f"Markdown length: {len(md)} chars")
+    print(f"Has product description: {'mill type design' in md.lower()}")
+    print(f"Has technical specs:     {'CDH1' in md}")
+    print(f"Has downloads section:   {'Downloads' in md}")
+    print()
+
+    # Show the product content section
+    idx = md.find("Product Description")
+    if idx >= 0:
+        print("── Extracted product content ──")
+        print(md[idx:idx + 1200])
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/simple_anti_bot_examples.py b/docs/examples/simple_anti_bot_examples.py
new file mode 100644
index 0000000..075ee9a
--- /dev/null
+++ b/docs/examples/simple_anti_bot_examples.py
@@ -0,0 +1,59 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, UndetectedAdapter
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+# Example 1: Stealth Mode
+async def stealth_mode_example():
+    browser_config = BrowserConfig(
+        enable_stealth=True,
+        headless=False
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun("https://example.com")
+        return result.html[:500]
+
+# Example 2: Undetected Browser
+async def undetected_browser_example():
+    browser_config = BrowserConfig(
+        headless=False
+    )
+    
+    adapter = UndetectedAdapter()
+    strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=adapter
+    )
+    
+    async with AsyncWebCrawler(
+        crawler_strategy=strategy,
+        config=browser_config
+    ) as crawler:
+        result = await crawler.arun("https://example.com")
+        return result.html[:500]
+
+# Example 3: Both Combined
+async def combined_example():
+    browser_config = BrowserConfig(
+        enable_stealth=True,
+        headless=False
+    )
+    
+    adapter = UndetectedAdapter()
+    strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=adapter
+    )
+    
+    async with AsyncWebCrawler(
+        crawler_strategy=strategy,
+        config=browser_config
+    ) as crawler:
+        result = await crawler.arun("https://example.com")
+        return result.html[:500]
+
+# Run examples
+if __name__ == "__main__":
+    asyncio.run(stealth_mode_example())
+    asyncio.run(undetected_browser_example())
+    asyncio.run(combined_example())
\ No newline at end of file
diff --git a/docs/examples/ssl_example.py b/docs/examples/ssl_example.py
new file mode 100644
index 0000000..7379862
--- /dev/null
+++ b/docs/examples/ssl_example.py
@@ -0,0 +1,51 @@
+"""Example showing how to work with SSL certificates in Crawl4AI."""
+
+import asyncio
+import os
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+
+# Create tmp directory if it doesn't exist
+parent_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+tmp_dir = os.path.join(parent_dir, "tmp")
+os.makedirs(tmp_dir, exist_ok=True)
+
+
+async def main():
+    # Configure crawler to fetch SSL certificate
+    config = CrawlerRunConfig(
+        fetch_ssl_certificate=True,
+        cache_mode=CacheMode.BYPASS,  # Bypass cache to always get fresh certificates
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url="https://example.com", config=config)
+
+        if result.success and result.ssl_certificate:
+            cert = result.ssl_certificate
+
+            # 1. Access certificate properties directly
+            print("\nCertificate Information:")
+            print(f"Issuer: {cert.issuer.get('CN', '')}")
+            print(f"Valid until: {cert.valid_until}")
+            print(f"Fingerprint: {cert.fingerprint}")
+
+            # 2. Export certificate in different formats
+            cert.to_json(os.path.join(tmp_dir, "certificate.json"))  # For analysis
+            print("\nCertificate exported to:")
+            print(f"- JSON: {os.path.join(tmp_dir, 'certificate.json')}")
+
+            pem_data = cert.to_pem(
+                os.path.join(tmp_dir, "certificate.pem")
+            )  # For web servers
+            print(f"- PEM: {os.path.join(tmp_dir, 'certificate.pem')}")
+
+            der_data = cert.to_der(
+                os.path.join(tmp_dir, "certificate.der")
+            )  # For Java apps
+            print(f"- DER: {os.path.join(tmp_dir, 'certificate.der')}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/stealth_mode_example.py b/docs/examples/stealth_mode_example.py
new file mode 100644
index 0000000..baf735d
--- /dev/null
+++ b/docs/examples/stealth_mode_example.py
@@ -0,0 +1,522 @@
+"""
+Stealth Mode Example with Crawl4AI
+
+This example demonstrates how to use the stealth mode feature to bypass basic bot detection.
+The stealth mode uses playwright-stealth to modify browser fingerprints and behaviors
+that are commonly used to detect automated browsers.
+
+Key features demonstrated:
+1. Comparing crawling with and without stealth mode
+2. Testing against bot detection sites
+3. Accessing sites that block automated browsers
+4. Best practices for stealth crawling
+"""
+
+import asyncio
+import json
+from typing import Dict, Any
+from colorama import Fore, Style, init
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Initialize colorama for colored output
+init()
+
+# Create a logger for better output
+logger = AsyncLogger(verbose=True)
+
+
+async def test_bot_detection(use_stealth: bool = False) -> Dict[str, Any]:
+    """Test against a bot detection service"""
+    
+    logger.info(
+        f"Testing bot detection with stealth={'ON' if use_stealth else 'OFF'}",
+        tag="STEALTH"
+    )
+    
+    # Configure browser with or without stealth
+    browser_config = BrowserConfig(
+        headless=False,  # Use False to see the browser in action
+        enable_stealth=use_stealth,
+        viewport_width=1280,
+        viewport_height=800
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # JavaScript to extract bot detection results
+        detection_script = """
+        // Comprehensive bot detection checks
+        (() => {
+        const detectionResults = {
+            // Basic WebDriver detection
+            webdriver: navigator.webdriver,
+            
+            // Chrome specific
+            chrome: !!window.chrome,
+            chromeRuntime: !!window.chrome?.runtime,
+            
+            // Automation indicators
+            automationControlled: navigator.webdriver,
+            
+            // Permissions API
+            permissionsPresent: !!navigator.permissions?.query,
+            
+            // Plugins
+            pluginsLength: navigator.plugins.length,
+            pluginsArray: Array.from(navigator.plugins).map(p => p.name),
+            
+            // Languages
+            languages: navigator.languages,
+            language: navigator.language,
+            
+            // User agent
+            userAgent: navigator.userAgent,
+            
+            // Screen and window properties
+            screen: {
+                width: screen.width,
+                height: screen.height,
+                availWidth: screen.availWidth,
+                availHeight: screen.availHeight,
+                colorDepth: screen.colorDepth,
+                pixelDepth: screen.pixelDepth
+            },
+            
+            // WebGL vendor
+            webglVendor: (() => {
+                try {
+                    const canvas = document.createElement('canvas');
+                    const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+                    const ext = gl.getExtension('WEBGL_debug_renderer_info');
+                    return gl.getParameter(ext.UNMASKED_VENDOR_WEBGL);
+                } catch (e) {
+                    return 'Error';
+                }
+            })(),
+            
+            // Platform
+            platform: navigator.platform,
+            
+            // Hardware concurrency
+            hardwareConcurrency: navigator.hardwareConcurrency,
+            
+            // Device memory
+            deviceMemory: navigator.deviceMemory,
+            
+            // Connection
+            connection: navigator.connection?.effectiveType
+        };
+        
+        // Log results for console capture
+        console.log('DETECTION_RESULTS:', JSON.stringify(detectionResults, null, 2));
+        
+        // Return results
+        return detectionResults;
+        })();
+        """
+        
+        # Crawl bot detection test page
+        config = CrawlerRunConfig(
+            js_code=detection_script,
+            capture_console_messages=True,
+            wait_until="networkidle",
+            delay_before_return_html=2.0  # Give time for all checks to complete
+        )
+        
+        result = await crawler.arun(
+            url="https://bot.sannysoft.com",
+            config=config
+        )
+        
+        if result.success:
+            # Extract detection results from console
+            detection_data = None
+            for msg in result.console_messages or []:
+                if "DETECTION_RESULTS:" in msg.get("text", ""):
+                    try:
+                        json_str = msg["text"].replace("DETECTION_RESULTS:", "").strip()
+                        detection_data = json.loads(json_str)
+                    except:
+                        pass
+            
+            # Also try to get from JavaScript execution result
+            if not detection_data and result.js_execution_result:
+                detection_data = result.js_execution_result
+            
+            return {
+                "success": True,
+                "url": result.url,
+                "detection_data": detection_data,
+                "page_title": result.metadata.get("title", ""),
+                "stealth_enabled": use_stealth
+            }
+        else:
+            return {
+                "success": False,
+                "error": result.error_message,
+                "stealth_enabled": use_stealth
+            }
+
+
+async def test_cloudflare_site(use_stealth: bool = False) -> Dict[str, Any]:
+    """Test accessing a Cloudflare-protected site"""
+    
+    logger.info(
+        f"Testing Cloudflare site with stealth={'ON' if use_stealth else 'OFF'}",
+        tag="STEALTH"
+    )
+    
+    browser_config = BrowserConfig(
+        headless=True,  # Cloudflare detection works better in headless mode with stealth
+        enable_stealth=use_stealth,
+        viewport_width=1920,
+        viewport_height=1080
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        config = CrawlerRunConfig(
+            wait_until="networkidle",
+            page_timeout=30000,  # 30 seconds
+            delay_before_return_html=3.0
+        )
+        
+        # Test on a site that often shows Cloudflare challenges
+        result = await crawler.arun(
+            url="https://nowsecure.nl",
+            config=config
+        )
+        
+        # Check if we hit Cloudflare challenge
+        cloudflare_detected = False
+        if result.html:
+            cloudflare_indicators = [
+                "Checking your browser",
+                "Just a moment",
+                "cf-browser-verification",
+                "cf-challenge",
+                "ray ID"
+            ]
+            cloudflare_detected = any(indicator in result.html for indicator in cloudflare_indicators)
+        
+        return {
+            "success": result.success,
+            "url": result.url,
+            "cloudflare_challenge": cloudflare_detected,
+            "status_code": result.status_code,
+            "page_title": result.metadata.get("title", "") if result.metadata else "",
+            "stealth_enabled": use_stealth,
+            "html_snippet": result.html[:500] if result.html else ""
+        }
+
+
+async def test_anti_bot_site(use_stealth: bool = False) -> Dict[str, Any]:
+    """Test against sites with anti-bot measures"""
+    
+    logger.info(
+        f"Testing anti-bot site with stealth={'ON' if use_stealth else 'OFF'}",
+        tag="STEALTH"
+    )
+    
+    browser_config = BrowserConfig(
+        headless=False,
+        enable_stealth=use_stealth,
+        # Additional browser arguments that help with stealth
+        extra_args=[
+            "--disable-blink-features=AutomationControlled",
+            "--disable-features=site-per-process"
+        ] if not use_stealth else []  # These are automatically applied with stealth
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Some sites check for specific behaviors
+        behavior_script = """
+        (async () => {
+            // Simulate human-like behavior
+            const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+            
+            // Random mouse movement
+            const moveX = Math.random() * 100;
+            const moveY = Math.random() * 100;
+            
+            // Simulate reading time
+            await sleep(1000 + Math.random() * 2000);
+            
+            // Scroll slightly
+            window.scrollBy(0, 100 + Math.random() * 200);
+            
+            console.log('Human behavior simulation complete');
+            return true;
+        })()
+        """
+        
+        config = CrawlerRunConfig(
+            js_code=behavior_script,
+            wait_until="networkidle",
+            delay_before_return_html=5.0,  # Longer delay to appear more human
+            capture_console_messages=True
+        )
+        
+        # Test on a site that implements anti-bot measures
+        result = await crawler.arun(
+            url="https://www.g2.com/",
+            config=config
+        )
+        
+        # Check for common anti-bot blocks
+        blocked_indicators = [
+            "Access Denied",
+            "403 Forbidden", 
+            "Security Check",
+            "Verify you are human",
+            "captcha",
+            "challenge"
+        ]
+        
+        blocked = False
+        if result.html:
+            blocked = any(indicator.lower() in result.html.lower() for indicator in blocked_indicators)
+        
+        return {
+            "success": result.success and not blocked,
+            "url": result.url,
+            "blocked": blocked,
+            "status_code": result.status_code,
+            "page_title": result.metadata.get("title", "") if result.metadata else "",
+            "stealth_enabled": use_stealth
+        }
+
+
+async def compare_results():
+    """Run all tests with and without stealth mode and compare results"""
+    
+    print(f"\n{Fore.CYAN}{'='*60}{Style.RESET_ALL}")
+    print(f"{Fore.CYAN}Crawl4AI Stealth Mode Comparison{Style.RESET_ALL}")
+    print(f"{Fore.CYAN}{'='*60}{Style.RESET_ALL}\n")
+    
+    # Test 1: Bot Detection
+    print(f"{Fore.YELLOW}1. Bot Detection Test (bot.sannysoft.com){Style.RESET_ALL}")
+    print("-" * 40)
+    
+    # Without stealth
+    regular_detection = await test_bot_detection(use_stealth=False)
+    if regular_detection["success"] and regular_detection["detection_data"]:
+        print(f"{Fore.RED}Without Stealth:{Style.RESET_ALL}")
+        data = regular_detection["detection_data"]
+        print(f"  • WebDriver detected: {data.get('webdriver', 'Unknown')}")
+        print(f"  • Chrome: {data.get('chrome', 'Unknown')}")
+        print(f"  • Languages: {data.get('languages', 'Unknown')}")
+        print(f"  • Plugins: {data.get('pluginsLength', 'Unknown')}")
+        print(f"  • User Agent: {data.get('userAgent', 'Unknown')[:60]}...")
+    
+    # With stealth
+    stealth_detection = await test_bot_detection(use_stealth=True)
+    if stealth_detection["success"] and stealth_detection["detection_data"]:
+        print(f"\n{Fore.GREEN}With Stealth:{Style.RESET_ALL}")
+        data = stealth_detection["detection_data"]
+        print(f"  • WebDriver detected: {data.get('webdriver', 'Unknown')}")
+        print(f"  • Chrome: {data.get('chrome', 'Unknown')}")
+        print(f"  • Languages: {data.get('languages', 'Unknown')}")
+        print(f"  • Plugins: {data.get('pluginsLength', 'Unknown')}")
+        print(f"  • User Agent: {data.get('userAgent', 'Unknown')[:60]}...")
+    
+    # Test 2: Cloudflare Site
+    print(f"\n\n{Fore.YELLOW}2. Cloudflare Protected Site Test{Style.RESET_ALL}")
+    print("-" * 40)
+    
+    # Without stealth
+    regular_cf = await test_cloudflare_site(use_stealth=False)
+    print(f"{Fore.RED}Without Stealth:{Style.RESET_ALL}")
+    print(f"  • Success: {regular_cf['success']}")
+    print(f"  • Cloudflare Challenge: {regular_cf['cloudflare_challenge']}")
+    print(f"  • Status Code: {regular_cf['status_code']}")
+    print(f"  • Page Title: {regular_cf['page_title']}")
+    
+    # With stealth
+    stealth_cf = await test_cloudflare_site(use_stealth=True)
+    print(f"\n{Fore.GREEN}With Stealth:{Style.RESET_ALL}")
+    print(f"  • Success: {stealth_cf['success']}")
+    print(f"  • Cloudflare Challenge: {stealth_cf['cloudflare_challenge']}")
+    print(f"  • Status Code: {stealth_cf['status_code']}")
+    print(f"  • Page Title: {stealth_cf['page_title']}")
+    
+    # Test 3: Anti-bot Site
+    print(f"\n\n{Fore.YELLOW}3. Anti-Bot Site Test{Style.RESET_ALL}")
+    print("-" * 40)
+    
+    # Without stealth
+    regular_antibot = await test_anti_bot_site(use_stealth=False)
+    print(f"{Fore.RED}Without Stealth:{Style.RESET_ALL}")
+    print(f"  • Success: {regular_antibot['success']}")
+    print(f"  • Blocked: {regular_antibot['blocked']}")
+    print(f"  • Status Code: {regular_antibot['status_code']}")
+    print(f"  • Page Title: {regular_antibot['page_title']}")
+    
+    # With stealth
+    stealth_antibot = await test_anti_bot_site(use_stealth=True)
+    print(f"\n{Fore.GREEN}With Stealth:{Style.RESET_ALL}")
+    print(f"  • Success: {stealth_antibot['success']}")
+    print(f"  • Blocked: {stealth_antibot['blocked']}")
+    print(f"  • Status Code: {stealth_antibot['status_code']}")
+    print(f"  • Page Title: {stealth_antibot['page_title']}")
+    
+    # Summary
+    print(f"\n{Fore.CYAN}{'='*60}{Style.RESET_ALL}")
+    print(f"{Fore.CYAN}Summary:{Style.RESET_ALL}")
+    print(f"{Fore.CYAN}{'='*60}{Style.RESET_ALL}")
+    print(f"\nStealth mode helps bypass basic bot detection by:")
+    print(f"  • Hiding webdriver property")
+    print(f"  • Modifying browser fingerprints")
+    print(f"  • Adjusting navigator properties")
+    print(f"  • Emulating real browser plugin behavior")
+    print(f"\n{Fore.YELLOW}Note:{Style.RESET_ALL} Stealth mode is not a silver bullet.")
+    print(f"Advanced anti-bot systems may still detect automation.")
+    print(f"Always respect robots.txt and website terms of service.")
+
+
+async def stealth_best_practices():
+    """Demonstrate best practices for using stealth mode"""
+    
+    print(f"\n\n{Fore.CYAN}{'='*60}{Style.RESET_ALL}")
+    print(f"{Fore.CYAN}Stealth Mode Best Practices{Style.RESET_ALL}")
+    print(f"{Fore.CYAN}{'='*60}{Style.RESET_ALL}\n")
+    
+    # Best Practice 1: Combine with realistic behavior
+    print(f"{Fore.YELLOW}1. Combine with Realistic Behavior:{Style.RESET_ALL}")
+    
+    browser_config = BrowserConfig(
+        headless=False,
+        enable_stealth=True,
+        viewport_width=1920,
+        viewport_height=1080
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Simulate human-like behavior
+        human_behavior_script = """
+        (async () => {
+            // Wait random time between actions
+            const randomWait = () => Math.random() * 2000 + 1000;
+            
+            // Simulate reading
+            await new Promise(resolve => setTimeout(resolve, randomWait()));
+            
+            // Smooth scroll
+            const smoothScroll = async () => {
+                const totalHeight = document.body.scrollHeight;
+                const viewHeight = window.innerHeight;
+                let currentPosition = 0;
+                
+                while (currentPosition < totalHeight - viewHeight) {
+                    const scrollAmount = Math.random() * 300 + 100;
+                    window.scrollBy({
+                        top: scrollAmount,
+                        behavior: 'smooth'
+                    });
+                    currentPosition += scrollAmount;
+                    await new Promise(resolve => setTimeout(resolve, randomWait()));
+                }
+            };
+            
+            await smoothScroll();
+            console.log('Human-like behavior simulation completed');
+            return true;
+        })()
+        """
+        
+        config = CrawlerRunConfig(
+            js_code=human_behavior_script,
+            wait_until="networkidle",
+            delay_before_return_html=3.0,
+            capture_console_messages=True
+        )
+        
+        result = await crawler.arun(
+            url="https://example.com",
+            config=config
+        )
+        
+        print(f"  ✓ Simulated human-like scrolling and reading patterns")
+        print(f"  ✓ Added random delays between actions")
+        print(f"  ✓ Result: {result.success}")
+    
+    # Best Practice 2: Use appropriate viewport and user agent
+    print(f"\n{Fore.YELLOW}2. Use Realistic Viewport and User Agent:{Style.RESET_ALL}")
+    
+    # Get a realistic user agent
+    from crawl4ai.user_agent_generator import UserAgentGenerator
+    ua_generator = UserAgentGenerator()
+    
+    browser_config = BrowserConfig(
+        headless=True,
+        enable_stealth=True,
+        viewport_width=1920,
+        viewport_height=1080,
+        user_agent=ua_generator.generate(device_type="desktop", browser_type="chrome")
+    )
+    
+    print(f"  ✓ Using realistic viewport: 1920x1080")
+    print(f"  ✓ Using current Chrome user agent")
+    print(f"  ✓ Stealth mode will ensure consistency")
+    
+    # Best Practice 3: Manage request rate
+    print(f"\n{Fore.YELLOW}3. Manage Request Rate:{Style.RESET_ALL}")
+    print(f"  ✓ Add delays between requests")
+    print(f"  ✓ Randomize timing patterns")
+    print(f"  ✓ Respect robots.txt")
+    
+    # Best Practice 4: Session management
+    print(f"\n{Fore.YELLOW}4. Use Session Management:{Style.RESET_ALL}")
+    
+    browser_config = BrowserConfig(
+        headless=False,
+        enable_stealth=True
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Create a session for multiple requests
+        session_id = "stealth_session_1"
+        
+        config = CrawlerRunConfig(
+            session_id=session_id,
+            wait_until="domcontentloaded"
+        )
+        
+        # First request
+        result1 = await crawler.arun(
+            url="https://example.com",
+            config=config
+        )
+        
+        # Subsequent request reuses the same browser context
+        result2 = await crawler.arun(
+            url="https://example.com/about",
+            config=config
+        )
+        
+        print(f"  ✓ Reused browser session for multiple requests")
+        print(f"  ✓ Maintains cookies and state between requests")
+        print(f"  ✓ More efficient and realistic browsing pattern")
+    
+    print(f"\n{Fore.CYAN}{'='*60}{Style.RESET_ALL}")
+
+
+async def main():
+    """Run all examples"""
+    
+    # Run comparison tests
+    await compare_results()
+    
+    # Show best practices
+    await stealth_best_practices()
+    
+    print(f"\n{Fore.GREEN}Examples completed!{Style.RESET_ALL}")
+    print(f"\n{Fore.YELLOW}Remember:{Style.RESET_ALL}")
+    print(f"• Stealth mode helps with basic bot detection")
+    print(f"• Always respect website terms of service")
+    print(f"• Consider rate limiting and ethical scraping practices")
+    print(f"• For advanced protection, consider additional measures")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/stealth_mode_quick_start.py b/docs/examples/stealth_mode_quick_start.py
new file mode 100644
index 0000000..ee18913
--- /dev/null
+++ b/docs/examples/stealth_mode_quick_start.py
@@ -0,0 +1,215 @@
+"""
+Quick Start: Using Stealth Mode in Crawl4AI
+
+This example shows practical use cases for the stealth mode feature.
+Stealth mode helps bypass basic bot detection mechanisms.
+"""
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+
+async def example_1_basic_stealth():
+    """Example 1: Basic stealth mode usage"""
+    print("\n=== Example 1: Basic Stealth Mode ===")
+    
+    # Enable stealth mode in browser config
+    browser_config = BrowserConfig(
+        enable_stealth=True,  # This is the key parameter
+        headless=True
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print(f"✓ Crawled {result.url} successfully")
+        print(f"✓ Title: {result.metadata.get('title', 'N/A')}")
+
+
+async def example_2_stealth_with_screenshot():
+    """Example 2: Stealth mode with screenshot to show detection results"""
+    print("\n=== Example 2: Stealth Mode Visual Verification ===")
+    
+    browser_config = BrowserConfig(
+        enable_stealth=True,
+        headless=False  # Set to False to see the browser
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        config = CrawlerRunConfig(
+            screenshot=True,
+            wait_until="networkidle"
+        )
+        
+        result = await crawler.arun(
+            url="https://bot.sannysoft.com",
+            config=config
+        )
+        
+        if result.success:
+            print(f"✓ Successfully crawled bot detection site")
+            print(f"✓ With stealth enabled, many detection tests should show as passed")
+            
+            if result.screenshot:
+                # Save screenshot for verification
+                import base64
+                with open("stealth_detection_results.png", "wb") as f:
+                    f.write(base64.b64decode(result.screenshot))
+                print(f"✓ Screenshot saved as 'stealth_detection_results.png'")
+                print(f"  Check the screenshot to see detection results!")
+
+
+async def example_3_stealth_for_protected_sites():
+    """Example 3: Using stealth for sites with bot protection"""
+    print("\n=== Example 3: Stealth for Protected Sites ===")
+    
+    browser_config = BrowserConfig(
+        enable_stealth=True,
+        headless=True,
+        viewport_width=1920,
+        viewport_height=1080
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Add human-like behavior
+        config = CrawlerRunConfig(
+            wait_until="networkidle",
+            delay_before_return_html=2.0,  # Wait 2 seconds
+            js_code="""
+            // Simulate human-like scrolling
+            window.scrollTo({
+                top: document.body.scrollHeight / 2,
+                behavior: 'smooth'
+            });
+            """
+        )
+        
+        # Try accessing a site that might have bot protection
+        result = await crawler.arun(
+            url="https://www.g2.com/products/slack/reviews",
+            config=config
+        )
+        
+        if result.success:
+            print(f"✓ Successfully accessed protected site")
+            print(f"✓ Retrieved {len(result.html)} characters of HTML")
+        else:
+            print(f"✗ Failed to access site: {result.error_message}")
+
+
+async def example_4_stealth_with_sessions():
+    """Example 4: Stealth mode with session management"""
+    print("\n=== Example 4: Stealth + Session Management ===")
+    
+    browser_config = BrowserConfig(
+        enable_stealth=True,
+        headless=False
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        session_id = "my_stealth_session"
+        
+        # First request - establish session
+        config = CrawlerRunConfig(
+            session_id=session_id,
+            wait_until="domcontentloaded"
+        )
+        
+        result1 = await crawler.arun(
+            url="https://news.ycombinator.com",
+            config=config
+        )
+        print(f"✓ First request completed: {result1.url}")
+        
+        # Second request - reuse session
+        await asyncio.sleep(2)  # Brief delay between requests
+        
+        result2 = await crawler.arun(
+            url="https://news.ycombinator.com/best",
+            config=config
+        )
+        print(f"✓ Second request completed: {result2.url}")
+        print(f"✓ Session reused, maintaining cookies and state")
+
+
+async def example_5_stealth_comparison():
+    """Example 5: Compare results with and without stealth using screenshots"""
+    print("\n=== Example 5: Stealth Mode Comparison ===")
+    
+    test_url = "https://bot.sannysoft.com"
+    
+    # First test WITHOUT stealth
+    print("\nWithout stealth:")
+    regular_config = BrowserConfig(
+        enable_stealth=False,
+        headless=True
+    )
+    
+    async with AsyncWebCrawler(config=regular_config) as crawler:
+        config = CrawlerRunConfig(
+            screenshot=True,
+            wait_until="networkidle"
+        )
+        result = await crawler.arun(url=test_url, config=config)
+        
+        if result.success and result.screenshot:
+            import base64
+            with open("comparison_without_stealth.png", "wb") as f:
+                f.write(base64.b64decode(result.screenshot))
+            print(f"  ✓ Screenshot saved: comparison_without_stealth.png")
+            print(f"  Many tests will show as FAILED (red)")
+    
+    # Then test WITH stealth
+    print("\nWith stealth:")
+    stealth_config = BrowserConfig(
+        enable_stealth=True,
+        headless=True
+    )
+    
+    async with AsyncWebCrawler(config=stealth_config) as crawler:
+        config = CrawlerRunConfig(
+            screenshot=True,
+            wait_until="networkidle"
+        )
+        result = await crawler.arun(url=test_url, config=config)
+        
+        if result.success and result.screenshot:
+            import base64
+            with open("comparison_with_stealth.png", "wb") as f:
+                f.write(base64.b64decode(result.screenshot))
+            print(f"  ✓ Screenshot saved: comparison_with_stealth.png")
+            print(f"  More tests should show as PASSED (green)")
+    
+    print("\nCompare the two screenshots to see the difference!")
+
+
+async def main():
+    """Run all examples"""
+    print("Crawl4AI Stealth Mode Examples")
+    print("==============================")
+    
+    # Run basic example
+    await example_1_basic_stealth()
+    
+    # Run screenshot verification example
+    await example_2_stealth_with_screenshot()
+    
+    # Run protected site example
+    await example_3_stealth_for_protected_sites()
+    
+    # Run session example
+    await example_4_stealth_with_sessions()
+    
+    # Run comparison example
+    await example_5_stealth_comparison()
+    
+    print("\n" + "="*50)
+    print("Tips for using stealth mode effectively:")
+    print("- Use realistic viewport sizes (1920x1080, 1366x768)")
+    print("- Add delays between requests to appear more human")
+    print("- Combine with session management for better results")
+    print("- Remember: stealth mode is for legitimate scraping only")
+    print("="*50)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/stealth_test_simple.py b/docs/examples/stealth_test_simple.py
new file mode 100644
index 0000000..8bf9c2d
--- /dev/null
+++ b/docs/examples/stealth_test_simple.py
@@ -0,0 +1,62 @@
+"""
+Simple test to verify stealth mode is working
+"""
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+
+async def test_stealth():
+    """Test stealth mode effectiveness"""
+    
+    # Test WITHOUT stealth
+    print("=== WITHOUT Stealth ===")
+    config1 = BrowserConfig(
+        headless=False,
+        enable_stealth=False
+    )
+    
+    async with AsyncWebCrawler(config=config1) as crawler:
+        result = await crawler.arun(
+            url="https://bot.sannysoft.com",
+            config=CrawlerRunConfig(
+                wait_until="networkidle",
+                screenshot=True
+            )
+        )
+        print(f"Success: {result.success}")
+        # Take screenshot
+        if result.screenshot:
+            with open("without_stealth.png", "wb") as f:
+                import base64
+                f.write(base64.b64decode(result.screenshot))
+            print("Screenshot saved: without_stealth.png")
+    
+    # Test WITH stealth
+    print("\n=== WITH Stealth ===")
+    config2 = BrowserConfig(
+        headless=False,
+        enable_stealth=True
+    )
+    
+    async with AsyncWebCrawler(config=config2) as crawler:
+        result = await crawler.arun(
+            url="https://bot.sannysoft.com",
+            config=CrawlerRunConfig(
+                wait_until="networkidle",
+                screenshot=True
+            )
+        )
+        print(f"Success: {result.success}")
+        # Take screenshot
+        if result.screenshot:
+            with open("with_stealth.png", "wb") as f:
+                import base64
+                f.write(base64.b64decode(result.screenshot))
+            print("Screenshot saved: with_stealth.png")
+    
+    print("\nCheck the screenshots to see the difference in bot detection results!")
+
+
+if __name__ == "__main__":
+    asyncio.run(test_stealth())
\ No newline at end of file
diff --git a/docs/examples/storage_state_tutorial.md b/docs/examples/storage_state_tutorial.md
new file mode 100644
index 0000000..304e639
--- /dev/null
+++ b/docs/examples/storage_state_tutorial.md
@@ -0,0 +1,225 @@
+### Using `storage_state` to Pre-Load Cookies and LocalStorage
+
+Crawl4ai’s `AsyncWebCrawler` lets you preserve and reuse session data, including cookies and localStorage, across multiple runs. By providing a `storage_state`, you can start your crawls already “logged in” or with any other necessary session data—no need to repeat the login flow every time.
+
+#### What is `storage_state`?
+
+`storage_state` can be:
+
+- A dictionary containing cookies and localStorage data.
+- A path to a JSON file that holds this information.
+
+When you pass `storage_state` to the crawler, it applies these cookies and localStorage entries before loading any pages. This means your crawler effectively starts in a known authenticated or pre-configured state.
+
+#### Example Structure
+
+Here’s an example storage state:
+
+```json
+{
+  "cookies": [
+    {
+      "name": "session",
+      "value": "abcd1234",
+      "domain": "example.com",
+      "path": "/",
+      "expires": 1675363572.037711,
+      "httpOnly": false,
+      "secure": false,
+      "sameSite": "None"
+    }
+  ],
+  "origins": [
+    {
+      "origin": "https://example.com",
+      "localStorage": [
+        { "name": "token", "value": "my_auth_token" },
+        { "name": "refreshToken", "value": "my_refresh_token" }
+      ]
+    }
+  ]
+}
+```
+
+This JSON sets a `session` cookie and two localStorage entries (`token` and `refreshToken`) for `https://example.com`.
+
+---
+
+### Passing `storage_state` as a Dictionary
+
+You can directly provide the data as a dictionary:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+    storage_dict = {
+        "cookies": [
+            {
+                "name": "session",
+                "value": "abcd1234",
+                "domain": "example.com",
+                "path": "/",
+                "expires": 1675363572.037711,
+                "httpOnly": False,
+                "secure": False,
+                "sameSite": "None"
+            }
+        ],
+        "origins": [
+            {
+                "origin": "https://example.com",
+                "localStorage": [
+                    {"name": "token", "value": "my_auth_token"},
+                    {"name": "refreshToken", "value": "my_refresh_token"}
+                ]
+            }
+        ]
+    }
+
+    async with AsyncWebCrawler(
+        headless=True,
+        storage_state=storage_dict
+    ) as crawler:
+        result = await crawler.arun(url='https://example.com/protected')
+        if result.success:
+            print("Crawl succeeded with pre-loaded session data!")
+            print("Page HTML length:", len(result.html))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+### Passing `storage_state` as a File
+
+If you prefer a file-based approach, save the JSON above to `mystate.json` and reference it:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+    async with AsyncWebCrawler(
+        headless=True,
+        storage_state="mystate.json"  # Uses a JSON file instead of a dictionary
+    ) as crawler:
+        result = await crawler.arun(url='https://example.com/protected')
+        if result.success:
+            print("Crawl succeeded with pre-loaded session data!")
+            print("Page HTML length:", len(result.html))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+### Using `storage_state` to Avoid Repeated Logins (Sign In Once, Use Later)
+
+A common scenario is when you need to log in to a site (entering username/password, etc.) to access protected pages. Doing so every crawl is cumbersome. Instead, you can:
+
+1. Perform the login once in a hook.
+2. After login completes, export the resulting `storage_state` to a file.
+3. On subsequent runs, provide that `storage_state` to skip the login step.
+
+**Step-by-Step Example:**
+
+**First Run (Perform Login and Save State):**
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def on_browser_created_hook(browser):
+    # Access the default context and create a page
+    context = browser.contexts[0]
+    page = await context.new_page()
+    
+    # Navigate to the login page
+    await page.goto("https://example.com/login", wait_until="domcontentloaded")
+    
+    # Fill in credentials and submit
+    await page.fill("input[name='username']", "myuser")
+    await page.fill("input[name='password']", "mypassword")
+    await page.click("button[type='submit']")
+    await page.wait_for_load_state("networkidle")
+    
+    # Now the site sets tokens in localStorage and cookies
+    # Export this state to a file so we can reuse it
+    await context.storage_state(path="my_storage_state.json")
+    await page.close()
+
+async def main():
+    # First run: perform login and export the storage_state
+    async with AsyncWebCrawler(
+        headless=True,
+        verbose=True,
+        hooks={"on_browser_created": on_browser_created_hook},
+        use_persistent_context=True,
+        user_data_dir="./my_user_data"
+    ) as crawler:
+        
+        # After on_browser_created_hook runs, we have storage_state saved to my_storage_state.json
+        result = await crawler.arun(
+            url='https://example.com/protected-page',
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("First run result success:", result.success)
+        if result.success:
+            print("Protected page HTML length:", len(result.html))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Second Run (Reuse Saved State, No Login Needed):**
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def main():
+    # Second run: no need to hook on_browser_created this time.
+    # Just provide the previously saved storage state.
+    async with AsyncWebCrawler(
+        headless=True,
+        verbose=True,
+        use_persistent_context=True,
+        user_data_dir="./my_user_data",
+        storage_state="my_storage_state.json"  # Reuse previously exported state
+    ) as crawler:
+        
+        # Now the crawler starts already logged in
+        result = await crawler.arun(
+            url='https://example.com/protected-page',
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("Second run result success:", result.success)
+        if result.success:
+            print("Protected page HTML length:", len(result.html))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**What’s Happening Here?**
+
+- During the first run, the `on_browser_created_hook` logs into the site.  
+- After logging in, the crawler exports the current session (cookies, localStorage, etc.) to `my_storage_state.json`.  
+- On subsequent runs, passing `storage_state="my_storage_state.json"` starts the browser context with these tokens already in place, skipping the login steps.
+
+**Sign Out Scenario:**  
+If the website allows you to sign out by clearing tokens or by navigating to a sign-out URL, you can also run a script that uses `on_browser_created_hook` or `arun` to simulate signing out, then export the resulting `storage_state` again. That would give you a baseline “logged out” state to start fresh from next time.
+
+---
+
+### Conclusion
+
+By using `storage_state`, you can skip repetitive actions, like logging in, and jump straight into crawling protected content. Whether you provide a file path or a dictionary, this powerful feature helps maintain state between crawls, simplifying your data extraction pipelines.
\ No newline at end of file
diff --git a/docs/examples/summarize_page.py b/docs/examples/summarize_page.py
new file mode 100644
index 0000000..b54fd1a
--- /dev/null
+++ b/docs/examples/summarize_page.py
@@ -0,0 +1,53 @@
+import asyncio
+import json
+import os
+
+from pydantic import BaseModel, Field
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, LLMConfig
+from crawl4ai import CacheMode, LLMExtractionStrategy
+
+url = r"https://marketplace.visualstudio.com/items?itemName=Unclecode.groqopilot"
+
+class PageSummary(BaseModel):
+    title: str = Field(..., description="Title of the page.")
+    summary: str = Field(..., description="Summary of the page.")
+    brief_summary: str = Field(..., description="Brief summary of the page.")
+    keywords: list = Field(..., description="Keywords assigned to the page.")
+
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url=url,
+            config=CrawlerRunConfig(
+                word_count_threshold=1,
+                cache_mode=CacheMode.BYPASS,
+                extraction_strategy=LLMExtractionStrategy(
+                    llm_config=LLMConfig(
+                        provider="openai/gpt-4o",
+                        api_token=os.getenv("OPENAI_API_KEY"),
+                    ),
+                    schema=PageSummary.model_json_schema(),
+                    extraction_type="schema",
+                    apply_chunking=False,
+                    instruction="From the crawled content, extract the following details: "
+                    "1. Title of the page "
+                    "2. Summary of the page, which is a detailed summary "
+                    "3. Brief summary of the page, which is a paragraph text "
+                    "4. Keywords assigned to the page, which is a list of keywords. "
+                    'The extracted JSON format should look like this: '
+                    '{ "title": "Page Title", "summary": "Detailed summary of the page.", "brief_summary": "Brief summary in a paragraph.", "keywords": ["keyword1", "keyword2", "keyword3"] }',
+                ),
+            ),
+        )
+
+    page_summary = json.loads(result.extracted_content)
+    print(page_summary)
+
+    with open(".data/page_summary.json", "w", encoding="utf-8") as f:
+        f.write(result.extracted_content)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/table_extraction_example.py b/docs/examples/table_extraction_example.py
new file mode 100644
index 0000000..1291080
--- /dev/null
+++ b/docs/examples/table_extraction_example.py
@@ -0,0 +1,276 @@
+"""
+Example: Using Table Extraction Strategies in Crawl4AI
+
+This example demonstrates how to use different table extraction strategies
+to extract tables from web pages.
+"""
+
+import asyncio
+import pandas as pd
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    CacheMode,
+    DefaultTableExtraction,
+    NoTableExtraction,
+    TableExtractionStrategy
+)
+from typing import Dict, List, Any
+
+
+async def example_default_extraction():
+    """Example 1: Using default table extraction (automatic)."""
+    print("\n" + "="*50)
+    print("Example 1: Default Table Extraction")
+    print("="*50)
+    
+    async with AsyncWebCrawler() as crawler:
+        # No need to specify table_extraction - uses DefaultTableExtraction automatically
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            table_score_threshold=7  # Adjust sensitivity (default: 7)
+        )
+        
+        result = await crawler.arun(
+            "https://en.wikipedia.org/wiki/List_of_countries_by_GDP_(nominal)",
+            config=config
+        )
+        
+        if result.success and result.tables:
+            print(f"Found {len(result.tables)} tables")
+            
+            # Convert first table to pandas DataFrame
+            if result.tables:
+                first_table = result.tables[0]
+                df = pd.DataFrame(
+                    first_table['rows'],
+                    columns=first_table['headers'] if first_table['headers'] else None
+                )
+                print(f"\nFirst table preview:")
+                print(df.head())
+                print(f"Shape: {df.shape}")
+
+
+async def example_custom_configuration():
+    """Example 2: Custom table extraction configuration."""
+    print("\n" + "="*50)
+    print("Example 2: Custom Table Configuration")
+    print("="*50)
+    
+    async with AsyncWebCrawler() as crawler:
+        # Create custom extraction strategy with specific settings
+        table_strategy = DefaultTableExtraction(
+            table_score_threshold=5,  # Lower threshold for more permissive detection
+            min_rows=3,  # Only extract tables with at least 3 rows
+            min_cols=2,  # Only extract tables with at least 2 columns
+            verbose=True
+        )
+        
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            table_extraction=table_strategy,
+            # Target specific tables using CSS selector
+            css_selector="div.main-content"
+        )
+        
+        result = await crawler.arun(
+            "https://example.com/data",
+            config=config
+        )
+        
+        if result.success:
+            print(f"Found {len(result.tables)} tables matching criteria")
+            
+            for i, table in enumerate(result.tables):
+                print(f"\nTable {i+1}:")
+                print(f"  Caption: {table.get('caption', 'No caption')}")
+                print(f"  Size: {table['metadata']['row_count']} rows × {table['metadata']['column_count']} columns")
+                print(f"  Has headers: {table['metadata']['has_headers']}")
+
+
+async def example_disable_extraction():
+    """Example 3: Disable table extraction when not needed."""
+    print("\n" + "="*50)
+    print("Example 3: Disable Table Extraction")
+    print("="*50)
+    
+    async with AsyncWebCrawler() as crawler:
+        # Use NoTableExtraction to skip table processing entirely
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            table_extraction=NoTableExtraction()  # No tables will be extracted
+        )
+        
+        result = await crawler.arun(
+            "https://example.com",
+            config=config
+        )
+        
+        if result.success:
+            print(f"Tables extracted: {len(result.tables)} (should be 0)")
+            print("Table extraction disabled - better performance for non-table content")
+
+
+class FinancialTableExtraction(TableExtractionStrategy):
+    """
+    Custom strategy for extracting financial tables with specific requirements.
+    """
+    
+    def __init__(self, currency_symbols=None, **kwargs):
+        super().__init__(**kwargs)
+        self.currency_symbols = currency_symbols or ['$', '€', '£', '¥']
+    
+    def extract_tables(self, element, **kwargs):
+        """Extract only tables that appear to contain financial data."""
+        tables_data = []
+        
+        for table in element.xpath(".//table"):
+            # Check if table contains currency symbols
+            table_text = ''.join(table.itertext())
+            has_currency = any(symbol in table_text for symbol in self.currency_symbols)
+            
+            if not has_currency:
+                continue
+            
+            # Extract using base logic (could reuse DefaultTableExtraction logic)
+            headers = []
+            rows = []
+            
+            # Extract headers
+            for th in table.xpath(".//thead//th | .//tr[1]//th"):
+                headers.append(th.text_content().strip())
+            
+            # Extract rows
+            for tr in table.xpath(".//tbody//tr | .//tr[position()>1]"):
+                row = []
+                for td in tr.xpath(".//td"):
+                    cell_text = td.text_content().strip()
+                    # Clean currency values
+                    for symbol in self.currency_symbols:
+                        cell_text = cell_text.replace(symbol, '')
+                    row.append(cell_text)
+                if row:
+                    rows.append(row)
+            
+            if headers or rows:
+                tables_data.append({
+                    "headers": headers,
+                    "rows": rows,
+                    "caption": table.xpath(".//caption/text()")[0] if table.xpath(".//caption") else "",
+                    "summary": table.get("summary", ""),
+                    "metadata": {
+                        "type": "financial",
+                        "has_currency": True,
+                        "row_count": len(rows),
+                        "column_count": len(headers) if headers else len(rows[0]) if rows else 0
+                    }
+                })
+        
+        return tables_data
+
+
+async def example_custom_strategy():
+    """Example 4: Custom table extraction strategy."""
+    print("\n" + "="*50)
+    print("Example 4: Custom Financial Table Strategy")
+    print("="*50)
+    
+    async with AsyncWebCrawler() as crawler:
+        # Use custom strategy for financial tables
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            table_extraction=FinancialTableExtraction(
+                currency_symbols=['$', '€'],
+                verbose=True
+            )
+        )
+        
+        result = await crawler.arun(
+            "https://finance.yahoo.com/",
+            config=config
+        )
+        
+        if result.success:
+            print(f"Found {len(result.tables)} financial tables")
+            
+            for table in result.tables:
+                if table['metadata'].get('type') == 'financial':
+                    print(f"  ✓ Financial table with {table['metadata']['row_count']} rows")
+
+
+async def example_combined_extraction():
+    """Example 5: Combine table extraction with other strategies."""
+    print("\n" + "="*50)
+    print("Example 5: Combined Extraction Strategies")
+    print("="*50)
+    
+    from crawl4ai import LLMExtractionStrategy, LLMConfig
+    
+    async with AsyncWebCrawler() as crawler:
+        # Define schema for structured extraction
+        schema = {
+            "type": "object",
+            "properties": {
+                "page_title": {"type": "string"},
+                "main_topic": {"type": "string"},
+                "key_figures": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                }
+            }
+        }
+        
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            # Table extraction
+            table_extraction=DefaultTableExtraction(
+                table_score_threshold=6,
+                min_rows=2
+            ),
+            # LLM extraction for structured data
+            extraction_strategy=LLMExtractionStrategy(
+                llm_config=LLMConfig(provider="openai"),
+                schema=schema
+            )
+        )
+        
+        result = await crawler.arun(
+            "https://en.wikipedia.org/wiki/Economy_of_the_United_States",
+            config=config
+        )
+        
+        if result.success:
+            print(f"Tables found: {len(result.tables)}")
+            
+            # Tables are in result.tables
+            if result.tables:
+                print(f"First table has {len(result.tables[0]['rows'])} rows")
+            
+            # Structured data is in result.extracted_content
+            if result.extracted_content:
+                import json
+                structured_data = json.loads(result.extracted_content)
+                print(f"Page title: {structured_data.get('page_title', 'N/A')}")
+                print(f"Main topic: {structured_data.get('main_topic', 'N/A')}")
+
+
+async def main():
+    """Run all examples."""
+    print("\n" + "="*60)
+    print("CRAWL4AI TABLE EXTRACTION EXAMPLES")
+    print("="*60)
+    
+    # Run examples
+    await example_default_extraction()
+    await example_custom_configuration()
+    await example_disable_extraction()
+    await example_custom_strategy()
+    # await example_combined_extraction()  # Requires OpenAI API key
+    
+    print("\n" + "="*60)
+    print("EXAMPLES COMPLETED")
+    print("="*60)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/tutorial_dynamic_clicks.md b/docs/examples/tutorial_dynamic_clicks.md
new file mode 100644
index 0000000..d966995
--- /dev/null
+++ b/docs/examples/tutorial_dynamic_clicks.md
@@ -0,0 +1,117 @@
+# Tutorial: Clicking Buttons to Load More Content with Crawl4AI
+
+## Introduction
+
+When scraping dynamic websites, it’s common to encounter “Load More” or “Next” buttons that must be clicked to reveal new content. Crawl4AI provides a straightforward way to handle these situations using JavaScript execution and waiting conditions. In this tutorial, we’ll cover two approaches:
+
+1. **Step-by-step (Session-based) Approach:** Multiple calls to `arun()` to progressively load more content.
+2. **Single-call Approach:** Execute a more complex JavaScript snippet inside a single `arun()` call to handle all clicks at once before the extraction.
+
+## Prerequisites
+
+- A working installation of Crawl4AI
+- Basic familiarity with Python’s `async`/`await` syntax
+
+## Step-by-Step Approach
+
+Use a session ID to maintain state across multiple `arun()` calls:
+
+```python
+from crawl4ai import AsyncWebCrawler, CacheMode
+
+js_code = [
+    # This JS finds the “Next” button and clicks it
+    "const nextButton = document.querySelector('button.next'); nextButton && nextButton.click();"
+]
+
+wait_for_condition = "css:.new-content-class"
+
+async with AsyncWebCrawler(headless=True, verbose=True) as crawler:
+    # 1. Load the initial page
+    result_initial = await crawler.arun(
+        url="https://example.com",
+        cache_mode=CacheMode.BYPASS,
+        session_id="my_session"
+    )
+
+    # 2. Click the 'Next' button and wait for new content
+    result_next = await crawler.arun(
+        url="https://example.com",
+        session_id="my_session",
+        js_code=js_code,
+        wait_for=wait_for_condition,
+        js_only=True,
+        cache_mode=CacheMode.BYPASS
+    )
+
+# `result_next` now contains the updated HTML after clicking 'Next'
+```
+
+**Key Points:**
+- **`session_id`**: Keeps the same browser context open.
+- **`js_code`**: Executes JavaScript in the context of the already loaded page.
+- **`wait_for`**: Ensures the crawler waits until new content is fully loaded.
+- **`js_only=True`**: Runs the JS in the current session without reloading the page.
+
+By repeating the `arun()` call multiple times and modifying the `js_code` (e.g., clicking different modules or pages), you can iteratively load all the desired content.
+
+## Single-call Approach
+
+If the page allows it, you can run a single `arun()` call with a more elaborate JavaScript snippet that:
+- Iterates over all the modules or "Next" buttons
+- Clicks them one by one
+- Waits for content updates between each click
+- Once done, returns control to Crawl4AI for extraction.
+
+Example snippet:
+
+```python
+from crawl4ai import AsyncWebCrawler, CacheMode
+
+js_code = [
+    # Example JS that clicks multiple modules:
+    """
+    (async () => {
+      const modules = document.querySelectorAll('.module-item');
+      for (let i = 0; i < modules.length; i++) {
+        modules[i].scrollIntoView();
+        modules[i].click();
+        // Wait for each module’s content to load, adjust 100ms as needed
+        await new Promise(r => setTimeout(r, 100));
+      }
+    })();
+    """
+]
+
+async with AsyncWebCrawler(headless=True, verbose=True) as crawler:
+    result = await crawler.arun(
+        url="https://example.com",
+        js_code=js_code,
+        wait_for="css:.final-loaded-content-class",
+        cache_mode=CacheMode.BYPASS
+    )
+
+# `result` now contains all content after all modules have been clicked in one go.
+```
+
+**Key Points:**
+- All interactions (clicks and waits) happen before the extraction.
+- Ideal for pages where all steps can be done in a single pass.
+
+## Choosing the Right Approach
+
+- **Step-by-Step (Session-based)**: 
+  - Good when you need fine-grained control or must dynamically check conditions before clicking the next page.
+  - Useful if the page requires multiple conditions checked at runtime.
+
+- **Single-call**:
+  - Perfect if the sequence of interactions is known in advance.
+  - Cleaner code if the page’s structure is consistent and predictable.
+
+## Conclusion
+
+Crawl4AI makes it easy to handle dynamic content:
+- Use session IDs and multiple `arun()` calls for stepwise crawling.
+- Or pack all actions into one `arun()` call if the interactions are well-defined upfront.
+
+This flexibility ensures you can handle a wide range of dynamic web pages efficiently.
diff --git a/docs/examples/tutorial_v0.5.py b/docs/examples/tutorial_v0.5.py
new file mode 100644
index 0000000..1693b1f
--- /dev/null
+++ b/docs/examples/tutorial_v0.5.py
@@ -0,0 +1,460 @@
+import asyncio
+import time
+import re
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, BrowserConfig, MemoryAdaptiveDispatcher, HTTPCrawlerConfig
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+from crawl4ai.deep_crawling import (
+    BestFirstCrawlingStrategy,
+    FilterChain,
+    URLPatternFilter,
+    DomainFilter,
+    ContentTypeFilter,
+)
+from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
+from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
+from crawl4ai import ProxyConfig
+from crawl4ai import RoundRobinProxyStrategy
+from crawl4ai.content_filter_strategy import LLMContentFilter
+from crawl4ai import DefaultMarkdownGenerator
+from crawl4ai import LLMConfig
+from crawl4ai import JsonCssExtractionStrategy
+from crawl4ai.processors.pdf import PDFCrawlerStrategy, PDFContentScrapingStrategy
+from pprint import pprint
+
+
+# 1️⃣ Deep Crawling with Best-First Strategy
+async def deep_crawl():
+    """
+    PART 1: Deep Crawling with Best-First Strategy
+    
+    This function demonstrates:
+    - Using the BestFirstCrawlingStrategy
+    - Creating filter chains to narrow down crawl targets
+    - Using a scorer to prioritize certain URLs
+    - Respecting robots.txt rules
+    """
+    print("\n===== DEEP CRAWLING =====")
+    print("This example shows how to implement deep crawling with filters, scorers, and robots.txt compliance.")
+    
+    # Create a filter chain to filter urls based on patterns, domains and content type
+    filter_chain = FilterChain(
+        [
+            DomainFilter(
+                allowed_domains=["docs.crawl4ai.com"],
+                blocked_domains=["old.docs.crawl4ai.com"],
+            ),
+            URLPatternFilter(patterns=["*core*", "*advanced*"],),
+            ContentTypeFilter(allowed_types=["text/html"]),
+        ]
+    )
+    
+    # Create a keyword scorer that prioritises the pages with certain keywords first
+    keyword_scorer = KeywordRelevanceScorer(
+        keywords=["crawl", "example", "async", "configuration"], weight=0.7
+    )
+    
+    # Set up the configuration with robots.txt compliance enabled
+    deep_crawl_config = CrawlerRunConfig(
+        deep_crawl_strategy=BestFirstCrawlingStrategy(
+            max_depth=2,
+            include_external=False,
+            filter_chain=filter_chain,
+            url_scorer=keyword_scorer,
+        ),
+        scraping_strategy=LXMLWebScrapingStrategy(),
+        stream=True,
+        verbose=True,
+        check_robots_txt=True,  # Enable robots.txt compliance
+    )
+    
+    # Execute the crawl
+    async with AsyncWebCrawler() as crawler:
+        print("\n📊 Starting deep crawl with Best-First strategy...")
+        print("  - Filtering by domain, URL patterns, and content type")
+        print("  - Scoring pages based on keyword relevance")
+        print("  - Respecting robots.txt rules")
+        
+        start_time = time.perf_counter()
+        results = []
+        
+        async for result in await crawler.arun(url="https://docs.crawl4ai.com", config=deep_crawl_config):
+            # Print each result as it comes in
+            depth = result.metadata.get("depth", 0)
+            score = result.metadata.get("score", 0)
+            print(f"Crawled: {result.url} (Depth: {depth}), score: {score:.2f}")
+            results.append(result)
+            
+        duration = time.perf_counter() - start_time
+        
+        # Print summary statistics
+        print(f"\n✅ Crawled {len(results)} high-value pages in {duration:.2f} seconds")
+        
+        # Group by depth
+        if results:
+            depth_counts = {}
+            for result in results:
+                depth = result.metadata.get("depth", 0)
+                depth_counts[depth] = depth_counts.get(depth, 0) + 1
+            
+            print("\n📊 Pages crawled by depth:")
+            for depth, count in sorted(depth_counts.items()):
+                print(f"  Depth {depth}: {count} pages")
+
+
+# 2️⃣ Memory-Adaptive Dispatcher
+async def memory_adaptive_dispatcher():
+    """
+    PART 2: Memory-Adaptive Dispatcher
+    
+    This function demonstrates:
+    - Using MemoryAdaptiveDispatcher to manage system memory
+    - Batch and streaming modes with multiple URLs
+    """
+    print("\n===== MEMORY-ADAPTIVE DISPATCHER =====")
+    print("This example shows how to use the memory-adaptive dispatcher for resource management.")
+    
+    # Configure the dispatcher (optional, defaults are used if not provided)
+    dispatcher = MemoryAdaptiveDispatcher(
+        memory_threshold_percent=80.0,  # Pause if memory usage exceeds 80%
+        check_interval=0.5,  # Check memory every 0.5 seconds
+    )
+    
+    # Test URLs
+    urls = [
+        "https://docs.crawl4ai.com", 
+        "https://github.com/unclecode/crawl4ai"
+    ]
+    
+    async def batch_mode():
+        print("\n📊 BATCH MODE:")
+        print("  In this mode, all results are collected before being returned.")
+        
+        async with AsyncWebCrawler() as crawler:
+            start_time = time.perf_counter()
+            results = await crawler.arun_many(
+                urls=urls,
+                config=CrawlerRunConfig(stream=False),  # Batch mode
+                dispatcher=dispatcher,
+            )
+            
+            print(f"  ✅ Received all {len(results)} results after {time.perf_counter() - start_time:.2f} seconds")
+            for result in results:
+                print(f"  → {result.url} with status code: {result.status_code}")
+    
+    async def stream_mode():
+        print("\n📊 STREAMING MODE:")
+        print("  In this mode, results are processed as they become available.")
+        
+        async with AsyncWebCrawler() as crawler:
+            start_time = time.perf_counter()
+            count = 0
+            first_result_time = None
+            
+            async for result in await crawler.arun_many(
+                urls=urls,
+                config=CrawlerRunConfig(stream=True),  # Stream mode
+                dispatcher=dispatcher,
+            ):
+                count += 1
+                current_time = time.perf_counter() - start_time
+                
+                if count == 1:
+                    first_result_time = current_time
+                    print(f"  ✅ First result after {first_result_time:.2f} seconds: {result.url}")
+                else:
+                    print(f"  → Result #{count} after {current_time:.2f} seconds: {result.url}")
+            
+            print(f"  ✅ Total: {count} results")
+            print(f"  ✅ First result: {first_result_time:.2f} seconds")
+            print(f"  ✅ All results: {time.perf_counter() - start_time:.2f} seconds")
+    
+    # Run both examples
+    await batch_mode()
+    await stream_mode()
+    
+    print("\n🔍 Key Takeaway: The memory-adaptive dispatcher prevents OOM errors")
+    print("  and manages concurrency based on system resources.")
+
+
+# 3️⃣ HTTP Crawler Strategy
+async def http_crawler_strategy():
+    """
+    PART 3: HTTP Crawler Strategy
+    
+    This function demonstrates:
+    - Using the lightweight HTTP-only crawler
+    - Setting custom headers and configurations
+    """
+    print("\n===== HTTP CRAWLER STRATEGY =====")
+    print("This example shows how to use the fast, lightweight HTTP-only crawler.")
+    
+    # Use the HTTP crawler strategy
+    http_config = HTTPCrawlerConfig(
+        method="GET",
+        headers={"User-Agent": "MyCustomBot/1.0"},
+        follow_redirects=True,
+        verify_ssl=True
+    )
+    
+    print("\n📊 Initializing HTTP crawler strategy...")
+    print("  - Using custom User-Agent: MyCustomBot/1.0")
+    print("  - Following redirects: Enabled")
+    print("  - Verifying SSL: Enabled")
+    
+    # Create crawler with HTTP strategy
+    async with AsyncWebCrawler(
+        crawler_strategy=AsyncHTTPCrawlerStrategy(browser_config=http_config)
+    ) as crawler:
+        start_time = time.perf_counter()
+        result = await crawler.arun("https://example.com")
+        duration = time.perf_counter() - start_time
+        
+        print(f"\n✅ Crawled in {duration:.2f} seconds")
+        print(f"✅ Status code: {result.status_code}")
+        print(f"✅ Content length: {len(result.html)} bytes")
+        
+        # Check if there was a redirect
+        if result.redirected_url and result.redirected_url != result.url:
+            print(f"ℹ️ Redirected from {result.url} to {result.redirected_url}")
+    
+    print("\n🔍 Key Takeaway: HTTP crawler is faster and more memory-efficient")
+    print("  than browser-based crawling for simple pages.")
+
+
+# 4️⃣ Proxy Rotation
+async def proxy_rotation():
+    """
+    PART 4: Proxy Rotation
+    
+    This function demonstrates:
+    - Setting up a proxy rotation strategy
+    - Using multiple proxies in a round-robin fashion
+    """
+    print("\n===== PROXY ROTATION =====")
+    print("This example shows how to implement proxy rotation for distributed crawling.")
+    
+    # Load proxies and create rotation strategy
+    proxies = ProxyConfig.from_env()
+    #eg: export PROXIES="ip1:port1:username1:password1,ip2:port2:username2:password2"
+    if not proxies:
+        print("No proxies found in environment. Set PROXIES env variable!")
+        return
+        
+    proxy_strategy = RoundRobinProxyStrategy(proxies)
+    
+    # Create configs
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        proxy_rotation_strategy=proxy_strategy
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        urls = ["https://httpbin.org/ip"] * (len(proxies) * 2)  # Test each proxy twice
+
+        print("\n📈 Initializing crawler with proxy rotation...")
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            print("\n🚀 Starting batch crawl with proxy rotation...")
+            results = await crawler.arun_many(
+                urls=urls,
+                config=run_config
+            )
+            for result in results:
+                if result.success:
+                    ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
+                    current_proxy = run_config.proxy_config if run_config.proxy_config else None
+                    
+                    if current_proxy and ip_match:
+                        print(f"URL {result.url}")
+                        print(f"Proxy {current_proxy.server} -> Response IP: {ip_match.group(0)}")
+                        verified = ip_match.group(0) == current_proxy.ip
+                        if verified:
+                            print(f"✅ Proxy working! IP matches: {current_proxy.ip}")
+                        else:
+                            print("❌ Proxy failed or IP mismatch!")
+                    print("---")
+                else:
+                    print(f"❌ Crawl via proxy failed!: {result.error_message}")
+
+
+# 5️⃣ LLM Content Filter (requires API key)
+async def llm_content_filter():
+    """
+    PART 5: LLM Content Filter
+    
+    This function demonstrates:
+    - Configuring LLM providers via LLMConfig
+    - Using LLM to generate focused markdown
+    - LLMConfig for configuration
+    
+    Note: Requires a valid API key for the chosen LLM provider
+    """
+    print("\n===== LLM CONTENT FILTER =====")
+    print("This example shows how to use LLM to generate focused markdown content.")
+    print("Note: This example requires an API key. Set it in environment variables.")
+    
+    # Create LLM configuration
+    # Replace with your actual API key or set as environment variable
+    llm_config = LLMConfig(
+        provider="gemini/gemini-1.5-pro", 
+        api_token="env:GEMINI_API_KEY"  # Will read from GEMINI_API_KEY environment variable
+    )
+    
+    print("\n📊 Setting up LLM content filter...")
+    print(f"  - Provider: {llm_config.provider}")
+    print("  - API token: Using environment variable")
+    print("  - Instruction: Extract key concepts and summaries")
+    
+    # Create markdown generator with LLM filter
+    markdown_generator = DefaultMarkdownGenerator(
+        content_filter=LLMContentFilter(
+            llm_config=llm_config,
+            instruction="Extract key concepts and summaries"
+        )
+    )
+    
+    config = CrawlerRunConfig(markdown_generator=markdown_generator)
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://docs.crawl4ai.com", config=config)
+        pprint(result.markdown.fit_markdown)
+        print("\n✅ Generated focused markdown:")
+
+
+
+# 6️⃣ PDF Processing
+async def pdf_processing():
+    """
+    PART 6: PDF Processing
+    
+    This function demonstrates:
+    - Using PDFCrawlerStrategy and PDFContentScrapingStrategy
+    - Extracting text and metadata from PDFs
+    """
+    print("\n===== PDF PROCESSING =====")
+    print("This example shows how to extract text and metadata from PDF files.")
+    
+    # Sample PDF URL
+    pdf_url = "https://arxiv.org/pdf/2310.06825.pdf"
+    
+    print("\n📊 Initializing PDF crawler...")
+    print(f"  - Target PDF: {pdf_url}")
+    print("  - Using PDFCrawlerStrategy and PDFContentScrapingStrategy")
+    
+    # Create crawler with PDF strategy
+    async with AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) as crawler:
+        print("\n🚀 Starting PDF processing...")
+        
+        start_time = time.perf_counter()
+        result = await crawler.arun(
+            pdf_url,
+            config=CrawlerRunConfig(scraping_strategy=PDFContentScrapingStrategy())
+        )
+        duration = time.perf_counter() - start_time
+        
+        print(f"\n✅ Processed PDF in {duration:.2f} seconds")
+        
+        # Show metadata
+        print("\n📄 PDF Metadata:")
+        if result.metadata:
+            for key, value in result.metadata.items():
+                if key not in ["html", "text", "markdown"] and value:
+                    print(f"  - {key}: {value}")
+        else:
+            print("  No metadata available")
+        
+        # Show sample of content
+        if result.markdown:
+            print("\n📝 PDF Content Sample:")
+            content_sample = result.markdown[:500] + "..." if len(result.markdown) > 500 else result.markdown
+            print(f"---\n{content_sample}\n---")
+        else:
+            print("\n⚠️ No content extracted")
+    
+    print("\n🔍 Key Takeaway: Crawl4AI can now process PDF files")
+    print("  to extract both text content and metadata.")
+
+
+# 7️⃣ LLM Schema Generation (requires API key)
+async def llm_schema_generation():
+    """
+    PART 7: LLM Schema Generation
+    
+    This function demonstrates:
+    - Configuring LLM providers via LLMConfig
+    - Using LLM to generate extraction schemas
+    - JsonCssExtractionStrategy
+    
+    Note: Requires a valid API key for the chosen LLM provider
+    """
+    print("\n===== LLM SCHEMA GENERATION =====")
+    print("This example shows how to use LLM to automatically generate extraction schemas.")
+    print("Note: This example requires an API key. Set it in environment variables.")
+    
+    # Sample HTML
+    sample_html = """
+    

+        
Awesome Gaming Laptop

+        
$1,299.99

+        

+            
    
+                
  • 16GB RAM
    • 
+                
  • 512GB SSD
    • 
+                
  • RTX 3080
    • 
+            

+        

+        
4.7/5

+    

+    """
+    print("\n📊 Setting up LLMConfig...")
+    # Create LLM configuration
+    llm_config = LLMConfig(
+        provider="gemini/gemini-1.5-pro", 
+        api_token="env:GEMINI_API_KEY"
+    )
+    print("\n🚀 Generating schema for product extraction...")
+    print("  This would use the LLM to analyze HTML and create an extraction schema")
+    schema = JsonCssExtractionStrategy.generate_schema(
+    html=sample_html,
+    llm_config = llm_config,
+    query="Extract product name and price"
+    )
+    print("\n✅ Generated Schema:")
+    pprint(schema)
+    
+# Run all sections
+async def run_tutorial():
+    """
+    Main function to run all tutorial sections.
+    """
+    print("\n🚀 CRAWL4AI v0.5.0 TUTORIAL 🚀")
+    print("===============================")
+    print("This tutorial demonstrates the key features of Crawl4AI v0.5.0")
+    print("Including deep crawling, memory-adaptive dispatching, advanced filtering,")
+    print("and more powerful extraction capabilities.")
+    
+    # Sections to run
+    sections = [
+        deep_crawl,                 # 1. Deep Crawling with Best-First Strategy
+        memory_adaptive_dispatcher, # 2. Memory-Adaptive Dispatcher
+        http_crawler_strategy,      # 3. HTTP Crawler Strategy
+        proxy_rotation,             # 4. Proxy Rotation
+        llm_content_filter,         # 5. LLM Content Filter
+        pdf_processing,             # 6. PDF Processing
+        llm_schema_generation,      # 7. Schema Generation using LLM
+    ]
+    
+    for section in sections:
+        try:
+            await section()
+        except Exception as e:
+            print(f"⚠️ Error in {section.__name__}: {e}")
+    
+    print("\n🎉 TUTORIAL COMPLETE! 🎉")
+    print("You've now explored the key features of Crawl4AI v0.5.0")
+    print("For more information, visit https://docs.crawl4ai.com")
+
+
+# Run the tutorial
+if __name__ == "__main__":
+    asyncio.run(run_tutorial())
\ No newline at end of file
diff --git a/docs/examples/undetectability/undetected_basic_test.py b/docs/examples/undetectability/undetected_basic_test.py
new file mode 100644
index 0000000..f28231f
--- /dev/null
+++ b/docs/examples/undetectability/undetected_basic_test.py
@@ -0,0 +1,74 @@
+"""
+Basic Undetected Browser Test
+Simple example to test if undetected mode works
+"""
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+async def test_regular_mode():
+    """Test with regular browser"""
+    print("Testing Regular Browser Mode...")
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://www.example.com")
+        print(f"Regular Mode - Success: {result.success}")
+        print(f"Regular Mode - Status: {result.status_code}")
+        print(f"Regular Mode - Content length: {len(result.markdown.raw_markdown)}")
+        print(f"Regular Mode - First 100 chars: {result.markdown.raw_markdown[:100]}...")
+        return result.success
+
+async def test_undetected_mode():
+    """Test with undetected browser"""
+    print("\nTesting Undetected Browser Mode...")
+    from crawl4ai import UndetectedAdapter
+    from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+    
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True
+    )
+    
+    # Create undetected adapter
+    undetected_adapter = UndetectedAdapter()
+    
+    # Create strategy with undetected adapter
+    crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=undetected_adapter
+    )
+    
+    async with AsyncWebCrawler(
+        crawler_strategy=crawler_strategy,
+        config=browser_config
+    ) as crawler:
+        result = await crawler.arun(url="https://www.example.com")
+        print(f"Undetected Mode - Success: {result.success}")
+        print(f"Undetected Mode - Status: {result.status_code}")
+        print(f"Undetected Mode - Content length: {len(result.markdown.raw_markdown)}")
+        print(f"Undetected Mode - First 100 chars: {result.markdown.raw_markdown[:100]}...")
+        return result.success
+
+async def main():
+    """Run both tests"""
+    print("🤖 Crawl4AI Basic Adapter Test\n")
+    
+    # Test regular mode
+    regular_success = await test_regular_mode()
+    
+    # Test undetected mode
+    undetected_success = await test_undetected_mode()
+    
+    # Summary
+    print("\n" + "="*50)
+    print("Summary:")
+    print(f"Regular Mode: {'✅ Success' if regular_success else '❌ Failed'}")
+    print(f"Undetected Mode: {'✅ Success' if undetected_success else '❌ Failed'}")
+    print("="*50)
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/undetectability/undetected_bot_test.py b/docs/examples/undetectability/undetected_bot_test.py
new file mode 100644
index 0000000..ba9a6ec
--- /dev/null
+++ b/docs/examples/undetectability/undetected_bot_test.py
@@ -0,0 +1,155 @@
+"""
+Bot Detection Test - Compare Regular vs Undetected
+Tests browser fingerprinting differences at bot.sannysoft.com
+"""
+
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler, 
+    BrowserConfig, 
+    CrawlerRunConfig,
+    UndetectedAdapter,
+    CrawlResult
+)
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+# Bot detection test site
+TEST_URL = "https://bot.sannysoft.com"
+
+def analyze_bot_detection(result: CrawlResult) -> dict:
+    """Analyze bot detection results from the page"""
+    detections = {
+        "webdriver": False,
+        "headless": False, 
+        "automation": False,
+        "user_agent": False,
+        "total_tests": 0,
+        "failed_tests": 0
+    }
+    
+    if not result.success or not result.html:
+        return detections
+    
+    # Look for specific test results in the HTML
+    html_lower = result.html.lower()
+    
+    # Check for common bot indicators
+    if "webdriver" in html_lower and ("fail" in html_lower or "true" in html_lower):
+        detections["webdriver"] = True
+        detections["failed_tests"] += 1
+    
+    if "headless" in html_lower and ("fail" in html_lower or "true" in html_lower):
+        detections["headless"] = True
+        detections["failed_tests"] += 1
+    
+    if "automation" in html_lower and "detected" in html_lower:
+        detections["automation"] = True
+        detections["failed_tests"] += 1
+    
+    # Count total tests (approximate)
+    detections["total_tests"] = html_lower.count("test") + html_lower.count("check")
+    
+    return detections
+
+async def test_browser_mode(adapter_name: str, adapter=None):
+    """Test a browser mode and return results"""
+    print(f"\n{'='*60}")
+    print(f"Testing: {adapter_name}")
+    print(f"{'='*60}")
+    
+    browser_config = BrowserConfig(
+        headless=False,  # Run in headed mode for better results
+        verbose=True,
+        viewport_width=1920,
+        viewport_height=1080,
+    )
+    
+    if adapter:
+        # Use undetected mode
+        crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+            browser_config=browser_config,
+            browser_adapter=adapter
+        )
+        crawler = AsyncWebCrawler(
+            crawler_strategy=crawler_strategy,
+            config=browser_config
+        )
+    else:
+        # Use regular mode
+        crawler = AsyncWebCrawler(config=browser_config)
+    
+    async with crawler:
+        config = CrawlerRunConfig(
+            delay_before_return_html=3.0,  # Let detection scripts run
+            wait_for_images=True,
+            screenshot=True,
+            simulate_user=False,  # Don't simulate for accurate detection
+        )
+        
+        result = await crawler.arun(url=TEST_URL, config=config)
+        
+        print(f"\n✓ Success: {result.success}")
+        print(f"✓ Status Code: {result.status_code}")
+        
+        if result.success:
+            # Analyze detection results
+            detections = analyze_bot_detection(result)
+            
+            print(f"\n🔍 Bot Detection Analysis:")
+            print(f"  - WebDriver Detected: {'❌ Yes' if detections['webdriver'] else '✅ No'}")
+            print(f"  - Headless Detected: {'❌ Yes' if detections['headless'] else '✅ No'}")
+            print(f"  - Automation Detected: {'❌ Yes' if detections['automation'] else '✅ No'}")
+            print(f"  - Failed Tests: {detections['failed_tests']}")
+            
+            # Show some content
+            if result.markdown.raw_markdown:
+                print(f"\nContent preview:")
+                lines = result.markdown.raw_markdown.split('\n')
+                for line in lines[:20]:  # Show first 20 lines
+                    if any(keyword in line.lower() for keyword in ['test', 'pass', 'fail', 'yes', 'no']):
+                        print(f"  {line.strip()}")
+        
+        return result, detections if result.success else {}
+
+async def main():
+    """Run the comparison"""
+    print("🤖 Crawl4AI - Bot Detection Test")
+    print(f"Testing at: {TEST_URL}")
+    print("This site runs various browser fingerprinting tests\n")
+    
+    # Test regular browser
+    regular_result, regular_detections = await test_browser_mode("Regular Browser")
+    
+    # Small delay
+    await asyncio.sleep(2)
+    
+    # Test undetected browser
+    undetected_adapter = UndetectedAdapter()
+    undetected_result, undetected_detections = await test_browser_mode(
+        "Undetected Browser", 
+        undetected_adapter
+    )
+    
+    # Summary comparison
+    print(f"\n{'='*60}")
+    print("COMPARISON SUMMARY")
+    print(f"{'='*60}")
+    
+    print(f"\n{'Test':<25} {'Regular':<15} {'Undetected':<15}")
+    print(f"{'-'*55}")
+    
+    if regular_detections and undetected_detections:
+        print(f"{'WebDriver Detection':<25} {'❌ Detected' if regular_detections['webdriver'] else '✅ Passed':<15} {'❌ Detected' if undetected_detections['webdriver'] else '✅ Passed':<15}")
+        print(f"{'Headless Detection':<25} {'❌ Detected' if regular_detections['headless'] else '✅ Passed':<15} {'❌ Detected' if undetected_detections['headless'] else '✅ Passed':<15}")
+        print(f"{'Automation Detection':<25} {'❌ Detected' if regular_detections['automation'] else '✅ Passed':<15} {'❌ Detected' if undetected_detections['automation'] else '✅ Passed':<15}")
+        print(f"{'Failed Tests':<25} {regular_detections['failed_tests']:<15} {undetected_detections['failed_tests']:<15}")
+    
+    print(f"\n{'='*60}")
+    
+    if undetected_detections.get('failed_tests', 0) < regular_detections.get('failed_tests', 1):
+        print("✅ Undetected browser performed better at evading detection!")
+    else:
+        print("ℹ️  Both browsers had similar detection results")
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/undetectability/undetected_cloudflare_test.py b/docs/examples/undetectability/undetected_cloudflare_test.py
new file mode 100644
index 0000000..2fc2ce0
--- /dev/null
+++ b/docs/examples/undetectability/undetected_cloudflare_test.py
@@ -0,0 +1,164 @@
+"""
+Undetected Browser Test - Cloudflare Protected Site
+Tests the difference between regular and undetected modes on a Cloudflare-protected site
+"""
+
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler, 
+    BrowserConfig, 
+    CrawlerRunConfig,
+    UndetectedAdapter
+)
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+# Test URL with Cloudflare protection
+TEST_URL = "https://nowsecure.nl"
+
+async def test_regular_browser():
+    """Test with regular browser - likely to be blocked"""
+    print("=" * 60)
+    print("Testing with Regular Browser")
+    print("=" * 60)
+    
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True,
+        viewport_width=1920,
+        viewport_height=1080,
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        config = CrawlerRunConfig(
+            delay_before_return_html=2.0,
+            simulate_user=True,
+            magic=True,  # Try with magic mode too
+        )
+        
+        result = await crawler.arun(url=TEST_URL, config=config)
+        
+        print(f"\n✓ Success: {result.success}")
+        print(f"✓ Status Code: {result.status_code}")
+        print(f"✓ HTML Length: {len(result.html)}")
+        
+        # Check for Cloudflare challenge
+        if result.html:
+            cf_indicators = [
+                "Checking your browser",
+                "Please stand by",
+                "cloudflare",
+                "cf-browser-verification",
+                "Access denied",
+                "Ray ID"
+            ]
+            
+            detected = False
+            for indicator in cf_indicators:
+                if indicator.lower() in result.html.lower():
+                    print(f"⚠️  Cloudflare Challenge Detected: '{indicator}' found")
+                    detected = True
+                    break
+            
+            if not detected and len(result.markdown.raw_markdown) > 100:
+                print("✅ Successfully bypassed Cloudflare!")
+                print(f"Content preview: {result.markdown.raw_markdown[:200]}...")
+            elif not detected:
+                print("⚠️  Page loaded but content seems minimal")
+        
+        return result
+
+async def test_undetected_browser():
+    """Test with undetected browser - should bypass Cloudflare"""
+    print("\n" + "=" * 60)
+    print("Testing with Undetected Browser")
+    print("=" * 60)
+    
+    browser_config = BrowserConfig(
+        headless=False,  # Headless is easier to detect
+        verbose=True,
+        viewport_width=1920,
+        viewport_height=1080,
+    )
+    
+    # Create undetected adapter
+    undetected_adapter = UndetectedAdapter()
+    
+    # Create strategy with undetected adapter
+    crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=undetected_adapter
+    )
+    
+    async with AsyncWebCrawler(
+        crawler_strategy=crawler_strategy,
+        config=browser_config
+    ) as crawler:
+        config = CrawlerRunConfig(
+            delay_before_return_html=2.0,
+            simulate_user=True,
+        )
+        
+        result = await crawler.arun(url=TEST_URL, config=config)
+        
+        print(f"\n✓ Success: {result.success}")
+        print(f"✓ Status Code: {result.status_code}")
+        print(f"✓ HTML Length: {len(result.html)}")
+        
+        # Check for Cloudflare challenge
+        if result.html:
+            cf_indicators = [
+                "Checking your browser",
+                "Please stand by",
+                "cloudflare",
+                "cf-browser-verification",
+                "Access denied",
+                "Ray ID"
+            ]
+            
+            detected = False
+            for indicator in cf_indicators:
+                if indicator.lower() in result.html.lower():
+                    print(f"⚠️  Cloudflare Challenge Detected: '{indicator}' found")
+                    detected = True
+                    break
+            
+            if not detected and len(result.markdown.raw_markdown) > 100:
+                print("✅ Successfully bypassed Cloudflare!")
+                print(f"Content preview: {result.markdown.raw_markdown[:200]}...")
+            elif not detected:
+                print("⚠️  Page loaded but content seems minimal")
+        
+        return result
+
+async def main():
+    """Compare regular vs undetected browser"""
+    print("🤖 Crawl4AI - Cloudflare Bypass Test")
+    print(f"Testing URL: {TEST_URL}\n")
+    
+    # Test regular browser
+    regular_result = await test_regular_browser()
+    
+    # Small delay
+    await asyncio.sleep(2)
+    
+    # Test undetected browser
+    undetected_result = await test_undetected_browser()
+    
+    # Summary
+    print("\n" + "=" * 60)
+    print("SUMMARY")
+    print("=" * 60)
+    print(f"Regular Browser:")
+    print(f"  - Success: {regular_result.success}")
+    print(f"  - Content Length: {len(regular_result.markdown.raw_markdown) if regular_result.markdown else 0}")
+    
+    print(f"\nUndetected Browser:")
+    print(f"  - Success: {undetected_result.success}")
+    print(f"  - Content Length: {len(undetected_result.markdown.raw_markdown) if undetected_result.markdown else 0}")
+    
+    if undetected_result.success and len(undetected_result.markdown.raw_markdown) > len(regular_result.markdown.raw_markdown):
+        print("\n✅ Undetected browser successfully bypassed protection!")
+    print("=" * 60)
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/undetectability/undetected_vs_regular_comparison.py b/docs/examples/undetectability/undetected_vs_regular_comparison.py
new file mode 100644
index 0000000..80972c0
--- /dev/null
+++ b/docs/examples/undetectability/undetected_vs_regular_comparison.py
@@ -0,0 +1,184 @@
+"""
+Undetected vs Regular Browser Comparison
+This example demonstrates the difference between regular and undetected browser modes
+when accessing sites with bot detection services.
+
+Based on tested anti-bot services:
+- Cloudflare
+- Kasada
+- Akamai
+- DataDome
+- Bet365
+- And others
+"""
+
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    PlaywrightAdapter,
+    UndetectedAdapter,
+    CrawlResult
+)
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+
+# Test URLs for various bot detection services
+TEST_SITES = {
+    "Cloudflare Protected": "https://nowsecure.nl",
+    # "Bot Detection Test": "https://bot.sannysoft.com",
+    # "Fingerprint Test": "https://fingerprint.com/products/bot-detection",
+    # "Browser Scan": "https://browserscan.net",
+    # "CreepJS": "https://abrahamjuliot.github.io/creepjs",
+}
+
+
+async def test_with_adapter(url: str, adapter_name: str, adapter):
+    """Test a URL with a specific adapter"""
+    browser_config = BrowserConfig(
+        headless=False,  # Better for avoiding detection
+        viewport_width=1920,
+        viewport_height=1080,
+        verbose=True,
+    )
+    
+    # Create the crawler strategy with the adapter
+    crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=adapter
+    )
+    
+    print(f"\n{'='*60}")
+    print(f"Testing with {adapter_name} adapter")
+    print(f"URL: {url}")
+    print(f"{'='*60}")
+    
+    try:
+        async with AsyncWebCrawler(
+            crawler_strategy=crawler_strategy,
+            config=browser_config
+        ) as crawler:
+            crawler_config = CrawlerRunConfig(
+                delay_before_return_html=3.0,  # Give page time to load
+                wait_for_images=True,
+                screenshot=True,
+                simulate_user=True,  # Add user simulation
+            )
+            
+            result: CrawlResult = await crawler.arun(
+                url=url,
+                config=crawler_config
+            )
+            
+            # Check results
+            print(f"✓ Status Code: {result.status_code}")
+            print(f"✓ Success: {result.success}")
+            print(f"✓ HTML Length: {len(result.html)}")
+            print(f"✓ Markdown Length: {len(result.markdown.raw_markdown)}")
+            
+            # Check for common bot detection indicators
+            detection_indicators = [
+                "Access denied",
+                "Please verify you are human",
+                "Checking your browser",
+                "Enable JavaScript",
+                "captcha",
+                "403 Forbidden",
+                "Bot detection",
+                "Security check"
+            ]
+            
+            content_lower = result.markdown.raw_markdown.lower()
+            detected = False
+            for indicator in detection_indicators:
+                if indicator.lower() in content_lower:
+                    print(f"⚠️  Possible detection: Found '{indicator}'")
+                    detected = True
+                    break
+            
+            if not detected:
+                print("✅ No obvious bot detection triggered!")
+                # Show first 200 chars of content
+                print(f"Content preview: {result.markdown.raw_markdown[:200]}...")
+            
+            return result.success and not detected
+            
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+        return False
+
+
+async def compare_adapters(url: str, site_name: str):
+    """Compare regular and undetected adapters on the same URL"""
+    print(f"\n{'#'*60}")
+    print(f"# Testing: {site_name}")
+    print(f"{'#'*60}")
+    
+    # Test with regular adapter
+    regular_adapter = PlaywrightAdapter()
+    regular_success = await test_with_adapter(url, "Regular", regular_adapter)
+    
+    # Small delay between tests
+    await asyncio.sleep(2)
+    
+    # Test with undetected adapter
+    undetected_adapter = UndetectedAdapter()
+    undetected_success = await test_with_adapter(url, "Undetected", undetected_adapter)
+    
+    # Summary
+    print(f"\n{'='*60}")
+    print(f"Summary for {site_name}:")
+    print(f"Regular Adapter: {'✅ Passed' if regular_success else '❌ Blocked/Detected'}")
+    print(f"Undetected Adapter: {'✅ Passed' if undetected_success else '❌ Blocked/Detected'}")
+    print(f"{'='*60}")
+    
+    return regular_success, undetected_success
+
+
+async def main():
+    """Run comparison tests on multiple sites"""
+    print("🤖 Crawl4AI Browser Adapter Comparison")
+    print("Testing regular vs undetected browser modes\n")
+    
+    results = {}
+    
+    # Test each site
+    for site_name, url in TEST_SITES.items():
+        regular, undetected = await compare_adapters(url, site_name)
+        results[site_name] = {
+            "regular": regular,
+            "undetected": undetected
+        }
+        
+        # Delay between different sites
+        await asyncio.sleep(3)
+    
+    # Final summary
+    print(f"\n{'#'*60}")
+    print("# FINAL RESULTS")
+    print(f"{'#'*60}")
+    print(f"{'Site':<30} {'Regular':<15} {'Undetected':<15}")
+    print(f"{'-'*60}")
+    
+    for site, result in results.items():
+        regular_status = "✅ Passed" if result["regular"] else "❌ Blocked"
+        undetected_status = "✅ Passed" if result["undetected"] else "❌ Blocked"
+        print(f"{site:<30} {regular_status:<15} {undetected_status:<15}")
+    
+    # Calculate success rates
+    regular_success = sum(1 for r in results.values() if r["regular"])
+    undetected_success = sum(1 for r in results.values() if r["undetected"])
+    total = len(results)
+    
+    print(f"\n{'='*60}")
+    print(f"Success Rates:")
+    print(f"Regular Adapter: {regular_success}/{total} ({regular_success/total*100:.1f}%)")
+    print(f"Undetected Adapter: {undetected_success}/{total} ({undetected_success/total*100:.1f}%)")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    # Note: This example may take a while to run as it tests multiple sites
+    # You can comment out sites in TEST_SITES to run faster tests
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/undetected_simple_demo.py b/docs/examples/undetected_simple_demo.py
new file mode 100644
index 0000000..93954c9
--- /dev/null
+++ b/docs/examples/undetected_simple_demo.py
@@ -0,0 +1,118 @@
+"""
+Simple Undetected Browser Demo
+Demonstrates the basic usage of undetected browser mode
+"""
+
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler, 
+    BrowserConfig, 
+    CrawlerRunConfig,
+    UndetectedAdapter
+)
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+async def crawl_with_regular_browser(url: str):
+    """Crawl with regular browser"""
+    print("\n[Regular Browser Mode]")
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True,
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=url,
+            config=CrawlerRunConfig(
+                delay_before_return_html=2.0
+            )
+        )
+        
+        print(f"Success: {result.success}")
+        print(f"Status: {result.status_code}")
+        print(f"Content length: {len(result.markdown.raw_markdown)}")
+        
+        # Check for bot detection keywords
+        content = result.markdown.raw_markdown.lower()
+        if any(word in content for word in ["cloudflare", "checking your browser", "please wait"]):
+            print("⚠️  Bot detection triggered!")
+        else:
+            print("✅ Page loaded successfully")
+        
+        return result
+
+async def crawl_with_undetected_browser(url: str):
+    """Crawl with undetected browser"""
+    print("\n[Undetected Browser Mode]")
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True,
+    )
+    
+    # Create undetected adapter and strategy
+    undetected_adapter = UndetectedAdapter()
+    crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=undetected_adapter
+    )
+    
+    async with AsyncWebCrawler(
+        crawler_strategy=crawler_strategy,
+        config=browser_config
+    ) as crawler:
+        result = await crawler.arun(
+            url=url,
+            config=CrawlerRunConfig(
+                delay_before_return_html=2.0
+            )
+        )
+        
+        print(f"Success: {result.success}")
+        print(f"Status: {result.status_code}")
+        print(f"Content length: {len(result.markdown.raw_markdown)}")
+        
+        # Check for bot detection keywords
+        content = result.markdown.raw_markdown.lower()
+        if any(word in content for word in ["cloudflare", "checking your browser", "please wait"]):
+            print("⚠️  Bot detection triggered!")
+        else:
+            print("✅ Page loaded successfully")
+        
+        return result
+
+async def main():
+    """Demo comparing regular vs undetected modes"""
+    print("🤖 Crawl4AI Undetected Browser Demo")
+    print("="*50)
+    
+    # Test URLs - you can change these
+    test_urls = [
+        "https://www.example.com",  # Simple site
+        "https://httpbin.org/headers",  # Shows request headers
+    ]
+    
+    for url in test_urls:
+        print(f"\n📍 Testing URL: {url}")
+        
+        # Test with regular browser
+        regular_result = await crawl_with_regular_browser(url)
+        
+        # Small delay
+        await asyncio.sleep(2)
+        
+        # Test with undetected browser
+        undetected_result = await crawl_with_undetected_browser(url)
+        
+        # Compare results
+        print(f"\n📊 Comparison for {url}:")
+        print(f"Regular browser content: {len(regular_result.markdown.raw_markdown)} chars")
+        print(f"Undetected browser content: {len(undetected_result.markdown.raw_markdown)} chars")
+        
+        if url == "https://httpbin.org/headers":
+            # Show headers for comparison
+            print("\nHeaders seen by server:")
+            print("Regular:", regular_result.markdown.raw_markdown[:500])
+            print("\nUndetected:", undetected_result.markdown.raw_markdown[:500])
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/url_seeder/Crawl4AI_URL_Seeder_Tutorial.ipynb b/docs/examples/url_seeder/Crawl4AI_URL_Seeder_Tutorial.ipynb
new file mode 100644
index 0000000..634edf0
--- /dev/null
+++ b/docs/examples/url_seeder/Crawl4AI_URL_Seeder_Tutorial.ipynb
@@ -0,0 +1,1171 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# 🔬 Building an AI Research Assistant with Crawl4AI: Smart URL Discovery\n",
+        "\n",
+        "## Welcome to the Research Pipeline Workshop!\n",
+        "\n",
+        "In this tutorial, we'll build an **AI-powered research assistant** that intelligently discovers, filters, and analyzes web content. Instead of blindly crawling hundreds of pages, we'll use Crawl4AI's URL Seeder to:\n",
+        "\n",
+        "- 🔍 **Discover all available URLs** without crawling them first\n",
+        "- 🎯 **Score and rank** them by relevance using AI\n",
+        "- 🕷️ **Crawl only the most relevant** content\n",
+        "- 🤖 **Generate research insights** with proper citations\n",
+        "\n",
+        "By the end, you'll have a complete research pipeline that can analyze any topic across multiple websites efficiently.\n",
+        "\n",
+        "## What You'll Build\n",
+        "\n",
+        "A **smart research assistant** that:\n",
+        "1. Takes any research query (e.g., \"Premier League transfer news\")\n",
+        "2. Discovers relevant articles from news sites\n",
+        "3. Ranks them by relevance using BM25 scoring\n",
+        "4. Crawls only the top-ranked articles\n",
+        "5. Synthesizes findings into a comprehensive report\n",
+        "\n",
+        "## Prerequisites\n",
+        "\n",
+        "- Python 3.8+ environment\n",
+        "- Basic understanding of async Python\n",
+        "- API keys for LLM (Gemini or OpenAI recommended)\n",
+        "\n",
+        "## Pipeline Overview\n",
+        "\n",
+        "```\n",
+        "User Query → Query Enhancement → URL Discovery → Relevance Scoring → Smart Crawling → AI Synthesis → Research Report\n",
+        "```\n",
+        "\n",
+        "Each step builds on the previous one, creating an efficient research system that saves time and resources.\n",
+        "\n",
+        "Let's begin! 🚀\n",
+        "\n",
+        "---"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 0: Environment Setup and Dependencies\n",
+        "\n",
+        "First, we'll set up our environment with all necessary libraries. We need Crawl4AI for intelligent web crawling, LiteLLM for AI integration, and Rich for beautiful terminal output. This foundation ensures our research assistant has all the tools it needs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "%%capture\n",
+        "!git clone -b next https://github.com/unclecode/crawl4ai.git\n",
+        "!cp -r /content/crawl4ai/docs/apps/linkdin/{snippets,schemas} /content/\n",
+        "%cd crawl4ai\n",
+        "!uv pip install -e .\n",
+        "!crawl4ai-setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!crawl4ai-doctor"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import asyncio\n",
+        "import json\n",
+        "import os\n",
+        "from typing import List, Dict, Optional, Tuple\n",
+        "from dataclasses import dataclass, asdict\n",
+        "from datetime import datetime\n",
+        "from pathlib import Path\n",
+        "\n",
+        "# Rich for beautiful console output\n",
+        "from rich.console import Console\n",
+        "from rich.panel import Panel\n",
+        "from rich.table import Table\n",
+        "from rich.progress import Progress, SpinnerColumn, TextColumn\n",
+        "\n",
+        "# Crawl4AI imports for intelligent crawling\n",
+        "from crawl4ai import (\n",
+        "    AsyncWebCrawler, \n",
+        "    BrowserConfig, \n",
+        "    CrawlerRunConfig,\n",
+        "    AsyncUrlSeeder, \n",
+        "    SeedingConfig,\n",
+        "    AsyncLogger\n",
+        ")\n",
+        "from crawl4ai.content_filter_strategy import PruningContentFilter\n",
+        "from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator\n",
+        "\n",
+        "# LiteLLM for AI capabilities\n",
+        "import litellm\n",
+        "\n",
+        "# Initialize Rich console for pretty output\n",
+        "console = Console()\n",
+        "\n",
+        "print(\"✅ Environment ready! All dependencies loaded successfully.\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 1: Configuration and Data Classes\n",
+        "\n",
+        "Here we define our research pipeline configuration. These dataclasses act as our control center, allowing us to fine-tune every aspect of the research process. Think of them as the settings panel for your research assistant - from discovery limits to AI model choices."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "@dataclass\n",
+        "class ResearchConfig:\n",
+        "    \"\"\"Configuration for the research pipeline\n",
+        "    \n",
+        "    This class controls every aspect of our research assistant:\n",
+        "    - How many URLs to discover and crawl\n",
+        "    - Which scoring methods to use\n",
+        "    - Whether to use AI enhancement\n",
+        "    - Output preferences\n",
+        "    \"\"\"\n",
+        "    # Core settings\n",
+        "    domain: str = \"www.bbc.com/sport\"\n",
+        "    max_urls_discovery: int = 100    # Cast a wide net initially\n",
+        "    max_urls_to_crawl: int = 10      # But only crawl the best\n",
+        "    top_k_urls: int = 10             # Focus on top results\n",
+        "    \n",
+        "    # Scoring and filtering\n",
+        "    score_threshold: float = 0.3     # Minimum relevance score\n",
+        "    scoring_method: str = \"bm25\"     # BM25 is great for relevance\n",
+        "    \n",
+        "    # AI and processing\n",
+        "    use_llm_enhancement: bool = True  # Enhance queries with AI\n",
+        "    llm_model: str = \"gemini/gemini-1.5-flash\"  # Fast and capable\n",
+        "    \n",
+        "    # URL discovery options\n",
+        "    extract_head_metadata: bool = True  # Get titles, descriptions\n",
+        "    live_check: bool = False           # Verify URLs are accessible\n",
+        "    force_refresh: bool = False        # Bypass cache\n",
+        "    \n",
+        "    # Crawler settings\n",
+        "    max_concurrent_crawls: int = 5     # Parallel crawling\n",
+        "    timeout: int = 30000              # 30 second timeout\n",
+        "    headless: bool = True             # No browser window\n",
+        "    \n",
+        "    # Output settings\n",
+        "    output_dir: Path = Path(\"research_results\")\n",
+        "    verbose: bool = True\n",
+        "\n",
+        "@dataclass\n",
+        "class ResearchQuery:\n",
+        "    \"\"\"Container for research query and metadata\"\"\"\n",
+        "    original_query: str\n",
+        "    enhanced_query: Optional[str] = None\n",
+        "    search_patterns: List[str] = None\n",
+        "    timestamp: str = None\n",
+        "\n",
+        "@dataclass\n",
+        "class ResearchResult:\n",
+        "    \"\"\"Container for research results\"\"\"\n",
+        "    query: ResearchQuery\n",
+        "    discovered_urls: List[Dict]\n",
+        "    crawled_content: List[Dict]\n",
+        "    synthesis: str\n",
+        "    citations: List[Dict]\n",
+        "    metadata: Dict\n",
+        "\n",
+        "# Create default configuration\n",
+        "config = ResearchConfig()\n",
+        "console.print(Panel(\n",
+        "    f\"[bold cyan]Research Configuration[/bold cyan]\\n\\n\"\n",
+        "    f\"🌐 Domain: {config.domain}\\n\"\n",
+        "    f\"🔍 Max Discovery: {config.max_urls_discovery} URLs\\n\"\n",
+        "    f\"🕷️ Max Crawl: {config.max_urls_to_crawl} pages\\n\"\n",
+        "    f\"🤖 AI Model: {config.llm_model}\",\n",
+        "    title=\"⚙️ Settings\"\n",
+        "))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 2: Query Enhancement with AI\n",
+        "\n",
+        "Not all search queries are created equal. Here we use AI to transform simple queries into comprehensive search strategies. The LLM analyzes your query, extracts key concepts, and generates related terms - turning \"football news\" into a rich set of search patterns."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "async def enhance_query_with_llm(query: str, config: ResearchConfig) -> ResearchQuery:\n",
+        "    \"\"\"\n",
+        "    Transform simple queries into comprehensive search strategies\n",
+        "    \n",
+        "    Why enhance queries?\n",
+        "    - Users often use simple terms (\"football news\")\n",
+        "    - But relevant content might use varied terminology\n",
+        "    - AI helps capture all relevant variations\n",
+        "    \"\"\"\n",
+        "    console.print(f\"\\n[cyan]🤖 Enhancing query: '{query}'...[/cyan]\")\n",
+        "    \n",
+        "    try:\n",
+        "        # Ask AI to analyze and expand the query\n",
+        "        response = await litellm.acompletion(\n",
+        "            model=config.llm_model,\n",
+        "            messages=[{\n",
+        "                \"role\": \"user\", \n",
+        "                \"content\": f\"\"\"Given this research query: \"{query}\"\n",
+        "                \n",
+        "                Extract:\n",
+        "                1. Key terms and concepts (as a list)\n",
+        "                2. Related search terms\n",
+        "                3. A more specific/enhanced version of the query\n",
+        "                \n",
+        "                Return as JSON:\n",
+        "                {{\n",
+        "                    \"key_terms\": [\"term1\", \"term2\"],\n",
+        "                    \"related_terms\": [\"related1\", \"related2\"],\n",
+        "                    \"enhanced_query\": \"enhanced version of query\"\n",
+        "                }}\"\"\"\n",
+        "            }],\n",
+        "            temperature=0.3,  # Low temperature for consistency\n",
+        "            response_format={\"type\": \"json_object\"}\n",
+        "        )\n",
+        "        \n",
+        "        data = json.loads(response.choices[0].message.content)\n",
+        "        \n",
+        "        # Create search patterns from extracted terms\n",
+        "        # These patterns help the URL seeder find relevant pages\n",
+        "        all_terms = data[\"key_terms\"] + data[\"related_terms\"]\n",
+        "        patterns = [f\"*{term.lower()}*\" for term in all_terms]\n",
+        "        \n",
+        "        result = ResearchQuery(\n",
+        "            original_query=query,\n",
+        "            enhanced_query=data[\"enhanced_query\"],\n",
+        "            search_patterns=patterns[:10],  # Limit to 10 patterns\n",
+        "            timestamp=datetime.now().isoformat()\n",
+        "        )\n",
+        "        \n",
+        "        # Show the enhancement\n",
+        "        console.print(Panel(\n",
+        "            f\"[green]✅ Enhanced Query:[/green] {result.enhanced_query}\\n\"\n",
+        "            f\"[dim]Key terms: {', '.join(data['key_terms'])}[/dim]\",\n",
+        "            title=\"🔍 Query Enhancement\"\n",
+        "        ))\n",
+        "        \n",
+        "        return result\n",
+        "        \n",
+        "    except Exception as e:\n",
+        "        console.print(f\"[yellow]⚠️ Enhancement failed, using original query: {e}[/yellow]\")\n",
+        "        # Fallback to simple tokenization\n",
+        "        words = query.lower().split()\n",
+        "        patterns = [f\"*{word}*\" for word in words if len(word) > 2]\n",
+        "        \n",
+        "        return ResearchQuery(\n",
+        "            original_query=query,\n",
+        "            enhanced_query=query,\n",
+        "            search_patterns=patterns,\n",
+        "            timestamp=datetime.now().isoformat()\n",
+        "        )\n",
+        "\n",
+        "# Example usage\n",
+        "test_query = \"Premier League transfer news\"\n",
+        "enhanced = await enhance_query_with_llm(test_query, config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 3: Smart URL Discovery with AsyncUrlSeeder\n",
+        "\n",
+        "This is where the magic begins! Instead of crawling pages to find links, AsyncUrlSeeder discovers URLs from sitemaps and Common Crawl data. It's like having a map of the entire website before you start exploring. We'll discover hundreds of URLs in seconds, complete with metadata."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "async def discover_urls(\n",
+        "    domain: str, \n",
+        "    query: ResearchQuery, \n",
+        "    config: ResearchConfig\n",
+        ") -> List[Dict]:\n",
+        "    \"\"\"\n",
+        "    Discover and rank URLs without crawling them\n",
+        "    \n",
+        "    The URL Seeder is incredibly powerful because it:\n",
+        "    1. Gets URLs from sitemaps (official site maps)\n",
+        "    2. Gets URLs from Common Crawl (web-scale data)\n",
+        "    3. Extracts metadata without full page loads\n",
+        "    4. Scores relevance using BM25 algorithm\n",
+        "    \n",
+        "    This means we know which pages are worth crawling\n",
+        "    BEFORE we spend time crawling them!\n",
+        "    \"\"\"\n",
+        "    console.print(f\"\\n[cyan]🔍 Discovering URLs from {domain}...[/cyan]\")\n",
+        "    \n",
+        "    # Use context manager for automatic cleanup\n",
+        "    async with AsyncUrlSeeder(logger=AsyncLogger(verbose=config.verbose)) as seeder:\n",
+        "        # Configure the discovery process\n",
+        "        seeding_config = SeedingConfig(\n",
+        "            # Data sources\n",
+        "            source=\"sitemap+cc\",  # Use both sitemap AND Common Crawl\n",
+        "            \n",
+        "            # Metadata extraction\n",
+        "            extract_head=config.extract_head_metadata,  # Get titles, descriptions\n",
+        "            \n",
+        "            # Relevance scoring\n",
+        "            query=query.enhanced_query or query.original_query,\n",
+        "            scoring_method=config.scoring_method,  # BM25 scoring\n",
+        "            score_threshold=config.score_threshold,  # Minimum score\n",
+        "            \n",
+        "            # Limits and performance\n",
+        "            max_urls=config.max_urls_discovery,\n",
+        "            live_check=config.live_check,  # Verify URLs work\n",
+        "            force=config.force_refresh,  # Bypass cache if needed\n",
+        "            \n",
+        "            # Performance tuning\n",
+        "            concurrency=20,  # Parallel workers\n",
+        "        )\n",
+        "        \n",
+        "        try:\n",
+        "            # Discover URLs - this is FAST!\n",
+        "            urls = await seeder.urls(domain, seeding_config)\n",
+        "            \n",
+        "            # Results are already sorted by relevance\n",
+        "            # thanks to BM25 scoring\n",
+        "            top_urls = urls[:config.top_k_urls]\n",
+        "            \n",
+        "            # Show discovery results\n",
+        "            console.print(f\"[green]✅ Discovered {len(urls)} URLs, selected top {len(top_urls)}[/green]\")\n",
+        "            \n",
+        "            # Display a sample of what we found\n",
+        "            if top_urls:\n",
+        "                table = Table(title=\"🎯 Top Discovered URLs\")\n",
+        "                table.add_column(\"Score\", style=\"cyan\")\n",
+        "                table.add_column(\"Title\", style=\"green\")\n",
+        "                table.add_column(\"URL\", style=\"dim\")\n",
+        "                \n",
+        "                for url in top_urls[:5]:\n",
+        "                    score = f\"{url.get('relevance_score', 0):.3f}\"\n",
+        "                    title = \"N/A\"\n",
+        "                    if url.get('head_data') and url['head_data'].get('title'):\n",
+        "                        title = url['head_data']['title'][:50] + \"...\"\n",
+        "                    url_str = url['url'][:60] + \"...\"\n",
+        "                    \n",
+        "                    table.add_row(score, title, url_str)\n",
+        "                \n",
+        "                console.print(table)\n",
+        "            \n",
+        "            return top_urls\n",
+        "            \n",
+        "        except Exception as e:\n",
+        "            console.print(f\"[red]❌ URL discovery failed: {e}[/red]\")\n",
+        "            return []\n",
+        "\n",
+        "# Example discovery\n",
+        "discovered = await discover_urls(config.domain, enhanced, config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 4: Intelligent Content Crawling\n",
+        "\n",
+        "Now we crawl only the most relevant URLs. This is where our smart filtering pays off - instead of crawling hundreds of pages, we focus on the top 10-20 most relevant ones. We use content filtering to extract only the meaningful text, removing ads and navigation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "async def crawl_selected_urls(\n",
+        "    urls: List[Dict], \n",
+        "    query: ResearchQuery, \n",
+        "    config: ResearchConfig\n",
+        ") -> List[Dict]:\n",
+        "    \"\"\"\n",
+        "    Crawl only the most relevant URLs with smart content filtering\n",
+        "    \n",
+        "    Key optimizations:\n",
+        "    1. We already know these URLs are relevant (from scoring)\n",
+        "    2. We crawl them in parallel for speed\n",
+        "    3. We extract only meaningful content (no ads/nav)\n",
+        "    4. We generate clean markdown for analysis\n",
+        "    \"\"\"\n",
+        "    # Extract URLs from discovery results\n",
+        "    url_list = [u['url'] for u in urls if 'url' in u][:config.max_urls_to_crawl]\n",
+        "    \n",
+        "    if not url_list:\n",
+        "        console.print(\"[red]❌ No URLs to crawl[/red]\")\n",
+        "        return []\n",
+        "    \n",
+        "    console.print(f\"\\n[cyan]🕷️ Crawling {len(url_list)} URLs...[/cyan]\")\n",
+        "    \n",
+        "    # Configure intelligent content extraction\n",
+        "    # This removes ads, navigation, and other noise\n",
+        "    md_generator = DefaultMarkdownGenerator(\n",
+        "        content_filter=PruningContentFilter(\n",
+        "            threshold=0.48,  # Content relevance threshold\n",
+        "            threshold_type=\"dynamic\",  # Adapts to page structure\n",
+        "            min_word_threshold=10  # Ignore tiny text blocks\n",
+        "        ),\n",
+        "    )\n",
+        "    \n",
+        "    # Configure the crawler\n",
+        "    crawler_config = CrawlerRunConfig(\n",
+        "        markdown_generator=md_generator,\n",
+        "        exclude_external_links=True,  # Focus on content, not links\n",
+        "        excluded_tags=['nav', 'header', 'footer', 'aside'],  # Skip UI elements\n",
+        "    )\n",
+        "    \n",
+        "    # Create crawler with browser config\n",
+        "    async with AsyncWebCrawler(\n",
+        "        config=BrowserConfig(\n",
+        "            headless=config.headless,\n",
+        "            verbose=config.verbose\n",
+        "        )\n",
+        "    ) as crawler:\n",
+        "        # Crawl URLs in parallel for speed\n",
+        "        # arun_many handles concurrency automatically\n",
+        "        results = await crawler.arun_many(\n",
+        "            url_list,\n",
+        "            config=crawler_config,\n",
+        "            max_concurrent=config.max_concurrent_crawls\n",
+        "        )\n",
+        "        \n",
+        "        # Process successful results\n",
+        "        crawled_content = []\n",
+        "        for url, result in zip(url_list, results):\n",
+        "            if result.success:\n",
+        "                # Extract the content we need\n",
+        "                content_data = {\n",
+        "                    'url': url,\n",
+        "                    'title': result.metadata.get('title', 'No title'),\n",
+        "                    'markdown': result.markdown.fit_markdown or result.markdown.raw_markdown,\n",
+        "                    'metadata': result.metadata\n",
+        "                }\n",
+        "                crawled_content.append(content_data)\n",
+        "                console.print(f\"  [green]✓[/green] Crawled: {url[:60]}...\")\n",
+        "            else:\n",
+        "                console.print(f\"  [red]✗[/red] Failed: {url[:50]}... - {result.error}\")\n",
+        "    \n",
+        "    console.print(f\"[green]✅ Successfully crawled {len(crawled_content)} pages[/green]\")\n",
+        "    return crawled_content\n",
+        "\n",
+        "# Example crawling\n",
+        "crawled = await crawl_selected_urls(discovered[:5], enhanced, config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 5: AI-Powered Research Synthesis\n",
+        "\n",
+        "This is where we transform raw content into insights. The AI analyzes all crawled articles, identifies key themes, and generates a comprehensive synthesis with proper citations. It's like having a research assistant read everything and write you a summary."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "async def generate_research_synthesis(\n",
+        "    query: ResearchQuery,\n",
+        "    crawled_content: List[Dict],\n",
+        "    config: ResearchConfig\n",
+        ") -> Tuple[str, List[Dict]]:\n",
+        "    \"\"\"\n",
+        "    Use AI to synthesize findings from multiple sources\n",
+        "    \n",
+        "    The synthesis process:\n",
+        "    1. Sends all content to the LLM\n",
+        "    2. Asks for key findings and analysis\n",
+        "    3. Ensures proper citation of sources\n",
+        "    4. Generates actionable insights\n",
+        "    \"\"\"\n",
+        "    if not crawled_content:\n",
+        "        return \"No content available for synthesis.\", []\n",
+        "    \n",
+        "    console.print(\"\\n[cyan]🤖 Generating research synthesis...[/cyan]\")\n",
+        "    \n",
+        "    # Prepare content for the AI\n",
+        "    # We include source info for proper citations\n",
+        "    content_sections = []\n",
+        "    for i, content in enumerate(crawled_content, 1):\n",
+        "        section = f\"\"\"\n",
+        "SOURCE {i}:\n",
+        "Title: {content['title']}\n",
+        "URL: {content['url']}\n",
+        "Content Preview:\n",
+        "{content['markdown'][:1500]}...\n",
+        "\"\"\"\n",
+        "        content_sections.append(section)\n",
+        "    \n",
+        "    combined_content = \"\\n---\\n\".join(content_sections)\n",
+        "    \n",
+        "    try:\n",
+        "        # Generate comprehensive synthesis\n",
+        "        response = await litellm.acompletion(\n",
+        "            model=config.llm_model,\n",
+        "            messages=[{\n",
+        "                \"role\": \"user\",\n",
+        "                \"content\": f\"\"\"Research Query: \"{query.original_query}\"\n",
+        "\n",
+        "Based on the following sources, provide a comprehensive research synthesis.\n",
+        "\n",
+        "{combined_content}\n",
+        "\n",
+        "Please provide:\n",
+        "1. An executive summary (2-3 sentences)\n",
+        "2. Key findings (3-5 bullet points)\n",
+        "3. Detailed analysis (2-3 paragraphs)\n",
+        "4. Future implications or trends\n",
+        "\n",
+        "Format your response with clear sections and cite sources using [Source N] notation.\n",
+        "Keep the total response under 800 words.\"\"\"\n",
+        "            }],\n",
+        "            temperature=0.7  # Some creativity for synthesis\n",
+        "        )\n",
+        "        \n",
+        "        synthesis = response.choices[0].message.content\n",
+        "        \n",
+        "        # Extract citations from the synthesis\n",
+        "        citations = []\n",
+        "        for i, content in enumerate(crawled_content, 1):\n",
+        "            # Check if this source was cited\n",
+        "            if f\"[Source {i}]\" in synthesis or f\"Source {i}\" in synthesis:\n",
+        "                citations.append({\n",
+        "                    'source_id': i,\n",
+        "                    'title': content['title'],\n",
+        "                    'url': content['url']\n",
+        "                })\n",
+        "        \n",
+        "        return synthesis, citations\n",
+        "        \n",
+        "    except Exception as e:\n",
+        "        console.print(f\"[red]❌ Synthesis generation failed: {e}[/red]\")\n",
+        "        # Fallback to simple summary\n",
+        "        summary = f\"Research on '{query.original_query}' found {len(crawled_content)} relevant articles:\\n\\n\"\n",
+        "        for content in crawled_content[:3]:\n",
+        "            summary += f\"- {content['title']}\\n  {content['url']}\\n\\n\"\n",
+        "        return summary, []\n",
+        "\n",
+        "# Example synthesis\n",
+        "synthesis, citations = await generate_research_synthesis(enhanced, crawled, config)\n",
+        "console.print(Panel(synthesis[:500] + \"...\", title=\"📝 Research Synthesis Preview\"))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 6: Complete Research Pipeline\n",
+        "\n",
+        "Now let's put it all together! This orchestrator function manages the entire research pipeline from query to final report. It coordinates all the components we've built, handling errors gracefully and providing progress updates."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "async def research_pipeline(\n",
+        "    query: str,\n",
+        "    config: ResearchConfig = None\n",
+        ") -> ResearchResult:\n",
+        "    \"\"\"\n",
+        "    Main research pipeline orchestrator\n",
+        "    \n",
+        "    This brings together all components:\n",
+        "    1. Query enhancement (AI-powered)\n",
+        "    2. URL discovery (AsyncUrlSeeder)\n",
+        "    3. Smart crawling (AsyncWebCrawler)\n",
+        "    4. AI synthesis (LiteLLM)\n",
+        "    \n",
+        "    Returns a complete research result\n",
+        "    \"\"\"\n",
+        "    if config is None:\n",
+        "        config = ResearchConfig()\n",
+        "    \n",
+        "    start_time = datetime.now()\n",
+        "    \n",
+        "    # Display pipeline header\n",
+        "    console.print(Panel(\n",
+        "        f\"[bold cyan]Research Pipeline[/bold cyan]\\n\\n\"\n",
+        "        f\"[dim]Query:[/dim] {query}\\n\"\n",
+        "        f\"[dim]Domain:[/dim] {config.domain}\",\n",
+        "        title=\"🚀 Starting Research\",\n",
+        "        border_style=\"cyan\"\n",
+        "    ))\n",
+        "    \n",
+        "    # Step 1: Enhance query\n",
+        "    console.print(f\"\\n[bold cyan]📝 Step 1: Query Processing[/bold cyan]\")\n",
+        "    if config.use_llm_enhancement:\n",
+        "        research_query = await enhance_query_with_llm(query, config)\n",
+        "    else:\n",
+        "        # Simple fallback without AI\n",
+        "        research_query = ResearchQuery(\n",
+        "            original_query=query,\n",
+        "            enhanced_query=query,\n",
+        "            search_patterns=[f\"*{word}*\" for word in query.lower().split()],\n",
+        "            timestamp=datetime.now().isoformat()\n",
+        "        )\n",
+        "    \n",
+        "    # Step 2: Discover URLs\n",
+        "    console.print(f\"\\n[bold cyan]🔍 Step 2: URL Discovery[/bold cyan]\")\n",
+        "    discovered_urls = await discover_urls(\n",
+        "        domain=config.domain,\n",
+        "        query=research_query,\n",
+        "        config=config\n",
+        "    )\n",
+        "    \n",
+        "    if not discovered_urls:\n",
+        "        # No URLs found - return empty result\n",
+        "        return ResearchResult(\n",
+        "            query=research_query,\n",
+        "            discovered_urls=[],\n",
+        "            crawled_content=[],\n",
+        "            synthesis=\"No relevant URLs found for the given query.\",\n",
+        "            citations=[],\n",
+        "            metadata={'duration': str(datetime.now() - start_time)}\n",
+        "        )\n",
+        "    \n",
+        "    # Step 3: Crawl selected URLs\n",
+        "    console.print(f\"\\n[bold cyan]🕷️ Step 3: Content Crawling[/bold cyan]\")\n",
+        "    crawled_content = await crawl_selected_urls(\n",
+        "        urls=discovered_urls,\n",
+        "        query=research_query,\n",
+        "        config=config\n",
+        "    )\n",
+        "    \n",
+        "    # Step 4: Generate synthesis\n",
+        "    console.print(f\"\\n[bold cyan]🤖 Step 4: Synthesis Generation[/bold cyan]\")\n",
+        "    synthesis, citations = await generate_research_synthesis(\n",
+        "        query=research_query,\n",
+        "        crawled_content=crawled_content,\n",
+        "        config=config\n",
+        "    )\n",
+        "    \n",
+        "    # Create final result\n",
+        "    result = ResearchResult(\n",
+        "        query=research_query,\n",
+        "        discovered_urls=discovered_urls,\n",
+        "        crawled_content=crawled_content,\n",
+        "        synthesis=synthesis,\n",
+        "        citations=citations,\n",
+        "        metadata={\n",
+        "            'duration': str(datetime.now() - start_time),\n",
+        "            'domain': config.domain,\n",
+        "            'timestamp': datetime.now().isoformat(),\n",
+        "            'total_discovered': len(discovered_urls),\n",
+        "            'total_crawled': len(crawled_content),\n",
+        "            'total_cited': len(citations)\n",
+        "        }\n",
+        "    )\n",
+        "    \n",
+        "    # Display summary\n",
+        "    duration = datetime.now() - start_time\n",
+        "    console.print(Panel(\n",
+        "        f\"[bold green]✅ Research completed in {duration}[/bold green]\\n\\n\"\n",
+        "        f\"📊 Discovered: {len(discovered_urls)} URLs\\n\"\n",
+        "        f\"🕷️ Crawled: {len(crawled_content)} pages\\n\"\n",
+        "        f\"📚 Citations: {len(citations)} sources\",\n",
+        "        title=\"🎉 Pipeline Complete\",\n",
+        "        border_style=\"green\"\n",
+        "    ))\n",
+        "    \n",
+        "    return result\n",
+        "\n",
+        "# Example: Run complete pipeline\n",
+        "result = await research_pipeline(\"Champions League latest results\", config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 7: Beautiful Output Formatting\n",
+        "\n",
+        "A good research report needs clear presentation. Here we format our results into a professional report with executive summary, key findings, and proper citations. This makes the research actionable and easy to share."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "def format_research_output(result: ResearchResult) -> None:\n",
+        "    \"\"\"\n",
+        "    Create a beautifully formatted research report\n",
+        "    \n",
+        "    Good formatting makes insights actionable:\n",
+        "    - Clear structure with sections\n",
+        "    - Highlighted key findings\n",
+        "    - Proper source attribution\n",
+        "    - Easy to scan and understand\n",
+        "    \"\"\"\n",
+        "    # Header\n",
+        "    console.print(\"\\n\" + \"=\" * 60)\n",
+        "    console.print(\"[bold cyan]🔬 RESEARCH REPORT[/bold cyan]\")\n",
+        "    console.print(\"=\" * 60)\n",
+        "    \n",
+        "    # Query information\n",
+        "    console.print(f\"\\n[bold]Query:[/bold] {result.query.original_query}\")\n",
+        "    if result.query.enhanced_query != result.query.original_query:\n",
+        "        console.print(f\"[dim]Enhanced: {result.query.enhanced_query}[/dim]\")\n",
+        "    \n",
+        "    # Statistics\n",
+        "    stats_table = Table(show_header=False, box=None)\n",
+        "    stats_table.add_column(style=\"cyan\")\n",
+        "    stats_table.add_column()\n",
+        "    \n",
+        "    stats_table.add_row(\"📊 URLs Discovered\", str(result.metadata['total_discovered']))\n",
+        "    stats_table.add_row(\"🕷️ Pages Crawled\", str(result.metadata['total_crawled']))\n",
+        "    stats_table.add_row(\"📚 Sources Cited\", str(result.metadata['total_cited']))\n",
+        "    stats_table.add_row(\"⏱️ Processing Time\", result.metadata['duration'])\n",
+        "    \n",
+        "    console.print(\"\\n[bold]Statistics:[/bold]\")\n",
+        "    console.print(stats_table)\n",
+        "    \n",
+        "    # Synthesis\n",
+        "    console.print(\"\\n[bold]📝 SYNTHESIS[/bold]\")\n",
+        "    console.print(\"-\" * 60)\n",
+        "    console.print(result.synthesis)\n",
+        "    \n",
+        "    # Citations\n",
+        "    if result.citations:\n",
+        "        console.print(\"\\n[bold]📚 SOURCES[/bold]\")\n",
+        "        console.print(\"-\" * 60)\n",
+        "        for citation in result.citations:\n",
+        "            console.print(f\"\\n[{citation['source_id']}] [cyan]{citation['title']}[/cyan]\")\n",
+        "            console.print(f\"    [dim]{citation['url']}[/dim]\")\n",
+        "    \n",
+        "    # Top discovered URLs\n",
+        "    console.print(\"\\n[bold]🔍 TOP DISCOVERED URLS[/bold]\")\n",
+        "    console.print(\"-\" * 60)\n",
+        "    \n",
+        "    urls_table = Table()\n",
+        "    urls_table.add_column(\"Score\", style=\"cyan\")\n",
+        "    urls_table.add_column(\"Title\")\n",
+        "    urls_table.add_column(\"URL\", style=\"dim\")\n",
+        "    \n",
+        "    for url_data in result.discovered_urls[:5]:\n",
+        "        score = f\"{url_data.get('relevance_score', 0):.3f}\"\n",
+        "        title = \"N/A\"\n",
+        "        if url_data.get('head_data') and url_data['head_data'].get('title'):\n",
+        "            title = url_data['head_data']['title'][:40] + \"...\"\n",
+        "        url = url_data['url'][:50] + \"...\"\n",
+        "        \n",
+        "        urls_table.add_row(score, title, url)\n",
+        "    \n",
+        "    console.print(urls_table)\n",
+        "\n",
+        "# Display the formatted report\n",
+        "format_research_output(result)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 8: Save Research Results\n",
+        "\n",
+        "Finally, let's save our research for future reference. We'll create both JSON (for data analysis) and Markdown (for reading) formats. This ensures your research is preserved and shareable."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "async def save_research_results(\n",
+        "    result: ResearchResult, \n",
+        "    config: ResearchConfig\n",
+        ") -> Tuple[Path, Path]:\n",
+        "    \"\"\"\n",
+        "    Save research results in multiple formats\n",
+        "    \n",
+        "    Why save in multiple formats?\n",
+        "    - JSON: Perfect for further analysis or automation\n",
+        "    - Markdown: Human-readable, great for sharing\n",
+        "    \"\"\"\n",
+        "    # Create output directory\n",
+        "    config.output_dir.mkdir(parents=True, exist_ok=True)\n",
+        "    \n",
+        "    # Generate filename based on query and timestamp\n",
+        "    timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    query_slug = result.query.original_query[:30].replace(\" \", \"_\").replace(\"/\", \"_\")\n",
+        "    base_filename = f\"{timestamp}_{query_slug}\"\n",
+        "    \n",
+        "    # Save JSON\n",
+        "    json_path = config.output_dir / f\"{base_filename}.json\"\n",
+        "    with open(json_path, 'w') as f:\n",
+        "        json.dump(asdict(result), f, indent=2, default=str)\n",
+        "    \n",
+        "    # Create markdown report\n",
+        "    md_content = [\n",
+        "        f\"# Research Report: {result.query.original_query}\",\n",
+        "        f\"\\n**Generated on:** {result.metadata.get('timestamp', 'N/A')}\",\n",
+        "        f\"\\n**Domain:** {result.metadata.get('domain', 'N/A')}\",\n",
+        "        f\"\\n**Processing time:** {result.metadata.get('duration', 'N/A')}\",\n",
+        "        \"\\n---\\n\",\n",
+        "        \"## Query Information\",\n",
+        "        f\"- **Original Query:** {result.query.original_query}\",\n",
+        "        f\"- **Enhanced Query:** {result.query.enhanced_query or 'N/A'}\",\n",
+        "        \"\\n## Statistics\",\n",
+        "        f\"- **URLs Discovered:** {result.metadata['total_discovered']}\",\n",
+        "        f\"- **Pages Crawled:** {result.metadata['total_crawled']}\",\n",
+        "        f\"- **Sources Cited:** {result.metadata['total_cited']}\",\n",
+        "        \"\\n## Research Synthesis\\n\",\n",
+        "        result.synthesis,\n",
+        "        \"\\n## Sources\\n\"\n",
+        "    ]\n",
+        "    \n",
+        "    # Add citations\n",
+        "    for citation in result.citations:\n",
+        "        md_content.extend([\n",
+        "            f\"### [{citation['source_id']}] {citation['title']}\",\n",
+        "            f\"- **URL:** [{citation['url']}]({citation['url']})\",\n",
+        "            \"\"\n",
+        "        ])\n",
+        "    \n",
+        "    # Add discovered URLs\n",
+        "    md_content.extend([\n",
+        "        \"\\n## Discovered URLs (Top 10)\\n\",\n",
+        "        \"| Score | Title | URL |\",\n",
+        "        \"|-------|-------|-----|\"\n",
+        "    ])\n",
+        "    \n",
+        "    for url_data in result.discovered_urls[:10]:\n",
+        "        score = url_data.get('relevance_score', 0)\n",
+        "        title = 'N/A'\n",
+        "        if url_data.get('head_data') and url_data['head_data'].get('title'):\n",
+        "            title = url_data['head_data']['title'][:50] + '...'\n",
+        "        url = url_data['url'][:60] + '...'\n",
+        "        md_content.append(f\"| {score:.3f} | {title} | {url} |\")\n",
+        "    \n",
+        "    # Save markdown\n",
+        "    md_path = config.output_dir / f\"{base_filename}.md\"\n",
+        "    with open(md_path, 'w') as f:\n",
+        "        f.write('\\n'.join(md_content))\n",
+        "    \n",
+        "    console.print(f\"\\n[green]💾 Results saved:[/green]\")\n",
+        "    console.print(f\"  JSON: {json_path}\")\n",
+        "    console.print(f\"  Markdown: {md_path}\")\n",
+        "    \n",
+        "    return json_path, md_path\n",
+        "\n",
+        "# Save our results\n",
+        "json_path, md_path = await save_research_results(result, config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## 🎯 Putting It All Together: Interactive Research Assistant\n",
+        "\n",
+        "Now let's create an interactive version where you can research any topic! This brings together everything we've learned into a user-friendly tool."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "async def interactive_research_assistant():\n",
+        "    \"\"\"\n",
+        "    Interactive research assistant with example queries\n",
+        "    \n",
+        "    This demonstrates how to build a user-friendly interface\n",
+        "    for your research pipeline.\n",
+        "    \"\"\"\n",
+        "    # Welcome message\n",
+        "    console.print(Panel.fit(\n",
+        "        \"[bold cyan]🔬 AI Research Assistant[/bold cyan]\\n\\n\"\n",
+        "        \"Powered by Crawl4AI's intelligent URL discovery\\n\"\n",
+        "        \"[dim]• Discover without crawling\\n\"\n",
+        "        \"• Score by relevance\\n\"\n",
+        "        \"• Crawl only what matters\\n\"\n",
+        "        \"• Generate AI insights[/dim]\",\n",
+        "        title=\"Welcome\",\n",
+        "        border_style=\"cyan\"\n",
+        "    ))\n",
+        "    \n",
+        "    # Example queries\n",
+        "    examples = [\n",
+        "        \"Premier League transfer news and rumors\",\n",
+        "        \"Champions League match results and analysis\",\n",
+        "        \"Tennis grand slam tournament updates\",\n",
+        "        \"Formula 1 race results and standings\",\n",
+        "        \"NBA playoff predictions and analysis\"\n",
+        "    ]\n",
+        "    \n",
+        "    # Display examples\n",
+        "    console.print(\"\\n[bold]📋 Example queries:[/bold]\")\n",
+        "    for i, example in enumerate(examples, 1):\n",
+        "        console.print(f\"  {i}. {example}\")\n",
+        "    \n",
+        "    # Get user input\n",
+        "    console.print(\"\\n[bold]Enter a number (1-5) or type your own query:[/bold]\")\n",
+        "    user_input = input(\"🔍 > \").strip()\n",
+        "    \n",
+        "    # Determine query\n",
+        "    if user_input.isdigit() and 1 <= int(user_input) <= len(examples):\n",
+        "        query = examples[int(user_input) - 1]\n",
+        "    else:\n",
+        "        query = user_input if user_input else examples[0]\n",
+        "    \n",
+        "    console.print(f\"\\n[cyan]Selected query: {query}[/cyan]\")\n",
+        "    \n",
+        "    # Configuration options\n",
+        "    console.print(\"\\n[bold]Choose configuration:[/bold]\")\n",
+        "    console.print(\"  1. Quick (5 URLs, fast)\")\n",
+        "    console.print(\"  2. Standard (10 URLs, balanced)\")\n",
+        "    console.print(\"  3. Comprehensive (20 URLs, thorough)\")\n",
+        "    \n",
+        "    config_choice = input(\"⚙️ > \").strip()\n",
+        "    \n",
+        "    # Create configuration\n",
+        "    if config_choice == \"1\":\n",
+        "        config = ResearchConfig(max_urls_to_crawl=5, top_k_urls=5)\n",
+        "    elif config_choice == \"3\":\n",
+        "        config = ResearchConfig(max_urls_to_crawl=20, top_k_urls=20)\n",
+        "    else:\n",
+        "        config = ResearchConfig()  # Standard\n",
+        "    \n",
+        "    # Run research\n",
+        "    result = await research_pipeline(query, config)\n",
+        "    \n",
+        "    # Display results\n",
+        "    format_research_output(result)\n",
+        "    \n",
+        "    # Save results\n",
+        "    save_choice = input(\"\\n💾 Save results? (y/n): \").strip().lower()\n",
+        "    if save_choice == 'y':\n",
+        "        await save_research_results(result, config)\n",
+        "\n",
+        "# Run the interactive assistant\n",
+        "await interactive_research_assistant()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## 🚀 Advanced Tips and Best Practices\n",
+        "\n",
+        "### 1. Domain-Specific Research\n",
+        "\n",
+        "Customize the pipeline for specific domains:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Research across multiple sports sites\n",
+        "async def multi_domain_research(query: str):\n",
+        "    \"\"\"Research across multiple sports websites\"\"\"\n",
+        "    \n",
+        "    domains = [\n",
+        "        \"www.bbc.com/sport\",\n",
+        "        \"www.espn.com\",\n",
+        "        \"www.skysports.com\"\n",
+        "    ]\n",
+        "    \n",
+        "    all_results = []\n",
+        "    \n",
+        "    for domain in domains:\n",
+        "        config = ResearchConfig(\n",
+        "            domain=domain,\n",
+        "            max_urls_to_crawl=5  # 5 per domain\n",
+        "        )\n",
+        "        \n",
+        "        console.print(f\"\\n[cyan]Researching {domain}...[/cyan]\")\n",
+        "        result = await research_pipeline(query, config)\n",
+        "        all_results.append(result)\n",
+        "    \n",
+        "    # Combine insights from all domains\n",
+        "    console.print(\"\\n[bold green]✅ Multi-domain research complete![/bold green]\")\n",
+        "    return all_results\n",
+        "\n",
+        "# Example usage\n",
+        "# results = await multi_domain_research(\"World Cup 2024\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### 2. Performance Optimization\n",
+        "\n",
+        "Tips for faster research:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Optimized configuration for speed\n",
+        "speed_config = ResearchConfig(\n",
+        "    # Reduce discovery scope\n",
+        "    max_urls_discovery=50,  # Don't discover too many\n",
+        "    \n",
+        "    # Skip live checking (trust the sitemap)\n",
+        "    live_check=False,\n",
+        "    \n",
+        "    # Increase parallelism\n",
+        "    max_concurrent_crawls=10,\n",
+        "    \n",
+        "    # Skip AI enhancement for simple queries\n",
+        "    use_llm_enhancement=False,\n",
+        "    \n",
+        "    # Use faster model\n",
+        "    llm_model=\"gemini/gemini-1.5-flash\"\n",
+        ")\n",
+        "\n",
+        "console.print(Panel(\n",
+        "    \"[green]⚡ Speed Optimizations:[/green]\\n\\n\"\n",
+        "    \"• Reduced discovery scope\\n\"\n",
+        "    \"• Disabled live URL checking\\n\"\n",
+        "    \"• Increased parallelism\\n\"\n",
+        "    \"• Using faster AI model\",\n",
+        "    title=\"Performance Tips\"\n",
+        "))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### 3. Caching Strategy\n",
+        "\n",
+        "The URL Seeder automatically caches results for efficiency:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Cache demonstration\n",
+        "console.print(\"[bold]🗄️ Understanding Caching:[/bold]\\n\")\n",
+        "\n",
+        "console.print(\"1. [cyan]First run:[/cyan] Fetches fresh data\")\n",
+        "console.print(\"   - Discovers URLs from sitemap/Common Crawl\")\n",
+        "console.print(\"   - Extracts metadata\")\n",
+        "console.print(\"   - Caches results for 7 days\")\n",
+        "\n",
+        "console.print(\"\\n2. [cyan]Subsequent runs:[/cyan] Uses cache (instant!)\")\n",
+        "console.print(\"   - No network requests needed\")\n",
+        "console.print(\"   - Same query returns cached results\")\n",
+        "\n",
+        "console.print(\"\\n3. [cyan]Force refresh:[/cyan] Bypass cache when needed\")\n",
+        "console.print(\"   - Set `force_refresh=True` in config\")\n",
+        "console.print(\"   - Useful for breaking news or updates\")\n",
+        "\n",
+        "# Example with cache control\n",
+        "cache_config = ResearchConfig(\n",
+        "    force_refresh=True  # Always get fresh data\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## 🎓 Summary & Next Steps\n",
+        "\n",
+        "### What You've Learned\n",
+        "\n",
+        "You've built a complete AI research assistant that:\n",
+        "\n",
+        "✅ **Discovers URLs intelligently** - No blind crawling  \n",
+        "✅ **Scores by relevance** - Focus on what matters  \n",
+        "✅ **Crawls efficiently** - Parallel processing  \n",
+        "✅ **Generates insights** - AI-powered synthesis  \n",
+        "✅ **Saves results** - JSON and Markdown formats  \n",
+        "\n",
+        "### Key Advantages\n",
+        "\n",
+        "1. **Efficiency**: Discover 1000s of URLs in seconds, crawl only the best\n",
+        "2. **Intelligence**: BM25 scoring ensures relevance\n",
+        "3. **Scalability**: Works across multiple domains\n",
+        "4. **Flexibility**: Configurable for any use case\n",
+        "\n",
+        "### Next Steps\n",
+        "\n",
+        "1. **Customize for your domain**: Adapt the pipeline for your specific needs\n",
+        "2. **Add persistence**: Store results in a database\n",
+        "3. **Build an API**: Turn this into a web service\n",
+        "4. **Schedule updates**: Monitor topics over time\n",
+        "5. **Enhance with more AI**: Add summarization, sentiment analysis, etc.\n",
+        "\n",
+        "### Resources\n",
+        "\n",
+        "- 🐙 **GitHub**: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)\n",
+        "- 📚 **Documentation**: [crawl4ai.com/docs](https://crawl4ai.com/docs)\n",
+        "- 💬 **Discord**: [Join our community](https://discord.gg/crawl4ai)\n",
+        "\n",
+        "Thank you for learning with Crawl4AI! 🙏\n",
+        "\n",
+        "Happy researching! 🚀🔬"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "Crawl4AI_URL_Seeder_Tutorial.ipynb",
+      "provenance": [],
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/docs/examples/url_seeder/bbc_sport_research_assistant.py b/docs/examples/url_seeder/bbc_sport_research_assistant.py
new file mode 100644
index 0000000..51bcf17
--- /dev/null
+++ b/docs/examples/url_seeder/bbc_sport_research_assistant.py
@@ -0,0 +1,807 @@
+"""
+BBC Sport Research Assistant Pipeline
+=====================================
+
+This example demonstrates how URLSeeder helps create an efficient research pipeline:
+1. Discover all available URLs without crawling
+2. Filter and rank them based on relevance
+3. Crawl only the most relevant content
+4. Generate comprehensive research insights
+
+Pipeline Steps:
+1. Get user query
+2. Optionally enhance query using LLM
+3. Use URLSeeder to discover and rank URLs
+4. Crawl top K URLs with BM25 filtering
+5. Generate detailed response with citations
+
+Requirements:
+- pip install crawl4ai
+- pip install litellm
+- export GEMINI_API_KEY="your-api-key"
+
+Usage:
+- Run normally: python bbc_sport_research_assistant.py
+- Run test mode: python bbc_sport_research_assistant.py test
+
+Note: AsyncUrlSeeder now uses context manager for automatic cleanup.
+"""
+
+import asyncio
+import json
+import os
+import hashlib
+import pickle
+from typing import List, Dict, Optional, Tuple
+from dataclasses import dataclass, asdict
+from datetime import datetime
+from pathlib import Path
+
+# Rich for colored output
+from rich.console import Console
+from rich.text import Text
+from rich.panel import Panel
+from rich.table import Table
+from rich.progress import Progress, SpinnerColumn, TextColumn
+
+# Crawl4AI imports
+from crawl4ai import (
+    AsyncWebCrawler, 
+    BrowserConfig, 
+    CrawlerRunConfig,
+    AsyncUrlSeeder, 
+    SeedingConfig,
+    AsyncLogger
+)
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+# LiteLLM for AI communication
+import litellm
+
+# Initialize Rich console
+console = Console()
+
+# Get the current directory where this script is located
+SCRIPT_DIR = Path(__file__).parent.resolve()
+
+# Cache configuration - relative to script directory
+CACHE_DIR = SCRIPT_DIR / "temp_cache"
+CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+# Testing limits
+TESTING_MODE = True
+MAX_URLS_DISCOVERY = 100 if TESTING_MODE else 1000
+MAX_URLS_TO_CRAWL = 5 if TESTING_MODE else 10
+
+
+def get_cache_key(prefix: str, *args) -> str:
+    """Generate cache key from prefix and arguments"""
+    content = f"{prefix}:{'|'.join(str(arg) for arg in args)}"
+    return hashlib.md5(content.encode()).hexdigest()
+
+
+def load_from_cache(cache_key: str) -> Optional[any]:
+    """Load data from cache if exists"""
+    cache_path = CACHE_DIR / f"{cache_key}.pkl"
+    if cache_path.exists():
+        with open(cache_path, 'rb') as f:
+            return pickle.load(f)
+    return None
+
+
+def save_to_cache(cache_key: str, data: any) -> None:
+    """Save data to cache"""
+    cache_path = CACHE_DIR / f"{cache_key}.pkl"
+    with open(cache_path, 'wb') as f:
+        pickle.dump(data, f)
+
+
+@dataclass
+class ResearchConfig:
+    """Configuration for research pipeline"""
+    # Core settings
+    domain: str = "www.bbc.com/sport"
+    max_urls_discovery: int = 100
+    max_urls_to_crawl: int = 10
+    top_k_urls: int = 10
+    
+    # Scoring and filtering
+    score_threshold: float = 0.1
+    scoring_method: str = "bm25"
+    
+    # Processing options
+    use_llm_enhancement: bool = True
+    extract_head_metadata: bool = True
+    live_check: bool = True
+    force_refresh: bool = False
+    
+    # Crawler settings
+    max_concurrent_crawls: int = 5
+    timeout: int = 30000
+    headless: bool = True
+    
+    # Output settings
+    save_json: bool = True
+    save_markdown: bool = True
+    output_dir: str = None  # Will be set in __post_init__
+    
+    # Development settings
+    test_mode: bool = False
+    interactive_mode: bool = False
+    verbose: bool = True
+    
+    def __post_init__(self):
+        """Adjust settings based on test mode"""
+        if self.test_mode:
+            self.max_urls_discovery = 50
+            self.max_urls_to_crawl = 3
+            self.top_k_urls = 5
+        
+        # Set default output directory relative to script location
+        if self.output_dir is None:
+            self.output_dir = str(SCRIPT_DIR / "research_results")
+
+
+@dataclass
+class ResearchQuery:
+    """Container for research query and metadata"""
+    original_query: str
+    enhanced_query: Optional[str] = None
+    search_patterns: List[str] = None
+    timestamp: str = None
+
+
+@dataclass
+class ResearchResult:
+    """Container for research results"""
+    query: ResearchQuery
+    discovered_urls: List[Dict]
+    crawled_content: List[Dict]
+    synthesis: str
+    citations: List[Dict]
+    metadata: Dict
+
+
+async def get_user_query() -> str:
+    """
+    Get research query from user input
+    """
+    query = input("\n🔍 Enter your research query: ")
+    return query.strip()
+
+
+async def enhance_query_with_llm(query: str) -> ResearchQuery:
+    """
+    Use LLM to enhance the research query:
+    - Extract key terms
+    - Generate search patterns
+    - Identify related topics
+    """
+    # Check cache
+    cache_key = get_cache_key("enhanced_query", query)
+    cached_result = load_from_cache(cache_key)
+    if cached_result:
+        console.print("[dim cyan]📦 Using cached enhanced query[/dim cyan]")
+        return cached_result
+    
+    try:
+        response = await litellm.acompletion(
+            model="gemini/gemini-2.5-flash-preview-04-17",
+            messages=[{
+                "role": "user", 
+                "content": f"""Given this research query: "{query}"
+                
+                Extract:
+                1. Key terms and concepts (as a list)
+                2. Related search terms
+                3. A more specific/enhanced version of the query
+                
+                Return as JSON:
+                {{
+                    "key_terms": ["term1", "term2"],
+                    "related_terms": ["related1", "related2"],
+                    "enhanced_query": "enhanced version of query"
+                }}"""
+            }],
+            # reasoning_effort="low",
+            temperature=0.3,
+            response_format={"type": "json_object"}
+        )
+        
+        data = json.loads(response.choices[0].message.content)
+        
+        # Create search patterns
+        all_terms = data["key_terms"] + data["related_terms"]
+        patterns = [f"*{term.lower()}*" for term in all_terms]
+        
+        result = ResearchQuery(
+            original_query=query,
+            enhanced_query=data["enhanced_query"],
+            search_patterns=patterns[:10],  # Limit patterns
+            timestamp=datetime.now().isoformat()
+        )
+        
+        # Cache the result
+        save_to_cache(cache_key, result)
+        return result
+        
+    except Exception as e:
+        console.print(f"[yellow]⚠️ LLM enhancement failed: {e}[/yellow]")
+        # Fallback to simple tokenization
+        return ResearchQuery(
+            original_query=query,
+            enhanced_query=query,
+            search_patterns=tokenize_query_to_patterns(query),
+            timestamp=datetime.now().isoformat()
+        )
+
+
+def tokenize_query_to_patterns(query: str) -> List[str]:
+    """
+    Convert query into URL patterns for URLSeeder
+    Example: "AI startups funding" -> ["*ai*", "*startup*", "*funding*"]
+    """
+    # Simple tokenization - split and create patterns
+    words = query.lower().split()
+    # Filter out common words
+    stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'that'}
+    keywords = [w for w in words if w not in stop_words and len(w) > 2]
+    
+    # Create patterns
+    patterns = [f"*{keyword}*" for keyword in keywords]
+    return patterns[:8]  # Limit to 8 patterns
+
+
+async def discover_urls(domain: str, query: str, config: ResearchConfig) -> List[Dict]:
+    """
+    Use URLSeeder to discover and rank URLs:
+    1. Fetch all URLs from domain
+    2. Filter by patterns
+    3. Extract metadata (titles, descriptions)
+    4. Rank by BM25 relevance score
+    5. Return top K URLs
+    """
+    # Check cache
+    cache_key = get_cache_key("discovered_urls", domain, query, config.top_k_urls)
+    cached_result = load_from_cache(cache_key)
+    if cached_result and not config.force_refresh:
+        console.print("[dim cyan]📦 Using cached URL discovery[/dim cyan]")
+        return cached_result
+    
+    console.print(f"\n[cyan]🔍 Discovering URLs from {domain}...[/cyan]")
+    
+    # Initialize URL seeder with context manager
+    async with AsyncUrlSeeder(logger=AsyncLogger(verbose=config.verbose)) as seeder:
+        # Configure seeding
+        seeding_config = SeedingConfig(
+            source="sitemap+cc",  # Use both sitemap and Common Crawl
+            extract_head=config.extract_head_metadata,
+            query=query,
+            scoring_method=config.scoring_method,
+            score_threshold=config.score_threshold,
+            max_urls=config.max_urls_discovery,
+            live_check=config.live_check,
+            force=config.force_refresh
+        )
+        
+        try:
+            # Discover URLs
+            urls = await seeder.urls(domain, seeding_config)
+            
+            # Sort by relevance score (descending)
+            sorted_urls = sorted(
+                urls, 
+                key=lambda x: x.get('relevance_score', 0), 
+                reverse=True
+            )
+            
+            # Take top K
+            top_urls = sorted_urls[:config.top_k_urls]
+            
+            console.print(f"[green]✅ Discovered {len(urls)} URLs, selected top {len(top_urls)}[/green]")
+            
+            # Cache the result
+            save_to_cache(cache_key, top_urls)
+            return top_urls
+            
+        except Exception as e:
+            console.print(f"[red]❌ URL discovery failed: {e}[/red]")
+            return []
+
+
+async def crawl_selected_urls(urls: List[str], query: str, config: ResearchConfig) -> List[Dict]:
+    """
+    Crawl selected URLs with content filtering:
+    - Use AsyncWebCrawler.arun_many()
+    - Apply content filter
+    - Generate clean markdown
+    """
+    # Extract just URLs from the discovery results
+    url_list = [u['url'] for u in urls if 'url' in u][:config.max_urls_to_crawl]
+    
+    if not url_list:
+        console.print("[red]❌ No URLs to crawl[/red]")
+        return []
+    
+    console.print(f"\n[cyan]🕷️ Crawling {len(url_list)} URLs...[/cyan]")
+    
+    # Check cache for each URL
+    crawled_results = []
+    urls_to_crawl = []
+    
+    for url in url_list:
+        cache_key = get_cache_key("crawled_content", url, query)
+        cached_content = load_from_cache(cache_key)
+        if cached_content and not config.force_refresh:
+            crawled_results.append(cached_content)
+        else:
+            urls_to_crawl.append(url)
+    
+    if urls_to_crawl:
+        console.print(f"[cyan]📥 Crawling {len(urls_to_crawl)} new URLs (cached: {len(crawled_results)})[/cyan]")
+                
+        # Configure markdown generator with content filter
+        md_generator = DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(
+                threshold=0.48,
+                threshold_type="dynamic",
+                min_word_threshold=10
+            ),
+        )
+        
+        # Configure crawler
+        crawler_config = CrawlerRunConfig(
+            markdown_generator=md_generator,
+            exclude_external_links=True,
+            excluded_tags=['nav', 'header', 'footer', 'aside'],
+        )
+        
+        # Create crawler with browser config
+        async with AsyncWebCrawler(
+            config=BrowserConfig(
+                headless=config.headless,
+                verbose=config.verbose
+            )
+        ) as crawler:
+            # Crawl URLs
+            results = await crawler.arun_many(
+                urls_to_crawl,
+                config=crawler_config,
+                max_concurrent=config.max_concurrent_crawls
+            )
+            
+            # Process results
+            for url, result in zip(urls_to_crawl, results):
+                if result.success:
+                    content_data = {
+                        'url': url,
+                        'title': result.metadata.get('title', ''),
+                        'markdown': result.markdown.fit_markdown or result.markdown.raw_markdown,
+                        'raw_length': len(result.markdown.raw_markdown),
+                        'fit_length': len(result.markdown.fit_markdown) if result.markdown.fit_markdown else len(result.markdown.raw_markdown),
+                        'metadata': result.metadata
+                    }
+                    crawled_results.append(content_data)
+                    
+                    # Cache the result
+                    cache_key = get_cache_key("crawled_content", url, query)
+                    save_to_cache(cache_key, content_data)
+                else:
+                    console.print(f"  [red]❌ Failed: {url[:50]}... - {result.error}[/red]")
+    
+    console.print(f"[green]✅ Successfully crawled {len(crawled_results)} URLs[/green]")
+    return crawled_results
+
+
+async def generate_research_synthesis(
+    query: str, 
+    crawled_content: List[Dict]
+) -> Tuple[str, List[Dict]]:
+    """
+    Use LLM to synthesize research findings:
+    - Analyze all crawled content
+    - Generate comprehensive answer
+    - Extract citations and references
+    """
+    if not crawled_content:
+        return "No content available for synthesis.", []
+    
+    console.print("\n[cyan]🤖 Generating research synthesis...[/cyan]")
+    
+    # Prepare content for LLM
+    content_sections = []
+    for i, content in enumerate(crawled_content, 1):
+        section = f"""
+SOURCE {i}:
+Title: {content['title']}
+URL: {content['url']}
+Content Preview:
+{content['markdown'][:1500]}...
+"""
+        content_sections.append(section)
+    
+    combined_content = "\n---\n".join(content_sections)
+    
+    try:
+        response = await litellm.acompletion(
+            model="gemini/gemini-2.5-flash-preview-04-17",
+            messages=[{
+                "role": "user",
+                "content": f"""Research Query: "{query}"
+
+Based on the following sources, provide a comprehensive research synthesis.
+
+{combined_content}
+
+Please provide:
+1. An executive summary (2-3 sentences)
+2. Key findings (3-5 bullet points)
+3. Detailed analysis (2-3 paragraphs)
+4. Future implications or trends
+
+Format your response with clear sections and cite sources using [Source N] notation.
+Keep the total response under 800 words."""
+            }],
+            # reasoning_effort="medium",
+            temperature=0.7
+        )
+        
+        synthesis = response.choices[0].message.content
+        
+        # Extract citations from the synthesis
+        citations = []
+        for i, content in enumerate(crawled_content, 1):
+            if f"[Source {i}]" in synthesis or f"Source {i}" in synthesis:
+                citations.append({
+                    'source_id': i,
+                    'title': content['title'],
+                    'url': content['url']
+                })
+        
+        return synthesis, citations
+        
+    except Exception as e:
+        console.print(f"[red]❌ Synthesis generation failed: {e}[/red]")
+        # Fallback to simple summary
+        summary = f"Research on '{query}' found {len(crawled_content)} relevant articles:\n\n"
+        for content in crawled_content[:3]:
+            summary += f"- {content['title']}\n  {content['url']}\n\n"
+        return summary, []
+
+
+def format_research_output(result: ResearchResult) -> str:
+    """
+    Format the final research output with:
+    - Executive summary
+    - Key findings
+    - Detailed analysis
+    - Citations and sources
+    """
+    output = []
+    output.append("\n" + "=" * 60)
+    output.append("🔬 RESEARCH RESULTS")
+    output.append("=" * 60)
+    
+    # Query info
+    output.append(f"\n📋 Query: {result.query.original_query}")
+    if result.query.enhanced_query != result.query.original_query:
+        output.append(f"   Enhanced: {result.query.enhanced_query}")
+    
+    # Discovery stats
+    output.append(f"\n📊 Statistics:")
+    output.append(f"   - URLs discovered: {len(result.discovered_urls)}")
+    output.append(f"   - URLs crawled: {len(result.crawled_content)}")
+    output.append(f"   - Processing time: {result.metadata.get('duration', 'N/A')}")
+    
+    # Synthesis
+    output.append(f"\n📝 SYNTHESIS")
+    output.append("-" * 60)
+    output.append(result.synthesis)
+    
+    # Citations
+    if result.citations:
+        output.append(f"\n📚 SOURCES")
+        output.append("-" * 60)
+        for citation in result.citations:
+            output.append(f"[{citation['source_id']}] {citation['title']}")
+            output.append(f"    {citation['url']}")
+    
+    return "\n".join(output)
+
+
+async def save_research_results(result: ResearchResult, config: ResearchConfig) -> Tuple[str, str]:
+    """
+    Save research results in JSON and Markdown formats
+    
+    Returns:
+        Tuple of (json_path, markdown_path)
+    """
+    # Create output directory
+    output_dir = Path(config.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Generate filename based on query and timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    query_slug = result.query.original_query[:50].replace(" ", "_").replace("/", "_")
+    base_filename = f"{timestamp}_{query_slug}"
+    
+    json_path = None
+    md_path = None
+    
+    # Save JSON
+    if config.save_json:
+        json_path = output_dir / f"{base_filename}.json"
+        with open(json_path, 'w') as f:
+            json.dump(asdict(result), f, indent=2, default=str)
+        console.print(f"\n[green]💾 JSON saved: {json_path}[/green]")
+    
+    # Save Markdown
+    if config.save_markdown:
+        md_path = output_dir / f"{base_filename}.md"
+        
+        # Create formatted markdown
+        md_content = [
+            f"# Research Report: {result.query.original_query}",
+            f"\n**Generated on:** {result.metadata.get('timestamp', 'N/A')}",
+            f"\n**Domain:** {result.metadata.get('domain', 'N/A')}",
+            f"\n**Processing time:** {result.metadata.get('duration', 'N/A')}",
+            "\n---\n",
+            "## Query Information",
+            f"- **Original Query:** {result.query.original_query}",
+            f"- **Enhanced Query:** {result.query.enhanced_query or 'N/A'}",
+            f"- **Search Patterns:** {', '.join(result.query.search_patterns or [])}",
+            "\n## Statistics",
+            f"- **URLs Discovered:** {len(result.discovered_urls)}",
+            f"- **URLs Crawled:** {len(result.crawled_content)}",
+            f"- **Sources Cited:** {len(result.citations)}",
+            "\n## Research Synthesis\n",
+            result.synthesis,
+            "\n## Sources\n"
+        ]
+        
+        # Add citations
+        for citation in result.citations:
+            md_content.append(f"### [{citation['source_id']}] {citation['title']}")
+            md_content.append(f"- **URL:** [{citation['url']}]({citation['url']})")
+            md_content.append("")
+        
+        # Add discovered URLs summary
+        md_content.extend([
+            "\n## Discovered URLs (Top 10)\n",
+            "| Score | URL | Title |",
+            "|-------|-----|-------|"
+        ])
+        
+        for url_data in result.discovered_urls[:10]:
+            score = url_data.get('relevance_score', 0)
+            url = url_data.get('url', '')
+            title = 'N/A'
+            if 'head_data' in url_data and url_data['head_data']:
+                title = url_data['head_data'].get('title', 'N/A')[:60] + '...'
+            md_content.append(f"| {score:.3f} | {url[:50]}... | {title} |")
+        
+        # Write markdown
+        with open(md_path, 'w') as f:
+            f.write('\n'.join(md_content))
+        
+        console.print(f"[green]📄 Markdown saved: {md_path}[/green]")
+    
+    return str(json_path) if json_path else None, str(md_path) if md_path else None
+
+
+async def wait_for_user(message: str = "\nPress Enter to continue..."):
+    """Wait for user input in interactive mode"""
+    input(message)
+
+
+async def research_pipeline(
+    query: str,
+    config: ResearchConfig
+) -> ResearchResult:
+    """
+    Main research pipeline orchestrator with configurable settings
+    """
+    start_time = datetime.now()
+    
+    # Display pipeline header
+    header = Panel(
+        f"[bold cyan]Research Pipeline[/bold cyan]\n\n"
+        f"[dim]Domain:[/dim] {config.domain}\n"
+        f"[dim]Mode:[/dim] {'Test' if config.test_mode else 'Production'}\n"
+        f"[dim]Interactive:[/dim] {'Yes' if config.interactive_mode else 'No'}",
+        title="🚀 Starting",
+        border_style="cyan"
+    )
+    console.print(header)
+    
+    # Step 1: Enhance query (optional)
+    console.print(f"\n[bold cyan]📝 Step 1: Query Processing[/bold cyan]")
+    if config.interactive_mode:
+        await wait_for_user()
+        
+    if config.use_llm_enhancement:
+        research_query = await enhance_query_with_llm(query)
+    else:
+        research_query = ResearchQuery(
+            original_query=query,
+            enhanced_query=query,
+            search_patterns=tokenize_query_to_patterns(query),
+            timestamp=datetime.now().isoformat()
+        )
+    
+    console.print(f"   [green]✅ Query ready:[/green] {research_query.enhanced_query or query}")
+    
+    # Step 2: Discover URLs
+    console.print(f"\n[bold cyan]🔍 Step 2: URL Discovery[/bold cyan]")
+    if config.interactive_mode:
+        await wait_for_user()
+        
+    discovered_urls = await discover_urls(
+        domain=config.domain,
+        query=research_query.enhanced_query or query,
+        config=config
+    )
+    
+    if not discovered_urls:
+        return ResearchResult(
+            query=research_query,
+            discovered_urls=[],
+            crawled_content=[],
+            synthesis="No relevant URLs found for the given query.",
+            citations=[],
+            metadata={'duration': str(datetime.now() - start_time)}
+        )
+    
+    console.print(f"   [green]✅ Found {len(discovered_urls)} relevant URLs[/green]")
+    
+    # Step 3: Crawl selected URLs
+    console.print(f"\n[bold cyan]🕷️ Step 3: Content Crawling[/bold cyan]")
+    if config.interactive_mode:
+        await wait_for_user()
+        
+    crawled_content = await crawl_selected_urls(
+        urls=discovered_urls,
+        query=research_query.enhanced_query or query,
+        config=config
+    )
+    
+    console.print(f"   [green]✅ Successfully crawled {len(crawled_content)} pages[/green]")
+    
+    # Step 4: Generate synthesis
+    console.print(f"\n[bold cyan]🤖 Step 4: Synthesis Generation[/bold cyan]")
+    if config.interactive_mode:
+        await wait_for_user()
+        
+    synthesis, citations = await generate_research_synthesis(
+        query=research_query.enhanced_query or query,
+        crawled_content=crawled_content
+    )
+    
+    console.print(f"   [green]✅ Generated synthesis with {len(citations)} citations[/green]")
+    
+    # Step 5: Create result
+    result = ResearchResult(
+        query=research_query,
+        discovered_urls=discovered_urls,
+        crawled_content=crawled_content,
+        synthesis=synthesis,
+        citations=citations,
+        metadata={
+            'duration': str(datetime.now() - start_time),
+            'domain': config.domain,
+            'timestamp': datetime.now().isoformat(),
+            'config': asdict(config)
+        }
+    )
+    
+    duration = datetime.now() - start_time
+    console.print(f"\n[bold green]✅ Research completed in {duration}[/bold green]")
+    
+    return result
+
+
+async def main():
+    """
+    Main entry point for the BBC Sport Research Assistant
+    """
+    # Example queries
+    example_queries = [
+        "Premier League transfer news and rumors",
+        "Champions League match results and analysis", 
+        "World Cup qualifying updates",
+        "Football injury reports and return dates",
+        "Tennis grand slam tournament results"
+    ]
+    
+    # Display header
+    console.print(Panel.fit(
+        "[bold cyan]BBC Sport Research Assistant[/bold cyan]\n\n"
+        "This tool demonstrates efficient research using URLSeeder:\n"
+        "[dim]• Discover all URLs without crawling\n"
+        "• Filter and rank by relevance\n"
+        "• Crawl only the most relevant content\n"
+        "• Generate AI-powered insights with citations[/dim]\n\n"
+        f"[dim]📁 Working directory: {SCRIPT_DIR}[/dim]",
+        title="🔬 Welcome",
+        border_style="cyan"
+    ))
+    
+    # Configuration options table
+    config_table = Table(title="\n⚙️  Configuration Options", show_header=False, box=None)
+    config_table.add_column(style="bold cyan", width=3)
+    config_table.add_column()
+    
+    config_table.add_row("1", "Quick Test Mode (3 URLs, fast)")
+    config_table.add_row("2", "Standard Mode (10 URLs, balanced)")
+    config_table.add_row("3", "Comprehensive Mode (20 URLs, thorough)")
+    config_table.add_row("4", "Custom Configuration")
+    
+    console.print(config_table)
+    
+    config_choice = input("\nSelect configuration (1-4): ").strip()
+    
+    # Create config based on choice
+    if config_choice == "1":
+        config = ResearchConfig(test_mode=True, interactive_mode=False)
+    elif config_choice == "2":
+        config = ResearchConfig(max_urls_to_crawl=10, top_k_urls=10)
+    elif config_choice == "3":
+        config = ResearchConfig(max_urls_to_crawl=20, top_k_urls=20, max_urls_discovery=200)
+    else:
+        # Custom configuration
+        config = ResearchConfig()
+        config.test_mode = input("\nTest mode? (y/n): ").lower() == 'y'
+        config.interactive_mode = input("Interactive mode (pause between steps)? (y/n): ").lower() == 'y'
+        config.use_llm_enhancement = input("Use AI to enhance queries? (y/n): ").lower() == 'y'
+        
+        if not config.test_mode:
+            try:
+                config.max_urls_to_crawl = int(input("Max URLs to crawl (default 10): ") or "10")
+                config.top_k_urls = int(input("Top K URLs to select (default 10): ") or "10")
+            except ValueError:
+                console.print("[yellow]Using default values[/yellow]")
+    
+    # Display example queries
+    query_table = Table(title="\n📋 Example Queries", show_header=False, box=None)
+    query_table.add_column(style="bold cyan", width=3)
+    query_table.add_column()
+    
+    for i, q in enumerate(example_queries, 1):
+        query_table.add_row(str(i), q)
+    
+    console.print(query_table)
+    
+    query_input = input("\nSelect a query (1-5) or enter your own: ").strip()
+    
+    if query_input.isdigit() and 1 <= int(query_input) <= len(example_queries):
+        query = example_queries[int(query_input) - 1]
+    else:
+        query = query_input if query_input else example_queries[0]
+    
+    console.print(f"\n[bold cyan]📝 Selected Query:[/bold cyan] {query}")
+    
+    # Run the research pipeline
+    result = await research_pipeline(query=query, config=config)
+    
+    # Display results
+    formatted_output = format_research_output(result)
+    # print(formatted_output)
+    console.print(Panel.fit(
+        formatted_output,
+        title="🔬 Research Results",
+        border_style="green"
+    ))
+    
+    # Save results
+    if config.save_json or config.save_markdown:
+        json_path, md_path = await save_research_results(result, config)
+        # print(f"\n✅ Results saved successfully!")
+        if json_path:
+            console.print(f"[green]JSON saved at:[/green] {json_path}")
+        if md_path:
+            console.print(f"[green]Markdown saved at:[/green] {md_path}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/url_seeder/convert_tutorial_to_colab.py b/docs/examples/url_seeder/convert_tutorial_to_colab.py
new file mode 100644
index 0000000..034fcc9
--- /dev/null
+++ b/docs/examples/url_seeder/convert_tutorial_to_colab.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+Convert Crawl4AI URL Seeder tutorial markdown to Colab notebook format
+"""
+
+import json
+import re
+from pathlib import Path
+
+
+def parse_markdown_to_cells(markdown_content):
+    """Parse markdown content and convert to notebook cells"""
+    cells = []
+    
+    # Split content by cell markers
+    lines = markdown_content.split('\n')
+    
+    # Extract the header content before first cell marker
+    header_lines = []
+    i = 0
+    while i < len(lines) and not lines[i].startswith('# cell'):
+        header_lines.append(lines[i])
+        i += 1
+    
+    # Add header as markdown cell if it exists
+    if header_lines:
+        header_content = '\n'.join(header_lines).strip()
+        if header_content:
+            cells.append({
+                "cell_type": "markdown",
+                "metadata": {},
+                "source": header_content.split('\n')
+            })
+    
+    # Process cells marked with # cell X type:Y
+    current_cell_content = []
+    current_cell_type = None
+    
+    while i < len(lines):
+        line = lines[i]
+        
+        # Check for cell marker
+        cell_match = re.match(r'^# cell (\d+) type:(markdown|code)$', line)
+        
+        if cell_match:
+            # Save previous cell if exists
+            if current_cell_content and current_cell_type:
+                content = '\n'.join(current_cell_content).strip()
+                if content:
+                    if current_cell_type == 'code':
+                        cells.append({
+                            "cell_type": "code",
+                            "execution_count": None,
+                            "metadata": {},
+                            "outputs": [],
+                            "source": content.split('\n')
+                        })
+                    else:
+                        cells.append({
+                            "cell_type": "markdown",
+                            "metadata": {},
+                            "source": content.split('\n')
+                        })
+            
+            # Start new cell
+            current_cell_type = cell_match.group(2)
+            current_cell_content = []
+        else:
+            # Add line to current cell
+            current_cell_content.append(line)
+        
+        i += 1
+    
+    # Add last cell if exists
+    if current_cell_content and current_cell_type:
+        content = '\n'.join(current_cell_content).strip()
+        if content:
+            if current_cell_type == 'code':
+                cells.append({
+                    "cell_type": "code",
+                    "execution_count": None,
+                    "metadata": {},
+                    "outputs": [],
+                    "source": content.split('\n')
+                })
+            else:
+                cells.append({
+                    "cell_type": "markdown",
+                    "metadata": {},
+                    "source": content.split('\n')
+                })
+    
+    return cells
+
+
+def create_colab_notebook(cells):
+    """Create a Colab notebook structure"""
+    notebook = {
+        "nbformat": 4,
+        "nbformat_minor": 0,
+        "metadata": {
+            "colab": {
+                "name": "Crawl4AI_URL_Seeder_Tutorial.ipynb",
+                "provenance": [],
+                "collapsed_sections": [],
+                "toc_visible": True
+            },
+            "kernelspec": {
+                "name": "python3",
+                "display_name": "Python 3"
+            },
+            "language_info": {
+                "name": "python"
+            }
+        },
+        "cells": cells
+    }
+    
+    return notebook
+
+
+def main():
+    # Read the markdown file
+    md_path = Path("tutorial_url_seeder.md")
+    
+    if not md_path.exists():
+        print(f"Error: {md_path} not found!")
+        return
+    
+    print(f"Reading {md_path}...")
+    with open(md_path, 'r', encoding='utf-8') as f:
+        markdown_content = f.read()
+    
+    # Parse markdown to cells
+    print("Parsing markdown content...")
+    cells = parse_markdown_to_cells(markdown_content)
+    print(f"Created {len(cells)} cells")
+    
+    # Create notebook
+    print("Creating Colab notebook...")
+    notebook = create_colab_notebook(cells)
+    
+    # Save notebook
+    output_path = Path("Crawl4AI_URL_Seeder_Tutorial.ipynb")
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(notebook, f, indent=2, ensure_ascii=False)
+    
+    print(f"✅ Successfully created {output_path}")
+    print(f"   - Total cells: {len(cells)}")
+    print(f"   - Markdown cells: {sum(1 for c in cells if c['cell_type'] == 'markdown')}")
+    print(f"   - Code cells: {sum(1 for c in cells if c['cell_type'] == 'code')}")
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/docs/examples/url_seeder/tutorial_url_seeder.md b/docs/examples/url_seeder/tutorial_url_seeder.md
new file mode 100644
index 0000000..4b9a220
--- /dev/null
+++ b/docs/examples/url_seeder/tutorial_url_seeder.md
@@ -0,0 +1,1035 @@
+# 🔬 Building an AI Research Assistant with Crawl4AI: Smart URL Discovery
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1QIwVYrQaZGPJQGHQBvMSbkdnc5usqoGw#scrollTo=xbV1w9YM4LkW)
+
+## Welcome to the Research Pipeline Workshop!
+
+In this tutorial, we'll build an **AI-powered research assistant** that intelligently discovers, filters, and analyzes web content. Instead of blindly crawling hundreds of pages, we'll use Crawl4AI's URL Seeder to:
+
+- 🔍 **Discover all available URLs** without crawling them first
+- 🎯 **Score and rank** them by relevance using AI
+- 🕷️ **Crawl only the most relevant** content
+- 🤖 **Generate research insights** with proper citations
+
+By the end, you'll have a complete research pipeline that can analyze any topic across multiple websites efficiently.
+
+## What You'll Build
+
+A **smart research assistant** that:
+1. Takes any research query (e.g., "Premier League transfer news")
+2. Discovers relevant articles from news sites
+3. Ranks them by relevance using BM25 scoring
+4. Crawls only the top-ranked articles
+5. Synthesizes findings into a comprehensive report
+
+## Prerequisites
+
+- Python 3.8+ environment
+- Basic understanding of async Python
+- API keys for LLM (Gemini or OpenAI recommended)
+
+## Pipeline Overview
+
+```
+User Query → Query Enhancement → URL Discovery → Relevance Scoring → Smart Crawling → AI Synthesis → Research Report
+```
+
+Each step builds on the previous one, creating an efficient research system that saves time and resources.
+
+Let's begin! 🚀
+
+---
+
+# cell 1 type:markdown
+## Step 0: Environment Setup and Dependencies
+
+First, we'll set up our environment with all necessary libraries. We need Crawl4AI for intelligent web crawling, LiteLLM for AI integration, and Rich for beautiful terminal output. This foundation ensures our research assistant has all the tools it needs.
+
+# cell 2 type:code
+# Install required packages
+!pip install -q crawl4ai litellm rich
+
+# cell 3 type:code
+import asyncio
+import json
+import os
+from typing import List, Dict, Optional, Tuple
+from dataclasses import dataclass, asdict
+from datetime import datetime
+from pathlib import Path
+
+# Rich for beautiful console output
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich.progress import Progress, SpinnerColumn, TextColumn
+
+# Crawl4AI imports for intelligent crawling
+from crawl4ai import (
+    AsyncWebCrawler, 
+    BrowserConfig, 
+    CrawlerRunConfig,
+    AsyncUrlSeeder, 
+    SeedingConfig,
+    AsyncLogger
+)
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+# LiteLLM for AI capabilities
+import litellm
+
+# Initialize Rich console for pretty output
+console = Console()
+
+print("✅ Environment ready! All dependencies loaded successfully.")
+
+# cell 4 type:markdown
+## Step 1: Configuration and Data Classes
+
+Here we define our research pipeline configuration. These dataclasses act as our control center, allowing us to fine-tune every aspect of the research process. Think of them as the settings panel for your research assistant - from discovery limits to AI model choices.
+
+# cell 5 type:code
+@dataclass
+class ResearchConfig:
+    """Configuration for the research pipeline
+    
+    This class controls every aspect of our research assistant:
+    - How many URLs to discover and crawl
+    - Which scoring methods to use
+    - Whether to use AI enhancement
+    - Output preferences
+    """
+    # Core settings
+    domain: str = "www.bbc.com/sport"
+    max_urls_discovery: int = 100    # Cast a wide net initially
+    max_urls_to_crawl: int = 10      # But only crawl the best
+    top_k_urls: int = 10             # Focus on top results
+    
+    # Scoring and filtering
+    score_threshold: float = 0.3     # Minimum relevance score
+    scoring_method: str = "bm25"     # BM25 is great for relevance
+    
+    # AI and processing
+    use_llm_enhancement: bool = True  # Enhance queries with AI
+    llm_model: str = "gemini/gemini-1.5-flash"  # Fast and capable
+    
+    # URL discovery options
+    extract_head_metadata: bool = True  # Get titles, descriptions
+    live_check: bool = False           # Verify URLs are accessible
+    force_refresh: bool = False        # Bypass cache
+    
+    # Crawler settings
+    max_concurrent_crawls: int = 5     # Parallel crawling
+    timeout: int = 30000              # 30 second timeout
+    headless: bool = True             # No browser window
+    
+    # Output settings
+    output_dir: Path = Path("research_results")
+    verbose: bool = True
+
+@dataclass
+class ResearchQuery:
+    """Container for research query and metadata"""
+    original_query: str
+    enhanced_query: Optional[str] = None
+    search_patterns: List[str] = None
+    timestamp: str = None
+
+@dataclass
+class ResearchResult:
+    """Container for research results"""
+    query: ResearchQuery
+    discovered_urls: List[Dict]
+    crawled_content: List[Dict]
+    synthesis: str
+    citations: List[Dict]
+    metadata: Dict
+
+# Create default configuration
+config = ResearchConfig()
+console.print(Panel(
+    f"[bold cyan]Research Configuration[/bold cyan]\n\n"
+    f"🌐 Domain: {config.domain}\n"
+    f"🔍 Max Discovery: {config.max_urls_discovery} URLs\n"
+    f"🕷️ Max Crawl: {config.max_urls_to_crawl} pages\n"
+    f"🤖 AI Model: {config.llm_model}",
+    title="⚙️ Settings"
+))
+
+# cell 6 type:markdown
+## Step 2: Query Enhancement with AI
+
+Not all search queries are created equal. Here we use AI to transform simple queries into comprehensive search strategies. The LLM analyzes your query, extracts key concepts, and generates related terms - turning "football news" into a rich set of search patterns.
+
+# cell 7 type:code
+async def enhance_query_with_llm(query: str, config: ResearchConfig) -> ResearchQuery:
+    """
+    Transform simple queries into comprehensive search strategies
+    
+    Why enhance queries?
+    - Users often use simple terms ("football news")
+    - But relevant content might use varied terminology
+    - AI helps capture all relevant variations
+    """
+    console.print(f"\n[cyan]🤖 Enhancing query: '{query}'...[/cyan]")
+    
+    try:
+        # Ask AI to analyze and expand the query
+        response = await litellm.acompletion(
+            model=config.llm_model,
+            messages=[{
+                "role": "user", 
+                "content": f"""Given this research query: "{query}"
+                
+                Extract:
+                1. Key terms and concepts (as a list)
+                2. Related search terms
+                3. A more specific/enhanced version of the query
+                
+                Return as JSON:
+                {{
+                    "key_terms": ["term1", "term2"],
+                    "related_terms": ["related1", "related2"],
+                    "enhanced_query": "enhanced version of query"
+                }}"""
+            }],
+            temperature=0.3,  # Low temperature for consistency
+            response_format={"type": "json_object"}
+        )
+        
+        data = json.loads(response.choices[0].message.content)
+        
+        # Create search patterns from extracted terms
+        # These patterns help the URL seeder find relevant pages
+        all_terms = data["key_terms"] + data["related_terms"]
+        patterns = [f"*{term.lower()}*" for term in all_terms]
+        
+        result = ResearchQuery(
+            original_query=query,
+            enhanced_query=data["enhanced_query"],
+            search_patterns=patterns[:10],  # Limit to 10 patterns
+            timestamp=datetime.now().isoformat()
+        )
+        
+        # Show the enhancement
+        console.print(Panel(
+            f"[green]✅ Enhanced Query:[/green] {result.enhanced_query}\n"
+            f"[dim]Key terms: {', '.join(data['key_terms'])}[/dim]",
+            title="🔍 Query Enhancement"
+        ))
+        
+        return result
+        
+    except Exception as e:
+        console.print(f"[yellow]⚠️ Enhancement failed, using original query: {e}[/yellow]")
+        # Fallback to simple tokenization
+        words = query.lower().split()
+        patterns = [f"*{word}*" for word in words if len(word) > 2]
+        
+        return ResearchQuery(
+            original_query=query,
+            enhanced_query=query,
+            search_patterns=patterns,
+            timestamp=datetime.now().isoformat()
+        )
+
+# Example usage
+test_query = "Premier League transfer news"
+enhanced = await enhance_query_with_llm(test_query, config)
+
+# cell 8 type:markdown
+## Step 3: Smart URL Discovery with AsyncUrlSeeder
+
+This is where the magic begins! Instead of crawling pages to find links, AsyncUrlSeeder discovers URLs from sitemaps and Common Crawl data. It's like having a map of the entire website before you start exploring. We'll discover hundreds of URLs in seconds, complete with metadata.
+
+# cell 9 type:code
+async def discover_urls(
+    domain: str, 
+    query: ResearchQuery, 
+    config: ResearchConfig
+) -> List[Dict]:
+    """
+    Discover and rank URLs without crawling them
+    
+    The URL Seeder is incredibly powerful because it:
+    1. Gets URLs from sitemaps (official site maps)
+    2. Gets URLs from Common Crawl (web-scale data)
+    3. Extracts metadata without full page loads
+    4. Scores relevance using BM25 algorithm
+    
+    This means we know which pages are worth crawling
+    BEFORE we spend time crawling them!
+    """
+    console.print(f"\n[cyan]🔍 Discovering URLs from {domain}...[/cyan]")
+    
+    # Use context manager for automatic cleanup
+    async with AsyncUrlSeeder(logger=AsyncLogger(verbose=config.verbose)) as seeder:
+        # Configure the discovery process
+        seeding_config = SeedingConfig(
+            # Data sources
+            source="sitemap+cc",  # Use both sitemap AND Common Crawl
+            
+            # Metadata extraction
+            extract_head=config.extract_head_metadata,  # Get titles, descriptions
+            
+            # Relevance scoring
+            query=query.enhanced_query or query.original_query,
+            scoring_method=config.scoring_method,  # BM25 scoring
+            score_threshold=config.score_threshold,  # Minimum score
+            
+            # Limits and performance
+            max_urls=config.max_urls_discovery,
+            live_check=config.live_check,  # Verify URLs work
+            force=config.force_refresh,  # Bypass cache if needed
+            
+            # Performance tuning
+            concurrency=20,  # Parallel workers
+        )
+        
+        try:
+            # Discover URLs - this is FAST!
+            urls = await seeder.urls(domain, seeding_config)
+            
+            # Results are already sorted by relevance
+            # thanks to BM25 scoring
+            top_urls = urls[:config.top_k_urls]
+            
+            # Show discovery results
+            console.print(f"[green]✅ Discovered {len(urls)} URLs, selected top {len(top_urls)}[/green]")
+            
+            # Display a sample of what we found
+            if top_urls:
+                table = Table(title="🎯 Top Discovered URLs")
+                table.add_column("Score", style="cyan")
+                table.add_column("Title", style="green")
+                table.add_column("URL", style="dim")
+                
+                for url in top_urls[:5]:
+                    score = f"{url.get('relevance_score', 0):.3f}"
+                    title = "N/A"
+                    if url.get('head_data') and url['head_data'].get('title'):
+                        title = url['head_data']['title'][:50] + "..."
+                    url_str = url['url'][:60] + "..."
+                    
+                    table.add_row(score, title, url_str)
+                
+                console.print(table)
+            
+            return top_urls
+            
+        except Exception as e:
+            console.print(f"[red]❌ URL discovery failed: {e}[/red]")
+            return []
+
+# Example discovery
+discovered = await discover_urls(config.domain, enhanced, config)
+
+# cell 10 type:markdown
+## Step 4: Intelligent Content Crawling
+
+Now we crawl only the most relevant URLs. This is where our smart filtering pays off - instead of crawling hundreds of pages, we focus on the top 10-20 most relevant ones. We use content filtering to extract only the meaningful text, removing ads and navigation.
+
+# cell 11 type:code
+async def crawl_selected_urls(
+    urls: List[Dict], 
+    query: ResearchQuery, 
+    config: ResearchConfig
+) -> List[Dict]:
+    """
+    Crawl only the most relevant URLs with smart content filtering
+    
+    Key optimizations:
+    1. We already know these URLs are relevant (from scoring)
+    2. We crawl them in parallel for speed
+    3. We extract only meaningful content (no ads/nav)
+    4. We generate clean markdown for analysis
+    """
+    # Extract URLs from discovery results
+    url_list = [u['url'] for u in urls if 'url' in u][:config.max_urls_to_crawl]
+    
+    if not url_list:
+        console.print("[red]❌ No URLs to crawl[/red]")
+        return []
+    
+    console.print(f"\n[cyan]🕷️ Crawling {len(url_list)} URLs...[/cyan]")
+    
+    # Configure intelligent content extraction
+    # This removes ads, navigation, and other noise
+    md_generator = DefaultMarkdownGenerator(
+        content_filter=PruningContentFilter(
+            threshold=0.48,  # Content relevance threshold
+            threshold_type="dynamic",  # Adapts to page structure
+            min_word_threshold=10  # Ignore tiny text blocks
+        ),
+    )
+    
+    # Configure the crawler
+    crawler_config = CrawlerRunConfig(
+        markdown_generator=md_generator,
+        exclude_external_links=True,  # Focus on content, not links
+        excluded_tags=['nav', 'header', 'footer', 'aside'],  # Skip UI elements
+    )
+    
+    # Create crawler with browser config
+    async with AsyncWebCrawler(
+        config=BrowserConfig(
+            headless=config.headless,
+            verbose=config.verbose
+        )
+    ) as crawler:
+        # Crawl URLs in parallel for speed
+        # arun_many handles concurrency automatically
+        results = await crawler.arun_many(
+            url_list,
+            config=crawler_config,
+            max_concurrent=config.max_concurrent_crawls
+        )
+        
+        # Process successful results
+        crawled_content = []
+        for url, result in zip(url_list, results):
+            if result.success:
+                # Extract the content we need
+                content_data = {
+                    'url': url,
+                    'title': result.metadata.get('title', 'No title'),
+                    'markdown': result.markdown.fit_markdown or result.markdown.raw_markdown,
+                    'metadata': result.metadata
+                }
+                crawled_content.append(content_data)
+                console.print(f"  [green]✓[/green] Crawled: {url[:60]}...")
+            else:
+                console.print(f"  [red]✗[/red] Failed: {url[:50]}... - {result.error}")
+    
+    console.print(f"[green]✅ Successfully crawled {len(crawled_content)} pages[/green]")
+    return crawled_content
+
+# Example crawling
+crawled = await crawl_selected_urls(discovered[:5], enhanced, config)
+
+# cell 12 type:markdown
+## Step 5: AI-Powered Research Synthesis
+
+This is where we transform raw content into insights. The AI analyzes all crawled articles, identifies key themes, and generates a comprehensive synthesis with proper citations. It's like having a research assistant read everything and write you a summary.
+
+# cell 13 type:code
+async def generate_research_synthesis(
+    query: ResearchQuery,
+    crawled_content: List[Dict],
+    config: ResearchConfig
+) -> Tuple[str, List[Dict]]:
+    """
+    Use AI to synthesize findings from multiple sources
+    
+    The synthesis process:
+    1. Sends all content to the LLM
+    2. Asks for key findings and analysis
+    3. Ensures proper citation of sources
+    4. Generates actionable insights
+    """
+    if not crawled_content:
+        return "No content available for synthesis.", []
+    
+    console.print("\n[cyan]🤖 Generating research synthesis...[/cyan]")
+    
+    # Prepare content for the AI
+    # We include source info for proper citations
+    content_sections = []
+    for i, content in enumerate(crawled_content, 1):
+        section = f"""
+SOURCE {i}:
+Title: {content['title']}
+URL: {content['url']}
+Content Preview:
+{content['markdown'][:1500]}...
+"""
+        content_sections.append(section)
+    
+    combined_content = "\n---\n".join(content_sections)
+    
+    try:
+        # Generate comprehensive synthesis
+        response = await litellm.acompletion(
+            model=config.llm_model,
+            messages=[{
+                "role": "user",
+                "content": f"""Research Query: "{query.original_query}"
+
+Based on the following sources, provide a comprehensive research synthesis.
+
+{combined_content}
+
+Please provide:
+1. An executive summary (2-3 sentences)
+2. Key findings (3-5 bullet points)
+3. Detailed analysis (2-3 paragraphs)
+4. Future implications or trends
+
+Format your response with clear sections and cite sources using [Source N] notation.
+Keep the total response under 800 words."""
+            }],
+            temperature=0.7  # Some creativity for synthesis
+        )
+        
+        synthesis = response.choices[0].message.content
+        
+        # Extract citations from the synthesis
+        citations = []
+        for i, content in enumerate(crawled_content, 1):
+            # Check if this source was cited
+            if f"[Source {i}]" in synthesis or f"Source {i}" in synthesis:
+                citations.append({
+                    'source_id': i,
+                    'title': content['title'],
+                    'url': content['url']
+                })
+        
+        return synthesis, citations
+        
+    except Exception as e:
+        console.print(f"[red]❌ Synthesis generation failed: {e}[/red]")
+        # Fallback to simple summary
+        summary = f"Research on '{query.original_query}' found {len(crawled_content)} relevant articles:\n\n"
+        for content in crawled_content[:3]:
+            summary += f"- {content['title']}\n  {content['url']}\n\n"
+        return summary, []
+
+# Example synthesis
+synthesis, citations = await generate_research_synthesis(enhanced, crawled, config)
+console.print(Panel(synthesis[:500] + "...", title="📝 Research Synthesis Preview"))
+
+# cell 14 type:markdown
+## Step 6: Complete Research Pipeline
+
+Now let's put it all together! This orchestrator function manages the entire research pipeline from query to final report. It coordinates all the components we've built, handling errors gracefully and providing progress updates.
+
+# cell 15 type:code
+async def research_pipeline(
+    query: str,
+    config: ResearchConfig = None
+) -> ResearchResult:
+    """
+    Main research pipeline orchestrator
+    
+    This brings together all components:
+    1. Query enhancement (AI-powered)
+    2. URL discovery (AsyncUrlSeeder)
+    3. Smart crawling (AsyncWebCrawler)
+    4. AI synthesis (LiteLLM)
+    
+    Returns a complete research result
+    """
+    if config is None:
+        config = ResearchConfig()
+    
+    start_time = datetime.now()
+    
+    # Display pipeline header
+    console.print(Panel(
+        f"[bold cyan]Research Pipeline[/bold cyan]\n\n"
+        f"[dim]Query:[/dim] {query}\n"
+        f"[dim]Domain:[/dim] {config.domain}",
+        title="🚀 Starting Research",
+        border_style="cyan"
+    ))
+    
+    # Step 1: Enhance query
+    console.print(f"\n[bold cyan]📝 Step 1: Query Processing[/bold cyan]")
+    if config.use_llm_enhancement:
+        research_query = await enhance_query_with_llm(query, config)
+    else:
+        # Simple fallback without AI
+        research_query = ResearchQuery(
+            original_query=query,
+            enhanced_query=query,
+            search_patterns=[f"*{word}*" for word in query.lower().split()],
+            timestamp=datetime.now().isoformat()
+        )
+    
+    # Step 2: Discover URLs
+    console.print(f"\n[bold cyan]🔍 Step 2: URL Discovery[/bold cyan]")
+    discovered_urls = await discover_urls(
+        domain=config.domain,
+        query=research_query,
+        config=config
+    )
+    
+    if not discovered_urls:
+        # No URLs found - return empty result
+        return ResearchResult(
+            query=research_query,
+            discovered_urls=[],
+            crawled_content=[],
+            synthesis="No relevant URLs found for the given query.",
+            citations=[],
+            metadata={'duration': str(datetime.now() - start_time)}
+        )
+    
+    # Step 3: Crawl selected URLs
+    console.print(f"\n[bold cyan]🕷️ Step 3: Content Crawling[/bold cyan]")
+    crawled_content = await crawl_selected_urls(
+        urls=discovered_urls,
+        query=research_query,
+        config=config
+    )
+    
+    # Step 4: Generate synthesis
+    console.print(f"\n[bold cyan]🤖 Step 4: Synthesis Generation[/bold cyan]")
+    synthesis, citations = await generate_research_synthesis(
+        query=research_query,
+        crawled_content=crawled_content,
+        config=config
+    )
+    
+    # Create final result
+    result = ResearchResult(
+        query=research_query,
+        discovered_urls=discovered_urls,
+        crawled_content=crawled_content,
+        synthesis=synthesis,
+        citations=citations,
+        metadata={
+            'duration': str(datetime.now() - start_time),
+            'domain': config.domain,
+            'timestamp': datetime.now().isoformat(),
+            'total_discovered': len(discovered_urls),
+            'total_crawled': len(crawled_content),
+            'total_cited': len(citations)
+        }
+    )
+    
+    # Display summary
+    duration = datetime.now() - start_time
+    console.print(Panel(
+        f"[bold green]✅ Research completed in {duration}[/bold green]\n\n"
+        f"📊 Discovered: {len(discovered_urls)} URLs\n"
+        f"🕷️ Crawled: {len(crawled_content)} pages\n"
+        f"📚 Citations: {len(citations)} sources",
+        title="🎉 Pipeline Complete",
+        border_style="green"
+    ))
+    
+    return result
+
+# Example: Run complete pipeline
+result = await research_pipeline("Champions League latest results", config)
+
+# cell 16 type:markdown
+## Step 7: Beautiful Output Formatting
+
+A good research report needs clear presentation. Here we format our results into a professional report with executive summary, key findings, and proper citations. This makes the research actionable and easy to share.
+
+# cell 17 type:code
+def format_research_output(result: ResearchResult) -> None:
+    """
+    Create a beautifully formatted research report
+    
+    Good formatting makes insights actionable:
+    - Clear structure with sections
+    - Highlighted key findings
+    - Proper source attribution
+    - Easy to scan and understand
+    """
+    # Header
+    console.print("\n" + "=" * 60)
+    console.print("[bold cyan]🔬 RESEARCH REPORT[/bold cyan]")
+    console.print("=" * 60)
+    
+    # Query information
+    console.print(f"\n[bold]Query:[/bold] {result.query.original_query}")
+    if result.query.enhanced_query != result.query.original_query:
+        console.print(f"[dim]Enhanced: {result.query.enhanced_query}[/dim]")
+    
+    # Statistics
+    stats_table = Table(show_header=False, box=None)
+    stats_table.add_column(style="cyan")
+    stats_table.add_column()
+    
+    stats_table.add_row("📊 URLs Discovered", str(result.metadata['total_discovered']))
+    stats_table.add_row("🕷️ Pages Crawled", str(result.metadata['total_crawled']))
+    stats_table.add_row("📚 Sources Cited", str(result.metadata['total_cited']))
+    stats_table.add_row("⏱️ Processing Time", result.metadata['duration'])
+    
+    console.print("\n[bold]Statistics:[/bold]")
+    console.print(stats_table)
+    
+    # Synthesis
+    console.print("\n[bold]📝 SYNTHESIS[/bold]")
+    console.print("-" * 60)
+    console.print(result.synthesis)
+    
+    # Citations
+    if result.citations:
+        console.print("\n[bold]📚 SOURCES[/bold]")
+        console.print("-" * 60)
+        for citation in result.citations:
+            console.print(f"\n[{citation['source_id']}] [cyan]{citation['title']}[/cyan]")
+            console.print(f"    [dim]{citation['url']}[/dim]")
+    
+    # Top discovered URLs
+    console.print("\n[bold]🔍 TOP DISCOVERED URLS[/bold]")
+    console.print("-" * 60)
+    
+    urls_table = Table()
+    urls_table.add_column("Score", style="cyan")
+    urls_table.add_column("Title")
+    urls_table.add_column("URL", style="dim")
+    
+    for url_data in result.discovered_urls[:5]:
+        score = f"{url_data.get('relevance_score', 0):.3f}"
+        title = "N/A"
+        if url_data.get('head_data') and url_data['head_data'].get('title'):
+            title = url_data['head_data']['title'][:40] + "..."
+        url = url_data['url'][:50] + "..."
+        
+        urls_table.add_row(score, title, url)
+    
+    console.print(urls_table)
+
+# Display the formatted report
+format_research_output(result)
+
+# cell 18 type:markdown
+## Step 8: Save Research Results
+
+Finally, let's save our research for future reference. We'll create both JSON (for data analysis) and Markdown (for reading) formats. This ensures your research is preserved and shareable.
+
+# cell 19 type:code
+async def save_research_results(
+    result: ResearchResult, 
+    config: ResearchConfig
+) -> Tuple[Path, Path]:
+    """
+    Save research results in multiple formats
+    
+    Why save in multiple formats?
+    - JSON: Perfect for further analysis or automation
+    - Markdown: Human-readable, great for sharing
+    """
+    # Create output directory
+    config.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Generate filename based on query and timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    query_slug = result.query.original_query[:30].replace(" ", "_").replace("/", "_")
+    base_filename = f"{timestamp}_{query_slug}"
+    
+    # Save JSON
+    json_path = config.output_dir / f"{base_filename}.json"
+    with open(json_path, 'w') as f:
+        json.dump(asdict(result), f, indent=2, default=str)
+    
+    # Create markdown report
+    md_content = [
+        f"# Research Report: {result.query.original_query}",
+        f"\n**Generated on:** {result.metadata.get('timestamp', 'N/A')}",
+        f"\n**Domain:** {result.metadata.get('domain', 'N/A')}",
+        f"\n**Processing time:** {result.metadata.get('duration', 'N/A')}",
+        "\n---\n",
+        "## Query Information",
+        f"- **Original Query:** {result.query.original_query}",
+        f"- **Enhanced Query:** {result.query.enhanced_query or 'N/A'}",
+        "\n## Statistics",
+        f"- **URLs Discovered:** {result.metadata['total_discovered']}",
+        f"- **Pages Crawled:** {result.metadata['total_crawled']}",
+        f"- **Sources Cited:** {result.metadata['total_cited']}",
+        "\n## Research Synthesis\n",
+        result.synthesis,
+        "\n## Sources\n"
+    ]
+    
+    # Add citations
+    for citation in result.citations:
+        md_content.extend([
+            f"### [{citation['source_id']}] {citation['title']}",
+            f"- **URL:** [{citation['url']}]({citation['url']})",
+            ""
+        ])
+    
+    # Add discovered URLs
+    md_content.extend([
+        "\n## Discovered URLs (Top 10)\n",
+        "| Score | Title | URL |",
+        "|-------|-------|-----|"
+    ])
+    
+    for url_data in result.discovered_urls[:10]:
+        score = url_data.get('relevance_score', 0)
+        title = 'N/A'
+        if url_data.get('head_data') and url_data['head_data'].get('title'):
+            title = url_data['head_data']['title'][:50] + '...'
+        url = url_data['url'][:60] + '...'
+        md_content.append(f"| {score:.3f} | {title} | {url} |")
+    
+    # Save markdown
+    md_path = config.output_dir / f"{base_filename}.md"
+    with open(md_path, 'w') as f:
+        f.write('\n'.join(md_content))
+    
+    console.print(f"\n[green]💾 Results saved:[/green]")
+    console.print(f"  JSON: {json_path}")
+    console.print(f"  Markdown: {md_path}")
+    
+    return json_path, md_path
+
+# Save our results
+json_path, md_path = await save_research_results(result, config)
+
+# cell 20 type:markdown
+## 🎯 Putting It All Together: Interactive Research Assistant
+
+Now let's create an interactive version where you can research any topic! This brings together everything we've learned into a user-friendly tool.
+
+# cell 21 type:code
+async def interactive_research_assistant():
+    """
+    Interactive research assistant with example queries
+    
+    This demonstrates how to build a user-friendly interface
+    for your research pipeline.
+    """
+    # Welcome message
+    console.print(Panel.fit(
+        "[bold cyan]🔬 AI Research Assistant[/bold cyan]\n\n"
+        "Powered by Crawl4AI's intelligent URL discovery\n"
+        "[dim]• Discover without crawling\n"
+        "• Score by relevance\n"
+        "• Crawl only what matters\n"
+        "• Generate AI insights[/dim]",
+        title="Welcome",
+        border_style="cyan"
+    ))
+    
+    # Example queries
+    examples = [
+        "Premier League transfer news and rumors",
+        "Champions League match results and analysis",
+        "Tennis grand slam tournament updates",
+        "Formula 1 race results and standings",
+        "NBA playoff predictions and analysis"
+    ]
+    
+    # Display examples
+    console.print("\n[bold]📋 Example queries:[/bold]")
+    for i, example in enumerate(examples, 1):
+        console.print(f"  {i}. {example}")
+    
+    # Get user input
+    console.print("\n[bold]Enter a number (1-5) or type your own query:[/bold]")
+    user_input = input("🔍 > ").strip()
+    
+    # Determine query
+    if user_input.isdigit() and 1 <= int(user_input) <= len(examples):
+        query = examples[int(user_input) - 1]
+    else:
+        query = user_input if user_input else examples[0]
+    
+    console.print(f"\n[cyan]Selected query: {query}[/cyan]")
+    
+    # Configuration options
+    console.print("\n[bold]Choose configuration:[/bold]")
+    console.print("  1. Quick (5 URLs, fast)")
+    console.print("  2. Standard (10 URLs, balanced)")
+    console.print("  3. Comprehensive (20 URLs, thorough)")
+    
+    config_choice = input("⚙️ > ").strip()
+    
+    # Create configuration
+    if config_choice == "1":
+        config = ResearchConfig(max_urls_to_crawl=5, top_k_urls=5)
+    elif config_choice == "3":
+        config = ResearchConfig(max_urls_to_crawl=20, top_k_urls=20)
+    else:
+        config = ResearchConfig()  # Standard
+    
+    # Run research
+    result = await research_pipeline(query, config)
+    
+    # Display results
+    format_research_output(result)
+    
+    # Save results
+    save_choice = input("\n💾 Save results? (y/n): ").strip().lower()
+    if save_choice == 'y':
+        await save_research_results(result, config)
+
+# Run the interactive assistant
+await interactive_research_assistant()
+
+# cell 22 type:markdown
+## 🚀 Advanced Tips and Best Practices
+
+### 1. Domain-Specific Research
+
+Customize the pipeline for specific domains:
+
+# cell 23 type:code
+# Research across multiple sports sites
+async def multi_domain_research(query: str):
+    """Research across multiple sports websites"""
+    
+    domains = [
+        "www.bbc.com/sport",
+        "www.espn.com",
+        "www.skysports.com"
+    ]
+    
+    all_results = []
+    
+    for domain in domains:
+        config = ResearchConfig(
+            domain=domain,
+            max_urls_to_crawl=5  # 5 per domain
+        )
+        
+        console.print(f"\n[cyan]Researching {domain}...[/cyan]")
+        result = await research_pipeline(query, config)
+        all_results.append(result)
+    
+    # Combine insights from all domains
+    console.print("\n[bold green]✅ Multi-domain research complete![/bold green]")
+    return all_results
+
+# Example usage
+# results = await multi_domain_research("World Cup 2024")
+
+# cell 24 type:markdown
+### 2. Performance Optimization
+
+Tips for faster research:
+
+# cell 25 type:code
+# Optimized configuration for speed
+speed_config = ResearchConfig(
+    # Reduce discovery scope
+    max_urls_discovery=50,  # Don't discover too many
+    
+    # Skip live checking (trust the sitemap)
+    live_check=False,
+    
+    # Increase parallelism
+    max_concurrent_crawls=10,
+    
+    # Skip AI enhancement for simple queries
+    use_llm_enhancement=False,
+    
+    # Use faster model
+    llm_model="gemini/gemini-1.5-flash"
+)
+
+console.print(Panel(
+    "[green]⚡ Speed Optimizations:[/green]\n\n"
+    "• Reduced discovery scope\n"
+    "• Disabled live URL checking\n"
+    "• Increased parallelism\n"
+    "• Using faster AI model",
+    title="Performance Tips"
+))
+
+# cell 26 type:markdown
+### 3. Caching Strategy
+
+The URL Seeder automatically caches results for efficiency:
+
+# cell 27 type:code
+# Cache demonstration
+console.print("[bold]🗄️ Understanding Caching:[/bold]\n")
+
+console.print("1. [cyan]First run:[/cyan] Fetches fresh data")
+console.print("   - Discovers URLs from sitemap/Common Crawl")
+console.print("   - Extracts metadata")
+console.print("   - Caches results for 7 days")
+
+console.print("\n2. [cyan]Subsequent runs:[/cyan] Uses cache (instant!)")
+console.print("   - No network requests needed")
+console.print("   - Same query returns cached results")
+
+console.print("\n3. [cyan]Force refresh:[/cyan] Bypass cache when needed")
+console.print("   - Set `force_refresh=True` in config")
+console.print("   - Useful for breaking news or updates")
+
+# Example with cache control
+cache_config = ResearchConfig(
+    force_refresh=True  # Always get fresh data
+)
+
+# cell 28 type:markdown
+## Agentic Design Patterns
+
+We've implemented a linear pipeline: Query → Enhance → Discover → Filter → Crawl → Synthesize. This is one of many possible agentic patterns.
+
+### Example: Reflection Pipeline
+
+Here's an advanced pattern with iterative refinement:
+
+```mermaid
+graph TD
+    A[🔍 User Query] --> B[🤖 Generate Multiple
Search Strategies]
+    B --> C1[Query 1]
+    B --> C2[Query 2]
+    B --> C3[Query N]
+    
+    C1 --> D[🌐 Parallel URL
Discovery]
+    C2 --> D
+    C3 --> D
+    
+    D --> E[🎯 Aggregate &
Score All URLs]
+    E --> F[🕷️ Smart Crawling]
+    
+    F --> G{📊 Sufficient
Information?}
+    G -->|No| H[🔄 Analyze Gaps]
+    H --> B
+    
+    G -->|Yes| K[🧠 AI Synthesis]
+    K --> L[📄 Comprehensive
Report]
+```
+
+This design:
+- Generates multiple search angles
+- Evaluates information completeness
+- Iteratively refines queries based on gaps
+- Continues until sufficient information is gathered
+
+Other patterns to consider:
+- **Comparative Analysis**: Research across multiple domains
+- **Fact Verification**: Cross-reference multiple sources
+- **Trend Detection**: Time-based discovery and analysis
+
+# cell 29 type:markdown
+## 🎓 Summary & Next Steps
+
+### What You've Learned
+
+You've built a complete AI research assistant that:
+
+✅ **Discovers URLs intelligently** - No blind crawling  
+✅ **Scores by relevance** - Focus on what matters  
+✅ **Crawls efficiently** - Parallel processing  
+✅ **Generates insights** - AI-powered synthesis  
+✅ **Saves results** - JSON and Markdown formats  
+
+### Key Advantages
+
+1. **Efficiency**: Discover 1000s of URLs in seconds, crawl only the best
+2. **Intelligence**: BM25 scoring ensures relevance
+3. **Scalability**: Works across multiple domains
+4. **Flexibility**: Configurable for any use case
+
+### Next Steps
+
+1. **Customize for your domain**: Adapt the pipeline for your specific needs
+2. **Add persistence**: Store results in a database
+3. **Build an API**: Turn this into a web service
+4. **Schedule updates**: Monitor topics over time
+5. **Enhance with more AI**: Add summarization, sentiment analysis, etc.
+
+### Resources
+
+- 🐙 **GitHub**: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
+- 📚 **Documentation**: [crawl4ai.com/docs](https://crawl4ai.com/docs)
+- 💬 **Discord**: [Join our community](https://discord.gg/crawl4ai)
+
+Thank you for learning with Crawl4AI! 🙏
+
+Happy researching! 🚀🔬
\ No newline at end of file
diff --git a/docs/examples/url_seeder/url_seeder_demo.py b/docs/examples/url_seeder/url_seeder_demo.py
new file mode 100644
index 0000000..1ab1f63
--- /dev/null
+++ b/docs/examples/url_seeder/url_seeder_demo.py
@@ -0,0 +1,263 @@
+"""
+URL Seeder Demo - Interactive showcase of Crawl4AI's URL discovery capabilities
+
+This demo shows:
+1. Basic URL discovery from sitemaps and Common Crawl
+2. Cache management and forced refresh
+3. Live URL validation and metadata extraction
+4. BM25 relevance scoring for intelligent filtering
+5. Integration with AsyncWebCrawler for the complete pipeline
+6. Multi-domain discovery across multiple sites
+
+Note: The AsyncUrlSeeder now supports context manager protocol for automatic cleanup.
+"""
+
+import asyncio
+import time
+from datetime import datetime
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, BarColumn, TimeElapsedColumn
+from rich.prompt import Prompt, Confirm
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    AsyncUrlSeeder,
+    SeedingConfig
+)
+
+console = Console()
+
+console.rule("[bold green]🌐 Crawl4AI URL Seeder: Interactive Demo")
+
+DOMAIN = "crawl4ai.com"
+
+# Utils
+
+def print_head_info(head_data):
+    table = Table(title=" Metadata", expand=True)
+    table.add_column("Key", style="cyan", no_wrap=True)
+    table.add_column("Value", style="magenta")
+    
+    if not head_data:
+        console.print("[yellow]No head data found.")
+        return
+
+    if head_data.get("title"):
+        table.add_row("title", head_data["title"])
+    if head_data.get("charset"):
+        table.add_row("charset", head_data["charset"])
+    for k, v in head_data.get("meta", {}).items():
+        table.add_row(f"meta:{k}", v)
+    for rel, items in head_data.get("link", {}).items():
+        for item in items:
+            table.add_row(f"link:{rel}", item.get("href", ""))
+    console.print(table)
+
+
+async def section_1_basic_exploration(seed: AsyncUrlSeeder):
+    console.rule("[bold cyan]1. Basic Seeding")
+    cfg = SeedingConfig(source="cc+sitemap", pattern="*", verbose=True)
+    
+    start_time = time.time()
+    with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
+        p.add_task(description="Fetching from Common Crawl + Sitemap...", total=None)
+        urls = await seed.urls(DOMAIN, cfg)
+    elapsed = time.time() - start_time
+
+    console.print(f"[green]✓ Fetched {len(urls)} URLs in {elapsed:.2f} seconds")
+    console.print(f"[dim]  Speed: {len(urls)/elapsed:.0f} URLs/second[/dim]\n")
+    
+    console.print("[bold]Sample URLs:[/bold]")
+    for u in urls[:5]:
+        console.print(f"  • {u['url']}")
+
+
+async def section_2_cache_demo(seed: AsyncUrlSeeder):
+    console.rule("[bold cyan]2. Caching Demonstration")   
+    console.print("[yellow]Using `force=True` to bypass cache and fetch fresh data.[/yellow]")
+    cfg = SeedingConfig(source="cc", pattern="*crawl4ai.com/core/*", verbose=False, force = True)
+    await seed.urls(DOMAIN, cfg)
+
+async def section_3_live_head(seed: AsyncUrlSeeder):
+    console.rule("[bold cyan]3. Live Check + Head Extraction")
+    cfg = SeedingConfig(
+        extract_head=True,
+        concurrency=10,
+        hits_per_sec=5,
+        pattern="*crawl4ai.com/*",
+        max_urls=10,
+        verbose=False,
+    )
+    urls = await seed.urls(DOMAIN, cfg)
+    
+    valid = [u for u in urls if u["status"] == "valid"]
+    console.print(f"[green]Valid: {len(valid)} / {len(urls)}")
+    if valid:
+        print_head_info(valid[0]["head_data"])
+
+
+async def section_4_bm25_scoring(seed: AsyncUrlSeeder):
+    console.rule("[bold cyan]4. BM25 Relevance Scoring")
+    console.print("[yellow]Using AI-powered relevance scoring to find the most relevant content[/yellow]")
+    
+    query = "markdown generation extraction strategies"
+    cfg = SeedingConfig(
+        source="sitemap",
+        extract_head=True,
+        query=query,
+        scoring_method="bm25",
+        score_threshold=0.3,  # Only URLs with >30% relevance
+        max_urls=20,
+        verbose=False
+    )
+    
+    with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
+        p.add_task(description=f"Searching for: '{query}'", total=None)
+        urls = await seed.urls(DOMAIN, cfg)
+    
+    console.print(f"[green]Found {len(urls)} relevant URLs (score > 0.3)")
+    
+    # Show top results with scores
+    table = Table(title="Top 5 Most Relevant Pages", expand=True)
+    table.add_column("Score", style="cyan", width=8)
+    table.add_column("Title", style="magenta")
+    table.add_column("URL", style="blue", overflow="fold")
+    
+    for url in urls[:5]:
+        score = f"{url['relevance_score']:.2f}"
+        title = url['head_data'].get('title', 'No title')[:60] + "..."
+        table.add_row(score, title, url['url'])
+    
+    console.print(table)
+
+async def section_5_keyword_filter_to_agent(seed: AsyncUrlSeeder):
+    console.rule("[bold cyan]5. Complete Pipeline: Discover → Filter → Crawl")
+    cfg = SeedingConfig(
+        extract_head=True,
+        concurrency=20,
+        hits_per_sec=10,
+        max_urls=10,
+        pattern="*crawl4ai.com/*",
+        force=True,
+    )
+    urls = await seed.urls(DOMAIN, cfg)
+
+    keywords = ["deep crawling", "markdown", "llm"]
+    selected = [u for u in urls if any(k in str(u["head_data"]).lower() for k in keywords)]
+
+    console.print(f"[cyan]Selected {len(selected)} URLs with relevant keywords:")
+    for u in selected[:10]:
+        console.print("•", u["url"])
+
+    console.print("\n[yellow]Passing above URLs to arun_many() LLM agent for crawling...")
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        crawl_run_config = CrawlerRunConfig(
+                # Example crawl settings for these URLs:
+                only_text=True, # Just get text content
+                screenshot=False,
+                pdf=False,
+                word_count_threshold=50, # Only process pages with at least 50 words
+                stream=True,
+                verbose=False # Keep logs clean for arun_many in this demo
+            )
+
+        # Extract just the URLs from the selected results
+        urls_to_crawl = [u["url"] for u in selected]
+        
+        # We'll stream results for large lists, but collect them here for demonstration
+        crawled_results_stream = await crawler.arun_many(urls_to_crawl, config=crawl_run_config)
+        final_crawled_data = []
+        async for result in crawled_results_stream:
+            final_crawled_data.append(result)
+            if len(final_crawled_data) % 5 == 0:
+                print(f"   Processed {len(final_crawled_data)}/{len(urls_to_crawl)} URLs...")
+
+        print(f"\n   Successfully crawled {len(final_crawled_data)} URLs.")
+        if final_crawled_data:
+            print("\n   Example of a crawled result's URL and Markdown (first successful one):")
+            for result in final_crawled_data:
+                if result.success and result.markdown.raw_markdown:
+                    print(f"     URL: {result.url}")
+                    print(f"     Markdown snippet: {result.markdown.raw_markdown[:200]}...")
+                    break
+            else:
+                print("   No successful crawls with markdown found.")
+        else:
+            print("   No successful crawls found.")    
+
+
+async def section_6_multi_domain(seed: AsyncUrlSeeder):
+    console.rule("[bold cyan]6. Multi-Domain Discovery")
+    console.print("[yellow]Discovering Python tutorials across multiple educational sites[/yellow]\n")
+    
+    domains = ["docs.python.org", "realpython.com", "docs.crawl4ai.com"]
+    cfg = SeedingConfig(
+        source="sitemap",
+        extract_head=True,
+        query="python tutorial guide",
+        scoring_method="bm25",
+        score_threshold=0.2,
+        max_urls=5  # Per domain
+    )
+    
+    start_time = time.time()
+    with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
+        task = p.add_task(description="Discovering across domains...", total=None)
+        results = await seed.many_urls(domains, cfg)
+    elapsed = time.time() - start_time
+    
+    total_urls = sum(len(urls) for urls in results.values())
+    console.print(f"[green]✓ Found {total_urls} relevant URLs across {len(domains)} domains in {elapsed:.2f}s\n")
+    
+    # Show results per domain
+    for domain, urls in results.items():
+        console.print(f"[bold]{domain}:[/bold] {len(urls)} relevant pages")
+        if urls:
+            top = urls[0]
+            console.print(f"  Top result: [{top['relevance_score']:.2f}] {top['head_data'].get('title', 'No title')}")
+
+
+async def main():
+    async with AsyncUrlSeeder() as seed:
+        # Interactive menu
+        sections = {
+            "1": ("Basic URL Discovery", section_1_basic_exploration),
+            "2": ("Cache Management Demo", section_2_cache_demo),
+            "3": ("Live Check & Metadata Extraction", section_3_live_head),
+            "4": ("BM25 Relevance Scoring", section_4_bm25_scoring),
+            "5": ("Complete Pipeline (Discover → Filter → Crawl)", section_5_keyword_filter_to_agent),
+            "6": ("Multi-Domain Discovery", section_6_multi_domain),
+            "7": ("Run All Demos", None)
+        }
+        
+        console.print("\n[bold]Available Demos:[/bold]")
+        for key, (title, _) in sections.items():
+            console.print(f"  {key}. {title}")
+        
+        choice = Prompt.ask("\n[cyan]Which demo would you like to run?[/cyan]", 
+                           choices=list(sections.keys()), 
+                           default="7")
+        
+        console.print()
+        
+        if choice == "7":
+            # Run all demos
+            for key, (title, func) in sections.items():
+                if key != "7" and func:
+                    await func(seed)
+                    if key != "6":  # Don't pause after the last demo
+                        if not Confirm.ask("\n[yellow]Continue to next demo?[/yellow]", default=True):
+                            break
+                        console.print()
+        else:
+            # Run selected demo
+            _, func = sections[choice]
+            await func(seed)
+        
+        console.rule("[bold green]Demo Complete ✔︎")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/examples/url_seeder/url_seeder_quick_demo.py b/docs/examples/url_seeder/url_seeder_quick_demo.py
new file mode 100644
index 0000000..f48c5ee
--- /dev/null
+++ b/docs/examples/url_seeder/url_seeder_quick_demo.py
@@ -0,0 +1,128 @@
+"""
+🚀 URL Seeder + AsyncWebCrawler = Magic!
+Quick demo showing discovery → filter → crawl pipeline
+
+Note: Uses context manager for automatic cleanup of resources.
+"""
+import asyncio, os
+from crawl4ai import AsyncUrlSeeder, AsyncWebCrawler, SeedingConfig, CrawlerRunConfig, AsyncLogger, DefaultMarkdownGenerator 
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# 🔍 Example 1: Discover ALL → Filter → Crawl
+async def discover_and_crawl():
+    """Find Python module tutorials & extract them all!"""
+    async with AsyncUrlSeeder(logger=AsyncLogger()) as seeder:
+        # Step 1: See how many URLs exist (spoiler: A LOT!)
+        print("📊 Let's see what RealPython has...")
+        all_urls = await seeder.urls("realpython.com", 
+                                    SeedingConfig(source="sitemap"))
+        print(f"😱 Found {len(all_urls)} total URLs!")
+        
+        # Step 2: Filter for Python modules (perfect size ~13)
+        print("\n🎯 Filtering for 'python-modules' tutorials...")
+        module_urls = await seeder.urls("realpython.com", 
+                                      SeedingConfig(
+                                          source="sitemap",
+                                          pattern="*python-modules*",
+                                          live_check=True  # Make sure they're alive!
+                                      ))
+        
+        print(f"✨ Found {len(module_urls)} module tutorials")
+        for url in module_urls[:3]:  # Show first 3
+            status = "✅" if url["status"] == "valid" else "❌"
+            print(f"{status} {url['url']}")
+    
+    # Step 3: Crawl them all with pruning (keep it lean!)
+    print("\n🕷️ Crawling all module tutorials...")
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(  # Smart filtering!
+                    threshold=0.48,  # Remove fluff
+                    threshold_type="fixed",
+                ),
+            ),
+            only_text=True,
+            stream=True,  
+        )
+        
+        # Extract just the URLs from the seeder results
+        urls_to_crawl = [u["url"] for u in module_urls[:5]]
+        results = await crawler.arun_many(urls_to_crawl, config=config)
+        
+        # Process & save
+        saved = 0
+        async for result in results:
+            if result.success:
+                # Save each tutorial (name from URL)
+                name = result.url.split("/")[-2] + ".md"
+                name = os.path.join(CURRENT_DIR, name)
+                with open(name, "w") as f:
+                    f.write(result.markdown.fit_markdown)
+                saved += 1
+                print(f"💾 Saved: {name}")
+        
+        print(f"\n🎉 Successfully saved {saved} tutorials!")
+
+# 🔍 Example 2: Beautiful Soup articles with metadata peek
+async def explore_beautifulsoup():
+    """Discover BeautifulSoup content & peek at metadata"""
+    async with AsyncUrlSeeder(logger=AsyncLogger()) as seeder:
+        print("🍲 Looking for Beautiful Soup articles...")
+        soup_urls = await seeder.urls("realpython.com",
+                                    SeedingConfig(
+                                        source="sitemap",
+                                        pattern="*beautiful-soup*",
+                                        extract_head=True  # Get the metadata!
+                                    ))
+        
+        print(f"\n📚 Found {len(soup_urls)} Beautiful Soup articles:\n")
+        
+        # Show what we discovered
+        for i, url in enumerate(soup_urls, 1):
+            meta = url["head_data"]["meta"]
+            
+            print(f"{i}. {url['head_data']['title']}")
+            print(f"   📝 {meta.get('description', 'No description')[:60]}...")
+            print(f"   👤 By: {meta.get('author', 'Unknown')}")
+            print(f"   🔗 {url['url']}\n")
+
+# 🔍 Example 3: Smart search with BM25 relevance scoring
+async def smart_search_with_bm25():
+    """Use AI-powered relevance scoring to find the best content"""
+    async with AsyncUrlSeeder(logger=AsyncLogger()) as seeder:
+        print("🧠 Smart search: 'web scraping tutorial quiz'")
+        
+        # Search with BM25 scoring - AI finds the best matches!
+        results = await seeder.urls("realpython.com",
+                                  SeedingConfig(
+                                      source="sitemap",
+                                      pattern="*beautiful-soup*",
+                                      extract_head=True,
+                                      query="web scraping tutorial quiz",  # Our search
+                                      scoring_method="bm25",
+                                      score_threshold=0.2  # Quality filter
+                                  ))
+        
+        print(f"\n🎯 Top {len(results)} most relevant results:\n")
+        
+        # Show ranked results with relevance scores
+        for i, result in enumerate(results[:3], 1):
+            print(f"{i}. [{result['relevance_score']:.2f}] {result['head_data']['title']}")
+            print(f"   🔗 {result['url'][:60]}...")
+        
+        print("\n✨ BM25 automatically ranked by relevance!")
+
+# 🎬 Run the show!
+async def main():
+    print("=" * 60)
+    await discover_and_crawl()
+    print("\n" + "=" * 60 + "\n")
+    await explore_beautifulsoup()
+    print("\n" + "=" * 60 + "\n")
+    await smart_search_with_bm25()
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/use_geo_location.py b/docs/examples/use_geo_location.py
new file mode 100644
index 0000000..2cfc866
--- /dev/null
+++ b/docs/examples/use_geo_location.py
@@ -0,0 +1,70 @@
+# use_geo_location.py
+"""
+Example: override locale, timezone, and geolocation using Crawl4ai patterns.
+
+This demo uses `AsyncWebCrawler.arun()` to fetch a page with
+browser context primed for specific locale, timezone, and GPS,
+and saves a screenshot for visual verification.
+"""
+
+import asyncio
+import base64
+from pathlib import Path
+from typing import List
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    BrowserConfig,
+    GeolocationConfig,
+    CrawlResult,
+)
+
+async def demo_geo_override():
+    """Demo: Crawl a geolocation-test page with overrides and screenshot."""
+    print("\n=== Geo-Override Crawl ===")
+
+    # 1) Browser setup: use Playwright-managed contexts
+    browser_cfg = BrowserConfig(
+        headless=False,
+        viewport_width=1280,
+        viewport_height=720,
+        use_managed_browser=False,
+    )
+
+    # 2) Run config: include locale, timezone_id, geolocation, and screenshot
+    run_cfg = CrawlerRunConfig(
+        url="https://browserleaks.com/geo",          # test page that shows your location
+        locale="en-US",                              # Accept-Language & UI locale
+        timezone_id="America/Los_Angeles",           # JS Date()/Intl timezone
+        geolocation=GeolocationConfig(                 # override GPS coords
+            latitude=34.0522,
+            longitude=-118.2437,
+            accuracy=10.0,
+        ),
+        screenshot=True,                               # capture screenshot after load
+        session_id="geo_test",                       # reuse context if rerunning
+        delay_before_return_html=5
+    )
+
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        # 3) Run crawl (returns list even for single URL)
+        results: List[CrawlResult] = await crawler.arun(
+            url=run_cfg.url,
+            config=run_cfg,            
+        )
+        result = results[0]
+
+        # 4) Save screenshot and report path
+        if result.screenshot:
+            __current_dir = Path(__file__).parent
+            out_dir = __current_dir / "tmp"
+            out_dir.mkdir(exist_ok=True)
+            shot_path = out_dir / "geo_test.png"
+            with open(shot_path, "wb") as f:
+                f.write(base64.b64decode(result.screenshot))
+            print(f"Saved screenshot to {shot_path}")
+        else:
+            print("No screenshot captured, check configuration.")
+
+if __name__ == "__main__":
+    asyncio.run(demo_geo_override())
diff --git a/docs/examples/virtual_scroll_example.py b/docs/examples/virtual_scroll_example.py
new file mode 100644
index 0000000..7be99e7
--- /dev/null
+++ b/docs/examples/virtual_scroll_example.py
@@ -0,0 +1,367 @@
+"""
+Example of using the virtual scroll feature to capture content from pages
+with virtualized scrolling (like Twitter, Instagram, or other infinite scroll feeds).
+
+This example demonstrates virtual scroll with a local test server serving
+different types of scrolling behaviors from HTML files in the assets directory.
+"""
+
+import asyncio
+import os
+import http.server
+import socketserver
+import threading
+from pathlib import Path
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, VirtualScrollConfig, CacheMode, BrowserConfig
+
+# Get the assets directory path
+ASSETS_DIR = Path(__file__).parent / "assets"
+
+class TestServer:
+    """Simple HTTP server to serve our test HTML files"""
+    
+    def __init__(self, port=8080):
+        self.port = port
+        self.httpd = None
+        self.server_thread = None
+        
+    async def start(self):
+        """Start the test server"""
+        Handler = http.server.SimpleHTTPRequestHandler
+        
+        # Save current directory and change to assets directory
+        self.original_cwd = os.getcwd()
+        os.chdir(ASSETS_DIR)
+        
+        # Try to find an available port
+        for _ in range(10):
+            try:
+                self.httpd = socketserver.TCPServer(("", self.port), Handler)
+                break
+            except OSError:
+                self.port += 1
+                
+        if self.httpd is None:
+            raise RuntimeError("Could not find available port")
+            
+        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+        
+        # Give server time to start
+        await asyncio.sleep(0.5)
+        
+        print(f"Test server started on http://localhost:{self.port}")
+        return self.port
+        
+    def stop(self):
+        """Stop the test server"""
+        if self.httpd:
+            self.httpd.shutdown()
+        # Restore original directory
+        if hasattr(self, 'original_cwd'):
+            os.chdir(self.original_cwd)
+            
+
+async def example_twitter_like_virtual_scroll():
+    """
+    Example 1: Twitter-like virtual scroll where content is REPLACED.
+    This is the classic virtual scroll use case - only visible items exist in DOM.
+    """
+    print("\n" + "="*60)
+    print("EXAMPLE 1: Twitter-like Virtual Scroll")
+    print("="*60)
+    
+    server = TestServer()
+    port = await server.start()
+    
+    try:
+        # Configure virtual scroll for Twitter-like timeline
+        virtual_config = VirtualScrollConfig(
+            container_selector="#timeline",  # The scrollable container
+            scroll_count=50,  # Scroll up to 50 times to get all content
+            scroll_by="container_height",  # Scroll by container's height
+            wait_after_scroll=0.3  # Wait 300ms after each scroll
+        )
+        
+        config = CrawlerRunConfig(
+            virtual_scroll_config=virtual_config,
+            cache_mode=CacheMode.BYPASS
+        )
+        
+        # TIP: Set headless=False to watch the scrolling happen!
+        browser_config = BrowserConfig(
+            headless=False,
+            viewport={"width": 1280, "height": 800}
+        )
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(
+                url=f"http://localhost:{port}/virtual_scroll_twitter_like.html",
+                config=config
+            )
+            
+            # Count tweets captured
+            import re
+            tweets = re.findall(r'data-tweet-id="(\d+)"', result.html)
+            unique_tweets = sorted(set(int(id) for id in tweets))
+            
+            print(f"\n📊 Results:")
+            print(f"   Total HTML length: {len(result.html):,} characters")
+            print(f"   Tweets captured: {len(unique_tweets)} unique tweets")
+            if unique_tweets:
+                print(f"   Tweet IDs range: {min(unique_tweets)} to {max(unique_tweets)}")
+                print(f"   Expected range: 0 to 499 (500 tweets total)")
+                
+                if len(unique_tweets) == 500:
+                    print(f"   ✅ SUCCESS! All tweets captured!")
+                else:
+                    print(f"   ⚠️  Captured {len(unique_tweets)}/500 tweets")
+                    
+    finally:
+        server.stop()
+
+
+async def example_traditional_append_scroll():
+    """
+    Example 2: Traditional infinite scroll where content is APPENDED.
+    No virtual scroll needed - all content stays in DOM.
+    """
+    print("\n" + "="*60)
+    print("EXAMPLE 2: Traditional Append-Only Scroll")
+    print("="*60)
+    
+    server = TestServer()
+    port = await server.start()
+    
+    try:
+        # Configure virtual scroll
+        virtual_config = VirtualScrollConfig(
+            container_selector=".posts-container",
+            scroll_count=15,  # Less scrolls needed since content accumulates
+            scroll_by=500,  # Scroll by 500 pixels
+            wait_after_scroll=0.4
+        )
+        
+        config = CrawlerRunConfig(
+            virtual_scroll_config=virtual_config,
+            cache_mode=CacheMode.BYPASS
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=f"http://localhost:{port}/virtual_scroll_append_only.html",
+                config=config
+            )
+            
+            # Count posts
+            import re
+            posts = re.findall(r'data-post-id="(\d+)"', result.html)
+            unique_posts = sorted(set(int(id) for id in posts))
+            
+            print(f"\n📊 Results:")
+            print(f"   Total HTML length: {len(result.html):,} characters")
+            print(f"   Posts captured: {len(unique_posts)} unique posts")
+            
+            if unique_posts:
+                print(f"   Post IDs range: {min(unique_posts)} to {max(unique_posts)}")
+                print(f"   ℹ️  Note: This page appends content, so virtual scroll")
+                print(f"       just helps trigger more loads. All content stays in DOM.")
+                
+    finally:
+        server.stop()
+
+
+async def example_instagram_grid():
+    """
+    Example 3: Instagram-like grid with virtual scroll.
+    Grid layout where only visible rows are rendered.
+    """
+    print("\n" + "="*60)
+    print("EXAMPLE 3: Instagram Grid Virtual Scroll")
+    print("="*60)
+    
+    server = TestServer()
+    port = await server.start()
+    
+    try:
+        # Configure for grid layout
+        virtual_config = VirtualScrollConfig(
+            container_selector=".feed-container",  # Container with the grid
+            scroll_count=100,  # Many scrolls for 999 posts
+            scroll_by="container_height",
+            wait_after_scroll=0.2  # Faster scrolling for grid
+        )
+        
+        config = CrawlerRunConfig(
+            virtual_scroll_config=virtual_config,
+            cache_mode=CacheMode.BYPASS,
+            screenshot=True  # Take a screenshot of the final grid
+        )
+        
+        # Show browser for this visual example
+        browser_config = BrowserConfig(
+            headless=False,
+            viewport={"width": 1200, "height": 900}
+        )
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(
+                url=f"http://localhost:{port}/virtual_scroll_instagram_grid.html",
+                config=config
+            )
+            
+            # Count posts in grid
+            import re
+            posts = re.findall(r'data-post-id="(\d+)"', result.html)
+            unique_posts = sorted(set(int(id) for id in posts))
+            
+            print(f"\n📊 Results:")
+            print(f"   Posts in grid: {len(unique_posts)} unique posts")
+            if unique_posts:
+                print(f"   Post IDs range: {min(unique_posts)} to {max(unique_posts)}")
+                print(f"   Expected: 0 to 998 (999 posts total)")
+                
+            # Save screenshot
+            if result.screenshot:
+                import base64
+                with open("instagram_grid_result.png", "wb") as f:
+                    f.write(base64.b64decode(result.screenshot))
+                print(f"   📸 Screenshot saved as instagram_grid_result.png")
+                
+    finally:
+        server.stop()
+
+
+async def example_mixed_content():
+    """
+    Example 4: News feed with mixed behavior.
+    Featured articles stay (no virtual scroll), regular articles are virtualized.
+    """
+    print("\n" + "="*60)
+    print("EXAMPLE 4: News Feed with Mixed Behavior")
+    print("="*60)
+    
+    server = TestServer()
+    port = await server.start()
+    
+    try:
+        # Configure virtual scroll
+        virtual_config = VirtualScrollConfig(
+            container_selector="#newsContainer",
+            scroll_count=25,
+            scroll_by="container_height",
+            wait_after_scroll=0.3
+        )
+        
+        config = CrawlerRunConfig(
+            virtual_scroll_config=virtual_config,
+            cache_mode=CacheMode.BYPASS
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=f"http://localhost:{port}/virtual_scroll_news_feed.html",
+                config=config
+            )
+            
+            # Count different types of articles
+            import re
+            featured = re.findall(r'data-article-id="featured-\d+"', result.html)
+            regular = re.findall(r'data-article-id="article-(\d+)"', result.html)
+            
+            print(f"\n📊 Results:")
+            print(f"   Featured articles: {len(set(featured))} (always visible)")
+            print(f"   Regular articles: {len(set(regular))} unique articles")
+            
+            if regular:
+                regular_ids = sorted(set(int(id) for id in regular))
+                print(f"   Regular article IDs: {min(regular_ids)} to {max(regular_ids)}")
+                print(f"   ℹ️  Note: Featured articles stay in DOM, only regular")
+                print(f"       articles are replaced during virtual scroll")
+                
+    finally:
+        server.stop()
+
+
+async def compare_with_without_virtual_scroll():
+    """
+    Comparison: Show the difference between crawling with and without virtual scroll.
+    """
+    print("\n" + "="*60)
+    print("COMPARISON: With vs Without Virtual Scroll")
+    print("="*60)
+    
+    server = TestServer()
+    port = await server.start()
+    
+    try:
+        url = f"http://localhost:{port}/virtual_scroll_twitter_like.html"
+        
+        # First, crawl WITHOUT virtual scroll
+        print("\n1️⃣  Crawling WITHOUT virtual scroll...")
+        async with AsyncWebCrawler() as crawler:
+            config_normal = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+            result_normal = await crawler.arun(url=url, config=config_normal)
+            
+            # Count items
+            import re
+            tweets_normal = len(set(re.findall(r'data-tweet-id="(\d+)"', result_normal.html)))
+            
+        # Then, crawl WITH virtual scroll  
+        print("2️⃣  Crawling WITH virtual scroll...")
+        virtual_config = VirtualScrollConfig(
+            container_selector="#timeline",
+            scroll_count=50,
+            scroll_by="container_height",
+            wait_after_scroll=0.2
+        )
+        
+        config_virtual = CrawlerRunConfig(
+            virtual_scroll_config=virtual_config,
+            cache_mode=CacheMode.BYPASS
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result_virtual = await crawler.arun(url=url, config=config_virtual)
+            
+            # Count items
+            tweets_virtual = len(set(re.findall(r'data-tweet-id="(\d+)"', result_virtual.html)))
+            
+        # Compare results
+        print(f"\n📊 Comparison Results:")
+        print(f"   Without virtual scroll: {tweets_normal} tweets (only initial visible)")
+        print(f"   With virtual scroll: {tweets_virtual} tweets (all content captured)")
+        print(f"   Improvement: {tweets_virtual / tweets_normal if tweets_normal > 0 else 'N/A':.1f}x more content!")
+        
+        print(f"\n   HTML size without: {len(result_normal.html):,} characters")
+        print(f"   HTML size with: {len(result_virtual.html):,} characters")
+        
+    finally:
+        server.stop()
+
+
+if __name__ == "__main__":
+    print("""
+╔════════════════════════════════════════════════════════════╗
+║           Virtual Scroll Examples for Crawl4AI             ║
+╚════════════════════════════════════════════════════════════╝
+
+These examples demonstrate different virtual scroll scenarios:
+1. Twitter-like (content replaced) - Classic virtual scroll
+2. Traditional append - Content accumulates 
+3. Instagram grid - Visual grid layout
+4. Mixed behavior - Some content stays, some virtualizes
+
+Starting examples...
+""")
+    
+    # Run all examples
+    asyncio.run(example_twitter_like_virtual_scroll())
+    asyncio.run(example_traditional_append_scroll())
+    asyncio.run(example_instagram_grid())
+    asyncio.run(example_mixed_content())
+    asyncio.run(compare_with_without_virtual_scroll())
+    
+    print("\n✅ All examples completed!")
+    print("\nTIP: Set headless=False in BrowserConfig to watch the scrolling in action!")
\ No newline at end of file
diff --git a/docs/examples/website-to-api/.gitignore b/docs/examples/website-to-api/.gitignore
new file mode 100644
index 0000000..8e88417
--- /dev/null
+++ b/docs/examples/website-to-api/.gitignore
@@ -0,0 +1,221 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis 
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+#directories
+models
+schemas
+saved_requests
\ No newline at end of file
diff --git a/docs/examples/website-to-api/README.md b/docs/examples/website-to-api/README.md
new file mode 100644
index 0000000..12ba4c4
--- /dev/null
+++ b/docs/examples/website-to-api/README.md
@@ -0,0 +1,252 @@
+# Web Scraper API with Custom Model Support
+
+A powerful web scraping API that converts any website into structured data using AI. Features a beautiful minimalist frontend interface and support for custom LLM models!
+
+## Features
+
+- **AI-Powered Scraping**: Provide a URL and plain English query to extract structured data
+- **Beautiful Frontend**: Modern minimalist black-and-white interface with smooth UX
+- **Custom Model Support**: Use any LLM provider (OpenAI, Gemini, Anthropic, etc.) with your own API keys
+- **Model Management**: Save, list, and manage multiple model configurations via web interface
+- **Dual Scraping Approaches**: Choose between Schema-based (faster) or LLM-based (more flexible) extraction
+- **API Request History**: Automatic saving and display of all API requests with cURL commands
+- **Schema Caching**: Intelligent caching of generated schemas for faster subsequent requests
+- **Duplicate Prevention**: Avoids saving duplicate requests (same URL + query)
+- **RESTful API**: Easy-to-use HTTP endpoints for all operations
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+### 2. Start the API Server
+
+```bash
+python app.py
+```
+
+The server will start on `http://localhost:8000` with a beautiful web interface!
+
+### 3. Using the Web Interface
+
+Once the server is running, open your browser and go to `http://localhost:8000` to access the modern web interface!
+
+#### Pages:
+- **Scrape Data**: Enter URLs and queries to extract structured data
+- **Models**: Manage your AI model configurations (add, list, delete)
+- **API Requests**: View history of all scraping requests with cURL commands
+
+#### Features:
+- **Minimalist Design**: Clean black-and-white theme inspired by modern web apps
+- **Real-time Results**: See extracted data in formatted JSON
+- **Copy to Clipboard**: Easy copying of results
+- **Toast Notifications**: User-friendly feedback
+- **Dual Scraping Modes**: Choose between Schema-based and LLM-based approaches
+
+## Model Management
+
+### Adding Models via Web Interface
+
+1. Go to the **Models** page
+2. Enter your model details:
+   - **Provider**: LLM provider (e.g., `gemini/gemini-2.5-flash`, `openai/gpt-4o`)
+   - **API Token**: Your API key for the provider
+3. Click "Add Model"
+
+### API Usage for Model Management
+
+#### Save a Model Configuration
+
+```bash
+curl -X POST "http://localhost:8000/models" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "provider": "gemini/gemini-2.5-flash",
+    "api_token": "your-api-key-here"
+  }'
+```
+
+#### List Saved Models
+
+```bash
+curl -X GET "http://localhost:8000/models"
+```
+
+#### Delete a Model Configuration
+
+```bash
+curl -X DELETE "http://localhost:8000/models/my-gemini"
+```
+
+## Scraping Approaches
+
+### 1. Schema-based Scraping (Faster)
+- Generates CSS selectors for targeted extraction
+- Caches schemas for repeated requests
+- Faster execution for structured websites
+
+### 2. LLM-based Scraping (More Flexible)
+- Direct LLM extraction without schema generation
+- More flexible for complex or dynamic content
+- Better for unstructured data extraction
+
+## Supported LLM Providers
+
+The API supports any LLM provider that crawl4ai supports, including:
+
+- **Google Gemini**: `gemini/gemini-2.5-flash`, `gemini/gemini-pro`
+- **OpenAI**: `openai/gpt-4`, `openai/gpt-3.5-turbo`
+- **Anthropic**: `anthropic/claude-3-opus`, `anthropic/claude-3-sonnet`
+- **And more...**
+
+## API Endpoints
+
+### Core Endpoints
+
+- `POST /scrape` - Schema-based scraping
+- `POST /scrape-with-llm` - LLM-based scraping
+- `GET /schemas` - List cached schemas
+- `POST /clear-cache` - Clear schema cache
+- `GET /health` - Health check
+
+### Model Management Endpoints
+
+- `GET /models` - List saved model configurations
+- `POST /models` - Save a new model configuration
+- `DELETE /models/{model_name}` - Delete a model configuration
+
+### API Request History
+
+- `GET /saved-requests` - List all saved API requests
+- `DELETE /saved-requests/{request_id}` - Delete a saved request
+
+## Request/Response Examples
+
+### Scrape Request
+
+```json
+{
+  "url": "https://example.com",
+  "query": "Extract the product name, price, and description",
+  "model_name": "my-custom-model"
+}
+```
+
+### Scrape Response
+
+```json
+{
+  "success": true,
+  "url": "https://example.com",
+  "query": "Extract the product name, price, and description",
+  "extracted_data": {
+    "product_name": "Example Product",
+    "price": "$99.99",
+    "description": "This is an example product description"
+  },
+  "schema_used": { ... },
+  "timestamp": "2024-01-01T12:00:00Z"
+}
+```
+
+### Model Configuration Request
+
+```json
+{
+  "provider": "gemini/gemini-2.5-flash",
+  "api_token": "your-api-key-here"
+}
+```
+
+## Testing
+
+Run the test script to verify the model management functionality:
+
+```bash
+python test_models.py
+```
+
+## File Structure
+
+```
+parse_example/
+├── api_server.py          # FastAPI server with all endpoints
+├── web_scraper_lib.py     # Core scraping library
+├── test_models.py         # Test script for model management
+├── requirements.txt       # Dependencies
+├── static/               # Frontend files
+│   ├── index.html        # Main HTML interface
+│   ├── styles.css        # CSS styles (minimalist theme)
+│   └── script.js         # JavaScript functionality
+├── schemas/              # Cached schemas
+├── models/               # Saved model configurations
+├── saved_requests/       # API request history
+└── README.md            # This file
+```
+
+## Advanced Usage
+
+### Using the Library Directly
+
+```python
+from web_scraper_lib import WebScraperAgent
+
+# Initialize agent
+agent = WebScraperAgent()
+
+# Save a model configuration
+agent.save_model_config(
+    model_name="my-model",
+    provider="openai/gpt-4",
+    api_token="your-api-key"
+)
+
+# Schema-based scraping
+result = await agent.scrape_data(
+    url="https://example.com",
+    query="Extract product information",
+    model_name="my-model"
+)
+
+# LLM-based scraping
+result = await agent.scrape_data_with_llm(
+    url="https://example.com",
+    query="Extract product information",
+    model_name="my-model"
+)
+```
+
+### Schema Caching
+
+The system automatically caches generated schemas based on URL and query combinations:
+
+- **First request**: Generates schema using AI
+- **Subsequent requests**: Uses cached schema for faster extraction
+
+### API Request History
+
+All API requests are automatically saved with:
+- Request details (URL, query, model used)
+- Response data
+- Timestamp
+- cURL command for re-execution
+
+### Duplicate Prevention
+
+The system prevents saving duplicate requests:
+- Same URL + query combinations are not saved multiple times
+- Returns existing request ID for duplicates
+- Keeps the API request history clean
+
+## Error Handling
+
+The API provides detailed error messages for common issues:
+
+- Invalid URLs
+- Missing model configurations
+- API key errors
+- Network timeouts
+- Parsing errors
diff --git a/docs/examples/website-to-api/api_server.py b/docs/examples/website-to-api/api_server.py
new file mode 100644
index 0000000..0d4982e
--- /dev/null
+++ b/docs/examples/website-to-api/api_server.py
@@ -0,0 +1,363 @@
+from fastapi import FastAPI, HTTPException
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+from pydantic import BaseModel, HttpUrl
+from typing import Dict, Any, Optional, Union, List
+import uvicorn
+import asyncio
+import os
+import json
+from datetime import datetime
+from web_scraper_lib import WebScraperAgent, scrape_website
+
+app = FastAPI(
+    title="Web Scraper API",
+    description="Convert any website into a structured data API. Provide a URL and tell AI what data you need in plain English.",
+    version="1.0.0"
+)
+
+# Mount static files
+if os.path.exists("static"):
+    app.mount("/static", StaticFiles(directory="static"), name="static")
+
+# Mount assets directory
+if os.path.exists("assets"):
+    app.mount("/assets", StaticFiles(directory="assets"), name="assets")
+
+# Initialize the scraper agent
+scraper_agent = WebScraperAgent()
+
+# Create directory for saved API requests
+os.makedirs("saved_requests", exist_ok=True)
+
+class ScrapeRequest(BaseModel):
+    url: HttpUrl
+    query: str
+    model_name: Optional[str] = None
+
+class ModelConfigRequest(BaseModel):
+    model_name: str
+    provider: str
+    api_token: str
+
+class ScrapeResponse(BaseModel):
+    success: bool
+    url: str
+    query: str
+    extracted_data: Union[Dict[str, Any], list]
+    schema_used: Optional[Dict[str, Any]] = None
+    timestamp: Optional[str] = None
+    error: Optional[str] = None
+
+class SavedApiRequest(BaseModel):
+    id: str
+    endpoint: str
+    method: str
+    headers: Dict[str, str]
+    body: Dict[str, Any]
+    timestamp: str
+    response: Optional[Dict[str, Any]] = None
+
+def save_api_request(endpoint: str, method: str, headers: Dict[str, str], body: Dict[str, Any], response: Optional[Dict[str, Any]] = None) -> str:
+    """Save an API request to a JSON file."""
+    
+    # Check for duplicate requests (same URL and query)
+    if endpoint in ["/scrape", "/scrape-with-llm"] and "url" in body and "query" in body:
+        existing_requests = get_saved_requests()
+        for existing_request in existing_requests:
+            if (existing_request.endpoint == endpoint and 
+                existing_request.body.get("url") == body["url"] and 
+                existing_request.body.get("query") == body["query"]):
+                print(f"Duplicate request found for URL: {body['url']} and query: {body['query']}")
+                return existing_request.id  # Return existing request ID instead of creating new one
+    
+    request_id = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]
+    
+    saved_request = SavedApiRequest(
+        id=request_id,
+        endpoint=endpoint,
+        method=method,
+        headers=headers,
+        body=body,
+        timestamp=datetime.now().isoformat(),
+        response=response
+    )
+    
+    file_path = os.path.join("saved_requests", f"{request_id}.json")
+    with open(file_path, "w") as f:
+        json.dump(saved_request.dict(), f, indent=2)
+    
+    return request_id
+
+def get_saved_requests() -> List[SavedApiRequest]:
+    """Get all saved API requests."""
+    requests = []
+    if os.path.exists("saved_requests"):
+        for filename in os.listdir("saved_requests"):
+            if filename.endswith('.json'):
+                file_path = os.path.join("saved_requests", filename)
+                try:
+                    with open(file_path, "r") as f:
+                        data = json.load(f)
+                        requests.append(SavedApiRequest(**data))
+                except Exception as e:
+                    print(f"Error loading saved request {filename}: {e}")
+    
+    # Sort by timestamp (newest first)
+    requests.sort(key=lambda x: x.timestamp, reverse=True)
+    return requests
+
+@app.get("/")
+async def root():
+    """Serve the frontend interface."""
+    if os.path.exists("static/index.html"):
+        return FileResponse("static/index.html")
+    else:
+        return {
+            "message": "Web Scraper API",
+            "description": "Convert any website into structured data with AI",
+            "endpoints": {
+                "/scrape": "POST - Scrape data from a website",
+                "/schemas": "GET - List cached schemas",
+                "/clear-cache": "POST - Clear schema cache",
+                "/models": "GET - List saved model configurations",
+                "/models": "POST - Save a new model configuration",
+                "/models/{model_name}": "DELETE - Delete a model configuration",
+                "/saved-requests": "GET - List saved API requests"
+            }
+        }
+
+@app.post("/scrape", response_model=ScrapeResponse)
+async def scrape_website_endpoint(request: ScrapeRequest):
+    """
+    Scrape structured data from any website.
+    
+    This endpoint:
+    1. Takes a URL and plain English query
+    2. Generates a custom scraper using AI
+    3. Returns structured data
+    """
+    try:
+        # Save the API request
+        headers = {"Content-Type": "application/json"}
+        body = {
+            "url": str(request.url),
+            "query": request.query,
+            "model_name": request.model_name
+        }
+        
+        result = await scraper_agent.scrape_data(
+            url=str(request.url),
+            query=request.query,
+            model_name=request.model_name
+        )
+        
+        response_data = ScrapeResponse(
+            success=True,
+            url=result["url"],
+            query=result["query"],
+            extracted_data=result["extracted_data"],
+            schema_used=result["schema_used"],
+            timestamp=result["timestamp"]
+        )
+        
+        # Save the request with response
+        save_api_request(
+            endpoint="/scrape",
+            method="POST",
+            headers=headers,
+            body=body,
+            response=response_data.dict()
+        )
+        
+        return response_data
+    
+    except Exception as e:
+        # Save the failed request
+        headers = {"Content-Type": "application/json"}
+        body = {
+            "url": str(request.url),
+            "query": request.query,
+            "model_name": request.model_name
+        }
+        
+        save_api_request(
+            endpoint="/scrape",
+            method="POST",
+            headers=headers,
+            body=body,
+            response={"error": str(e)}
+        )
+        
+        raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}")
+
+@app.post("/scrape-with-llm", response_model=ScrapeResponse)
+async def scrape_website_endpoint_with_llm(request: ScrapeRequest):
+    """
+    Scrape structured data from any website using a custom LLM model.
+    """
+    try:
+        # Save the API request
+        headers = {"Content-Type": "application/json"}
+        body = {
+            "url": str(request.url),
+            "query": request.query,
+            "model_name": request.model_name
+        }
+        
+        result = await scraper_agent.scrape_data_with_llm(
+            url=str(request.url),
+            query=request.query,
+            model_name=request.model_name
+        )
+        
+        response_data = ScrapeResponse(
+            success=True,
+            url=result["url"],
+            query=result["query"],
+            extracted_data=result["extracted_data"],
+            timestamp=result["timestamp"]
+        )
+        
+        # Save the request with response
+        save_api_request(
+            endpoint="/scrape-with-llm",
+            method="POST",
+            headers=headers,
+            body=body,
+            response=response_data.dict()
+        )
+        
+        return response_data
+    
+    except Exception as e:
+        # Save the failed request
+        headers = {"Content-Type": "application/json"}
+        body = {
+            "url": str(request.url),
+            "query": request.query,
+            "model_name": request.model_name
+        }
+        
+        save_api_request(
+            endpoint="/scrape-with-llm",
+            method="POST",
+            headers=headers,
+            body=body,
+            response={"error": str(e)}
+        )
+        
+        raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}")
+
+@app.get("/saved-requests")
+async def list_saved_requests():
+    """List all saved API requests."""
+    try:
+        requests = get_saved_requests()
+        return {
+            "success": True,
+            "requests": [req.dict() for req in requests],
+            "count": len(requests)
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to list saved requests: {str(e)}")
+
+@app.delete("/saved-requests/{request_id}")
+async def delete_saved_request(request_id: str):
+    """Delete a saved API request."""
+    try:
+        file_path = os.path.join("saved_requests", f"{request_id}.json")
+        if os.path.exists(file_path):
+            os.remove(file_path)
+            return {
+                "success": True,
+                "message": f"Saved request '{request_id}' deleted successfully"
+            }
+        else:
+            raise HTTPException(status_code=404, detail=f"Saved request '{request_id}' not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to delete saved request: {str(e)}")
+
+@app.get("/schemas")
+async def list_cached_schemas():
+    """List all cached schemas."""
+    try:
+        schemas = await scraper_agent.get_cached_schemas()
+        return {
+            "success": True,
+            "cached_schemas": schemas,
+            "count": len(schemas)
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to list schemas: {str(e)}")
+
+@app.post("/clear-cache")
+async def clear_schema_cache():
+    """Clear all cached schemas."""
+    try:
+        scraper_agent.clear_cache()
+        return {
+            "success": True,
+            "message": "Schema cache cleared successfully"
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to clear cache: {str(e)}")
+
+@app.get("/models")
+async def list_models():
+    """List all saved model configurations."""
+    try:
+        models = scraper_agent.list_saved_models()
+        return {
+            "success": True,
+            "models": models,
+            "count": len(models)
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to list models: {str(e)}")
+
+@app.post("/models")
+async def save_model_config(request: ModelConfigRequest):
+    """Save a new model configuration."""
+    try:
+        success = scraper_agent.save_model_config(
+            model_name=request.model_name,
+            provider=request.provider,
+            api_token=request.api_token
+        )
+        
+        if success:
+            return {
+                "success": True,
+                "message": f"Model configuration '{request.model_name}' saved successfully"
+            }
+        else:
+            raise HTTPException(status_code=500, detail="Failed to save model configuration")
+    
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to save model: {str(e)}")
+
+@app.delete("/models/{model_name}")
+async def delete_model_config(model_name: str):
+    """Delete a model configuration."""
+    try:
+        success = scraper_agent.delete_model_config(model_name)
+        
+        if success:
+            return {
+                "success": True,
+                "message": f"Model configuration '{model_name}' deleted successfully"
+            }
+        else:
+            raise HTTPException(status_code=404, detail=f"Model configuration '{model_name}' not found")
+    
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to delete model: {str(e)}")
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint."""
+    return {"status": "healthy", "service": "web-scraper-api"}
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)
\ No newline at end of file
diff --git a/docs/examples/website-to-api/app.py b/docs/examples/website-to-api/app.py
new file mode 100644
index 0000000..4571050
--- /dev/null
+++ b/docs/examples/website-to-api/app.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""
+Startup script for the Web Scraper API with frontend interface.
+"""
+
+import os
+import sys
+import uvicorn
+from pathlib import Path
+
+def main():
+    # Check if static directory exists
+    static_dir = Path("static")
+    if not static_dir.exists():
+        print("❌ Static directory not found!")
+        print("Please make sure the 'static' directory exists with the frontend files.")
+        sys.exit(1)
+    
+    # Check if required frontend files exist
+    required_files = ["index.html", "styles.css", "script.js"]
+    missing_files = []
+    
+    for file in required_files:
+        if not (static_dir / file).exists():
+            missing_files.append(file)
+    
+    if missing_files:
+        print(f"❌ Missing frontend files: {', '.join(missing_files)}")
+        print("Please make sure all frontend files are present in the static directory.")
+        sys.exit(1)
+    
+    print("🚀 Starting Web Scraper API with Frontend Interface")
+    print("=" * 50)
+    print("📁 Static files found and ready to serve")
+    print("🌐 Frontend will be available at: http://localhost:8000")
+    print("🔌 API endpoints available at: http://localhost:8000/docs")
+    print("=" * 50)
+    
+    # Start the server
+    uvicorn.run(
+        "api_server:app",
+        host="0.0.0.0",
+        port=8000,
+        reload=True,
+        log_level="info"
+    )
+
+if __name__ == "__main__":
+    main() 
\ No newline at end of file
diff --git a/docs/examples/website-to-api/assets/crawl4ai_logo.jpg b/docs/examples/website-to-api/assets/crawl4ai_logo.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6a808c043126f1c691e7bbf81c766a4980b734cf
GIT binary patch
literal 5920
zcmb7IWmHvL*WM@SI&c8#JakBRN`rJsDi}1dl~q97#=0uqA4
zx4rki_xtmWcZ_eI^<$5*=UijW{XFZLbDrzD>#qPpQ&mG1fIt8M0{(#O1waXaL7{&Z
zXmD_0;bLLI;aK=MIM}#^_=JQ6_yhz*#AFB}Vp3uP0t6+36iH4&K|x4zgNl-zij16s
z{Ldy37#IV`!o$MCBPSvtBL9Eebq_#_3k(1QFbE3(C56C9A=iBX9e7CCUkmt;VdLOp
zL7;FL_?8yT0}A_-%)bW5g2J$IAlIJ&LNE@BfFZz4heG~+|9?Js`S(kee5%JZUwOyg
zw_s)M-jQ4por5)<5q~-4+_v~R?U`0dv(i$d2Ifvcyg2?iyTUdM3lJzfEO)!X0kL)7P$B^{6oe8yI20*
zo!zCpE-eOOB5Snq{EggWvb_h|CzgZ01Xev;S6M}Tr9u;kHzDA8X}%>)n#frBr=zX`
zUZ>$lC+k0|Bp4qWJFZtqPSJ|9%=k(%c4c~Zl+)d+-_acVDi{w2C8~N^Zx8E=d2Mr1
zFse31e+t}A;%ygL=IDPlhOXq#^i;S8G~R#6Tzp#SE%_&^hpeQ55G(lA+|_
z5BRG9ssn&PYGrUsB*RpcW<8<;bog&355mZf0H9DP$e@3rgk!<5Ay9yvLI4Lr%0x!#
zh7<%5#RZXuU_r0M+F7@~j2aEq_((-E0R91u
zSN=yg%h+nG>n1l;;2ti7$y)3+&(~L~rXBX8gN{;7_h3g`_It2vV9}Jz^KHmQg5$*j
zg>Cm6*kPJ3<+SA&$g4XerG2BzJ|z4nu1hyLn;xT`SkEV*`a)DHCT;D-k)QYsKxZyn&MXb~){mex_fQ8E!(ZsGNw_{H$;%Ck@{Zi#yL+m~t
z(-Y++E_KEEOW|ze^c>-Hyrv_SL&x)aIb3?RlpAKdYv((g>p_0`JslRw=#GRxAHoH0
z53KKFer_ZMs#!7eCBMCkg9=n(6B?h7R8(xGhQR-;<%BfqCJ4fkF{CUmLwMv22z?5p=fmKe~CX<~0KmY)V_
z4O^k@)_!kOE)Jp>$=pX%I_G^4$3YGWk^&GI6oQ9?^&dHaKw)qIi@
zQGAw%`Ob>cbFb@O{)E00L@Hw#>9uMg;>j-{K5w3HRdze&*eH}ltBzWCu!b1}iVRyZYTpElzz7n~Xx
z8(-P0_D0rm)OF}GRdt*sz-OJlMOb2`TFGoC;fTPAmWWx$?BN3*N0Q^kYJo&Id#TwQ
zFU()?xmgb}*jD>KsOp&V?DD?`xMv>=-$?0%>`YcC_K6*9yEAGwEF3GT`@Y;+Lgn?a
zGTGK-a>Ll+mJFtMr?rEH3c30SAtkHrbRpYr{gsmo-thx-g}?f%Z}X$xGFz=B#{DX0
zR8Hxw+L!vc8F*yt;Vl3>UnI_i&
z{>8-IwZe}&!rn2@jyD}y*vuEX3^bgNu67prCF=@Ae9ot<7avI`RYQ*P0n|Nd{*>Q*
z)b^_kZ@I~vx&t}=qa3E>LRN@-tDW_e+__ClnuLa^6kO_R9W@%&627_Lc|Mufql`*w
z*1$opTmvAQY^0zffPw&u`#<6VssW4?<*LWMW3>kt^!k{DmO&4}urvmp=UN{1wf?
z?nl_ccvkoYUTsNPoJX3YSt@)=ftfC54<9u=KU9&GjuY1R5#uGVqrY3jq?z?nVTxT%
zD3eHgav@hYJCCPTmjR{xY1@6(~#JU81nT}Zufi*keR
zoh{O&yzZkiCHfCjonL)8kh-Qit0PkuF|T-PXXZ0q?j9kgWNM3sk|v={*gy)dFZdqu<0^K7gxQT
zRx@T~nO0QRRGR14fEahp1gK^*<_`!}m&d2~J>DbC1Ijqemh
z%XFCMTqbQVnok-{H%{Y}88v+xUG`OFq%00%)7_ul9fen12#C$UEkm8Tq&{o(77!gn
zkB27|)1M0+7^Xb8((ITY`xF-u)OZumxR~CJ`5JJ+R`S%DQ(<^CsVkhCJi6PzU|NNG
za@g>_6ay8`
zwHRy%MR*y+Z&VP-7;yT1wlq3u?nvPI9+3S|2lHL5^&TJF?Y#<`r*f15=`CN&+Eh(6
z?NvEGxeN(6dQA|#_C4T`W*{kcb$yX!vmhBP_5lu^<(E?%
z4qPQxX2_q1wdrC75*mztX6)f%@dG!pKigt`^FY_GPx9h5Q1xp_=MTcUStA79C|ACV
z3%~rmA&n9!^QZe<&dYj3}0AAN|bAaf@)$?-$ZCNX`UmtUj~c;L?i#@QI}jm>5gpq%xm7O7mDr-w}$$oQW~(S|t~rsxml?h#(S
z!Qyk2!Iac)&0w3@TbE1>i4{1Ihn5ssla*UNi{|wG^@(t_iN}U6c-4U;y=mRkU#`$w
zfHvFLm6eC>HV15v|3uPWbZU6ia<6okkw0^CL^4UuX+w4aOW%zO_#0LNQRPW%
zrb9>H9;2(>gud=o_kh|ZyC@YB+E?j1mzFmj9Ry>pf$Dgtdo8Qp1tp;+Ca4>bP4v*x
zEBQn5?VtrOK5lf(d*{>=-nI`kuS2LSA-+irKtQ~;L!ifNd
z6blN<%D?3iAY~TRSF}M0xc?E6?@V;9Mw<=j^4!{F&XAOiamNQ__RNkwB$;8@sIcNjH3Ufgkch!)9szL2gS_2Q{cjqVfj*}WZfv3fo_A{J?P
zG&5CVlCSJ(JjFMM-JgMi>%!-J+wW)Ri)d$C$Yx8paQ5Hs6gaWL#5-`dUcpXg!$bBE
z3@|&wk8wVXa>AnuL5)S-;(b6pc7;QtLkHahc{WP*Y|J~Z$@w3r%QXtJRnoFK=PW?R
z91~2fGty~Q1|qt%o08*>Uu0HY8$yBo${)$q!GLY)@ecfy<~^a>x5b$z_TBRRIm>@*
z?6VupID1clVk?kKRw#tt^^J!n_Iml8)1vAS;tYOVM!8rtI6}E^<^h_T+1qV*x0x0a)
zEb(=|xnEx7n{Yg0HJHpck2KpeHEOYK^EMyfJpHkuvwr`})mA{#Z9kw!V=n0z9ZiN{
zllGg)8YTGF4>TtUmd=Y(D!HK7e3xBm|$aPFb5Wz$e7kFkj4;Se2pIQ`!Y
zV_7K6VOz`QVA*A1rk|XxJ-26Houwbs7Go2l+xmK(+2pJJJ$^xz7#?!n%QCA-qk5P1
z!u*}~YZH{Yb@kSCP4sukTNt?w1{-SjCesiq;d=c?Jbp~Sh{sDJ-rN!Uv3u&Z{2(pG
zGc=Ki2Pdd9qGL!SA!y-4ePI3E$8kKz&yP!;@EN?X0lc5urbf;8wG3kS+zPUh1;5_P
zwtW=k7Mly&n}I?K$)G$9>(lRCd&{yv-oCn0<#pS(X~VWkc}+4x8UKDIx?d8lT^34+Es~xKSBkpW=>UC^-9mJr8Bk)D^8d&WDE9_q*CGe**BaX*q
z{@dD2J07RL20*8=gZqnI9C4tx-0V@FDs2!Lc9f*WRo(d~{;LCW>NM`pUOZR_T(oz=
z2U*c2nWOQJ|F%2m58i2ZQt+mNJuVL9-#bmptf&tN+PE|Qxzd>LbqKwG`&`5|kjNvF
zfGSwqxIH0mgq#gR)1(W&dCxt>f_|Pw;&ZLw*&&q{
zQx7!;nB@Z<);hoSEqe~FmQ*ToHuzoiE|fND*Hi5_Gs-d~(;?(P2DwZ^Xnv$_E*K%CqMjY1jg*+3qjF=SrD8Iw*P|}T_f`9kFV-aw^oBAY$PIgLltseP4QQmH4ug65bfLC9+0oLwH#Qw1mQzlJ
zLr7ek6|y_RJ^}yP02Nq~N#DjD97_;-&tCs$Ecw&aB2HfyFIURA
zu`Z;q2NVcXD?xX7Hi`NJwb`80L`N2N9@tGh)@)kY5{{5MO|!BX!$kC@(=ooyuZ;22
zy1$&9$s4yzuejcau`+W~Y}QPYvm{U>b|0-`#@{b^|MI6Ho4V;G-A1O#1RSTk$FSyx
zhWUv19yeW6)==v6)kqj14K|Fsn27kTk)xH0$HhPBC5981qX>}B&}To+UZQWhh;w2WPK;s0jI!8UUt-5CHRHa?8W~)1ZQxfu$v(kB!ys<(Ul?m0DO9
zDw!#TbQPf&?M+f|8Dp&EHEt6_^5E{?M?k@XF!Bs)dP_);dhr|H4?3;3vQbFFJ?Jzn
z0Y8nn*oEn4>y-N>k7!RPnOi}GnS=+IQh9&(B<^fWl2BC`Yu#1#fr>iv0|%`h-}+}y
zfsH-6m$<-+&ubvk*d7*?ku`H~Aa#I0+2-Dvs@nh{EJ>D4Z*b&*XmqK-Jk-&qBh92V
z2Aq}EudoP|xPOUN0X58o%8tdZQ$!2U3zpVc#KRay-&~SbyiO^m=n^4TW-@k(hq%n<
z;fUhgbvSB;&@HM6n-nLR=1ei<$-7OP)SF6D*zXKL$Y@DRtb(KxzAB9INg#RygQVeL|VX*Wly(A!iPMY@IRQ&s-K
zxU?K#@jZ08esraSTIVKPfy+5LL;<44PQk3x`+F?m8y4*J>{mQ2Ho?m)8|`czc7yEF
z4<1-J>Q)n$#T*HFPr#vJH!x7baXTInemK{lp0OSlFK$vX6_jQzI8&T?I4Q;==HaGs
zuT+0EVV{?BR00-Xicisf4ItcLfc<(!M0l&TCP@m!uYWS0_3ksFcH<>pqCX(9$COeY
zt5cFS_@IptkP$q3;o93}UpXMbb9iqaV6(YAZ=*I|#T^T0a*2^5A0gp)ycNp7g*%zd
zb%K0{Ow{R#p<^!7#p+{6W%o2_rpI%78a|RvbU}on0}`z1aBTL9&cyWLWy%l*mpdST
zQWu-aT1v$FYk>&x9J2@IQ_ipt5ve2@C9tYhov4@#-1jER+$K$0Y@@03$aq7oo3#>adFU
zUmQ?AIn^`MyNp9^8p=ybONOPzYrgG9g|Fq0*(C@QBDqOr+y@%vtcwnE)O}twfAX^?
zdhV?4ETEbN5_R*&y4S(Rn|G0O>+a}y~
+
+
+    
+    
+    Web2API Example
+    
+    
+
+
+    
+    

+        

+            
+            
+        

+    

+
+    
+    

+        
+        

+            

+                
Turn Any Website Into An API

+                
This example shows how to turn any website into an API using Crawl4AI.

+            

+
+            
+            

+                

+                    
1. Your Request

+                    

+                        

+                            
+                            
+                        

+                        

+                            
+                            
+                        

+                        

+                            

+                                
+                                
+                            

+                            

+                                
+                                
+                            

+                        

+                        
+                    

+                

+
+                
→

+
+                

+                    
2. Your Instant API & Data

+                    

+                        

+                            
+                            
curl -X POST http://localhost:8000/scrape -H "Content-Type: application/json" -d '{"url": "...", "query": "..."}'
+
+# Or for LLM-based approach:
+curl -X POST http://localhost:8000/scrape-with-llm -H "Content-Type: application/json" -d '{"url": "...", "query": "..."}'

+                        

+                        

+                            
+                            
{
+  "success": true,
+  "extracted_data": [
+    {
+      "title": "Example Book",
+      "author": "John Doe",
+      "description": "A great book..."
+    }
+  ]
+}

+                        

+                    

+                

+            

+
+            
+            
+
+            
+            
+        

+
+        
+        

+            

+                
Model Configuration

+                
Configure and manage your AI model configurations

+            

+
+            

+                
+                

+                    
Add New Model

+                    

+                        

+                            

+                                
+                                
+                            

+                            

+                                
+                                
+                            

+                        

+                        
+                        

+                            
+                            
+                        

+
+                        
+                    

+                

+
+                
+                

+                    
Saved Models

+                    

+                        
+                    

+                

+            

+        

+
+        
+        

+            

+                
Saved API Requests

+                
View and manage your previous API requests

+            

+
+            

+                

+                    
+                

+            

+        

+    

+
+    
+    

+
+    
+
+ 
\ No newline at end of file
diff --git a/docs/examples/website-to-api/static/script.js b/docs/examples/website-to-api/static/script.js
new file mode 100644
index 0000000..921598a
--- /dev/null
+++ b/docs/examples/website-to-api/static/script.js
@@ -0,0 +1,401 @@
+// API Configuration
+const API_BASE_URL = 'http://localhost:8000';
+
+// DOM Elements
+const navLinks = document.querySelectorAll('.nav-link');
+const pages = document.querySelectorAll('.page');
+const scrapeForm = document.getElementById('scrape-form');
+const modelForm = document.getElementById('model-form');
+const modelSelect = document.getElementById('model-select');
+const modelsList = document.getElementById('models-list');
+const resultsSection = document.getElementById('results-section');
+const loadingSection = document.getElementById('loading');
+const copyJsonBtn = document.getElementById('copy-json');
+
+// Navigation
+navLinks.forEach(link => {
+    link.addEventListener('click', (e) => {
+        e.preventDefault();
+        const targetPage = link.dataset.page;
+        
+        // Update active nav link
+        navLinks.forEach(l => l.classList.remove('active'));
+        link.classList.add('active');
+        
+        // Show target page
+        pages.forEach(page => page.classList.remove('active'));
+        document.getElementById(`${targetPage}-page`).classList.add('active');
+        
+        // Load data for the page
+        if (targetPage === 'models') {
+            loadModels();
+        } else if (targetPage === 'requests') {
+            loadSavedRequests();
+        }
+    });
+});
+
+// Scrape Form Handler
+document.getElementById('extract-btn').addEventListener('click', async (e) => {
+    e.preventDefault();
+    
+    // Scroll to results section immediately when button is clicked
+    document.getElementById('results-section').scrollIntoView({ 
+        behavior: 'smooth', 
+        block: 'start' 
+    });
+    
+    const url = document.getElementById('url').value;
+    const query = document.getElementById('query').value;
+    const headless = true; // Always use headless mode
+    const model_name = document.getElementById('model-select').value || null;
+    const scraping_approach = document.getElementById('scraping-approach').value;
+    
+    if (!url || !query) {
+        showToast('Please fill in both URL and query fields', 'error');
+        return;
+    }
+    
+    if (!model_name) {
+        showToast('Please select a model from the dropdown or add one from the Models page', 'error');
+        return;
+    }
+    
+    const data = {
+        url: url,
+        query: query,
+        headless: headless,
+        model_name: model_name
+    };
+    
+    // Show loading state
+    showLoading(true);
+    hideResults();
+    
+    try {
+        // Choose endpoint based on scraping approach
+        const endpoint = scraping_approach === 'llm' ? '/scrape-with-llm' : '/scrape';
+        
+        const response = await fetch(`${API_BASE_URL}${endpoint}`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify(data)
+        });
+        
+        const result = await response.json();
+        
+        if (response.ok) {
+            displayResults(result);
+            showToast(`Data extracted successfully using ${scraping_approach === 'llm' ? 'LLM-based' : 'Schema-based'} approach!`, 'success');
+        } else {
+            throw new Error(result.detail || 'Failed to extract data');
+        }
+    } catch (error) {
+        console.error('Scraping error:', error);
+        showToast(`Error: ${error.message}`, 'error');
+    } finally {
+        showLoading(false);
+    }
+});
+
+// Model Form Handler
+modelForm.addEventListener('submit', async (e) => {
+    e.preventDefault();
+    
+    const formData = new FormData(modelForm);
+    const data = {
+        model_name: formData.get('model_name'),
+        provider: formData.get('provider'),
+        api_token: formData.get('api_token')
+    };
+    
+    try {
+        const response = await fetch(`${API_BASE_URL}/models`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify(data)
+        });
+        
+        const result = await response.json();
+        
+        if (response.ok) {
+            showToast('Model saved successfully!', 'success');
+            modelForm.reset();
+            loadModels();
+            loadModelSelect();
+        } else {
+            throw new Error(result.detail || 'Failed to save model');
+        }
+    } catch (error) {
+        console.error('Model save error:', error);
+        showToast(`Error: ${error.message}`, 'error');
+    }
+});
+
+// Copy JSON Button
+copyJsonBtn.addEventListener('click', () => {
+    const actualJsonOutput = document.getElementById('actual-json-output');
+    const textToCopy = actualJsonOutput.textContent;
+    
+    navigator.clipboard.writeText(textToCopy).then(() => {
+        showToast('JSON copied to clipboard!', 'success');
+    }).catch(() => {
+        showToast('Failed to copy JSON', 'error');
+    });
+});
+
+// Load Models
+async function loadModels() {
+    try {
+        const response = await fetch(`${API_BASE_URL}/models`);
+        const result = await response.json();
+        
+        if (response.ok) {
+            displayModels(result.models);
+        } else {
+            throw new Error(result.detail || 'Failed to load models');
+        }
+    } catch (error) {
+        console.error('Load models error:', error);
+        showToast(`Error: ${error.message}`, 'error');
+    }
+}
+
+// Display Models
+function displayModels(models) {
+    if (models.length === 0) {
+        modelsList.innerHTML = '
No models saved yet. Add your first model above!
';
+        return;
+    }
+    
+    modelsList.innerHTML = models.map(model => `
+        

+            

+                
${model}

+                
Model Configuration

+            

+            

+                
+            

+        

+    `).join('');
+}
+
+// Delete Model
+async function deleteModel(modelName) {
+    if (!confirm(`Are you sure you want to delete the model "${modelName}"?`)) {
+        return;
+    }
+    
+    try {
+        const response = await fetch(`${API_BASE_URL}/models/${modelName}`, {
+            method: 'DELETE'
+        });
+        
+        const result = await response.json();
+        
+        if (response.ok) {
+            showToast('Model deleted successfully!', 'success');
+            loadModels();
+            loadModelSelect();
+        } else {
+            throw new Error(result.detail || 'Failed to delete model');
+        }
+    } catch (error) {
+        console.error('Delete model error:', error);
+        showToast(`Error: ${error.message}`, 'error');
+    }
+}
+
+// Load Model Select Options
+async function loadModelSelect() {
+    try {
+        const response = await fetch(`${API_BASE_URL}/models`);
+        const result = await response.json();
+        
+        if (response.ok) {
+            // Clear existing options
+            modelSelect.innerHTML = '';
+            
+            // Add model options
+            result.models.forEach(model => {
+                const option = document.createElement('option');
+                option.value = model;
+                option.textContent = model;
+                modelSelect.appendChild(option);
+            });
+        }
+    } catch (error) {
+        console.error('Load model select error:', error);
+    }
+}
+
+// Display Results
+function displayResults(result) {
+    // Update result info
+    document.getElementById('result-url').textContent = result.url;
+    document.getElementById('result-query').textContent = result.query;
+    document.getElementById('result-model').textContent = result.model_name || 'Default Model';
+    
+    // Display JSON in the actual results section
+    const actualJsonOutput = document.getElementById('actual-json-output');
+    actualJsonOutput.textContent = JSON.stringify(result.extracted_data, null, 2);
+    
+    // Don't update the sample JSON in the workflow demo - keep it as example
+    
+    // Update the cURL example based on the approach used
+    const scraping_approach = document.getElementById('scraping-approach').value;
+    const endpoint = scraping_approach === 'llm' ? '/scrape-with-llm' : '/scrape';
+    const curlExample = document.getElementById('curl-example');
+    curlExample.textContent = `curl -X POST http://localhost:8000${endpoint} -H "Content-Type: application/json" -d '{"url": "${result.url}", "query": "${result.query}"}'`;
+    
+    // Show results section
+    resultsSection.style.display = 'block';
+    resultsSection.scrollIntoView({ behavior: 'smooth' });
+}
+
+// Show/Hide Loading
+function showLoading(show) {
+    loadingSection.style.display = show ? 'block' : 'none';
+}
+
+// Hide Results
+function hideResults() {
+    resultsSection.style.display = 'none';
+}
+
+// Toast Notifications
+function showToast(message, type = 'info') {
+    const toastContainer = document.getElementById('toast-container');
+    const toast = document.createElement('div');
+    toast.className = `toast ${type}`;
+    
+    const icon = type === 'success' ? 'fas fa-check-circle' : 
+                 type === 'error' ? 'fas fa-exclamation-circle' : 
+                 'fas fa-info-circle';
+    
+    toast.innerHTML = `
+        
+        ${message}
+    `;
+    
+    toastContainer.appendChild(toast);
+    
+    // Auto remove after 5 seconds
+    setTimeout(() => {
+        toast.remove();
+    }, 5000);
+}
+
+// Load Saved Requests
+async function loadSavedRequests() {
+    try {
+        const response = await fetch(`${API_BASE_URL}/saved-requests`);
+        const result = await response.json();
+        
+        if (response.ok) {
+            displaySavedRequests(result.requests);
+        } else {
+            throw new Error(result.detail || 'Failed to load saved requests');
+        }
+    } catch (error) {
+        console.error('Load saved requests error:', error);
+        showToast(`Error: ${error.message}`, 'error');
+    }
+}
+
+// Display Saved Requests
+function displaySavedRequests(requests) {
+    const requestsList = document.getElementById('requests-list');
+    
+    if (requests.length === 0) {
+        requestsList.innerHTML = '
No saved API requests yet. Make your first request from the Scrape page!
';
+        return;
+    }
+    
+    requestsList.innerHTML = requests.map(request => {
+        const url = request.body.url;
+        const query = request.body.query;
+        const model = request.body.model_name || 'Default Model';
+        const endpoint = request.endpoint;
+        
+        // Create curl command
+        const curlCommand = `curl -X POST http://localhost:8000${endpoint} \\
+  -H "Content-Type: application/json" \\
+  -d '{
+    "url": "${url}",
+    "query": "${query}",
+    "model_name": "${model}"
+  }'`;
+        
+        return `
+            

+                

+                    

+                        
${url}

+                        
${query}

+                    

+                    

+                        
+                    

+                

+                
+                

+                    
cURL Command:

+                    
${curlCommand}

+                

+            

+        `;
+    }).join('');
+}
+
+// Delete Saved Request
+async function deleteSavedRequest(requestId) {
+    if (!confirm('Are you sure you want to delete this saved request?')) {
+        return;
+    }
+    
+    try {
+        const response = await fetch(`${API_BASE_URL}/saved-requests/${requestId}`, {
+            method: 'DELETE'
+        });
+        
+        const result = await response.json();
+        
+        if (response.ok) {
+            showToast('Saved request deleted successfully!', 'success');
+            loadSavedRequests();
+        } else {
+            throw new Error(result.detail || 'Failed to delete saved request');
+        }
+    } catch (error) {
+        console.error('Delete saved request error:', error);
+        showToast(`Error: ${error.message}`, 'error');
+    }
+}
+
+// Initialize
+document.addEventListener('DOMContentLoaded', () => {
+    loadModelSelect();
+    
+    // Check if API is available
+    fetch(`${API_BASE_URL}/health`)
+        .then(response => {
+            if (!response.ok) {
+                showToast('Warning: API server might not be running', 'error');
+            }
+        })
+        .catch(() => {
+            showToast('Warning: Cannot connect to API server. Make sure it\'s running on localhost:8000', 'error');
+        });
+}); 
\ No newline at end of file
diff --git a/docs/examples/website-to-api/static/styles.css b/docs/examples/website-to-api/static/styles.css
new file mode 100644
index 0000000..66d3976
--- /dev/null
+++ b/docs/examples/website-to-api/static/styles.css
@@ -0,0 +1,765 @@
+/* Reset and Base Styles */
+* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+    background: #000000;
+    color: #FFFFFF;
+    line-height: 1.6;
+    font-size: 16px;
+}
+
+/* Header */
+.header {
+    border-bottom: 1px solid #333;
+    padding: 1rem 0;
+    background: #000000;
+    position: sticky;
+    top: 0;
+    z-index: 100;
+}
+
+.header-content {
+    max-width: 1200px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.logo {
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    font-size: 1.5rem;
+    font-weight: 600;
+    color: #FFFFFF;
+}
+
+.logo-image {
+    width: 40px;
+    height: 40px;
+    border-radius: 4px;
+    object-fit: contain;
+}
+
+.nav-links {
+    display: flex;
+    gap: 2rem;
+}
+
+.nav-link {
+    color: #CCCCCC;
+    text-decoration: none;
+    font-weight: 500;
+    transition: color 0.2s ease;
+}
+
+.nav-link:hover,
+.nav-link.active {
+    color: #FFFFFF;
+}
+
+/* Main Content */
+.main-content {
+    max-width: 1200px;
+    margin: 0 auto;
+    padding: 2rem;
+}
+
+.page {
+    display: none;
+}
+
+.page.active {
+    display: block;
+}
+
+/* Hero Section */
+.hero-section {
+    text-align: center;
+    margin-bottom: 4rem;
+    padding: 2rem 0;
+}
+
+.hero-title {
+    font-size: 3rem;
+    font-weight: 700;
+    color: #FFFFFF;
+    margin-bottom: 1rem;
+    line-height: 1.2;
+}
+
+.hero-subtitle {
+    font-size: 1.25rem;
+    color: #CCCCCC;
+    max-width: 600px;
+    margin: 0 auto;
+}
+
+/* Workflow Demo */
+.workflow-demo {
+    display: grid;
+    grid-template-columns: 1fr auto 1fr;
+    gap: 2rem;
+    align-items: start;
+    margin-bottom: 4rem;
+}
+
+.workflow-step {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+}
+
+.step-title {
+    font-size: 1.25rem;
+    font-weight: 600;
+    color: #FFFFFF;
+    text-align: center;
+    margin-bottom: 1rem;
+}
+
+.workflow-arrow {
+    font-size: 2rem;
+    font-weight: 700;
+    color: #09b5a5;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    margin-top: 20rem;
+}
+
+/* Request Box */
+.request-box {
+    border: 2px solid #333;
+    border-radius: 8px;
+    padding: 2rem;
+    background: #111111;
+}
+
+.input-group {
+    margin-bottom: 1.5rem;
+}
+
+.input-group label {
+    display: block;
+    font-family: 'Courier New', monospace;
+    font-weight: 600;
+    color: #FFFFFF;
+    margin-bottom: 0.5rem;
+    font-size: 0.9rem;
+}
+
+.input-group input,
+.input-group textarea,
+.input-group select {
+    width: 100%;
+    padding: 0.75rem;
+    border: 1px solid #333;
+    border-radius: 4px;
+    font-family: 'Courier New', monospace;
+    font-size: 0.9rem;
+    background: #1A1A1A;
+    color: #FFFFFF;
+    transition: border-color 0.2s ease;
+}
+
+.input-group input:focus,
+.input-group textarea:focus,
+.input-group select:focus {
+    outline: none;
+    border-color: #09b5a5;
+}
+
+.input-group textarea {
+    min-height: 80px;
+    resize: vertical;
+}
+
+.form-options {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 1rem;
+    margin-bottom: 1.5rem;
+}
+
+.option-group {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.option-group label {
+    font-family: 'Courier New', monospace;
+    font-weight: 600;
+    color: #FFFFFF;
+    font-size: 0.9rem;
+}
+
+.option-group input[type="checkbox"] {
+    width: auto;
+    margin-right: 0.5rem;
+}
+
+.extract-btn {
+    width: 100%;
+    padding: 1rem;
+    background: #09b5a5;
+    color: #000000;
+    border: none;
+    border-radius: 4px;
+    font-size: 1rem;
+    font-weight: 600;
+    cursor: pointer;
+    transition: background-color 0.2s ease;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    gap: 0.5rem;
+}
+
+.extract-btn:hover {
+    background: #09b5a5;
+}
+
+/* Dropdown specific styling */
+select,
+.input-group select,
+.option-group select {
+    cursor: pointer !important;
+    appearance: none !important;
+    -webkit-appearance: none !important;
+    -moz-appearance: none !important;
+    -ms-appearance: none !important;
+    background-image: url("data:image/svg+xml;charset=UTF-8,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%23FFFFFF' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3e%3cpolyline points='6,9 12,15 18,9'%3e%3c/polyline%3e%3c/svg%3e") !important;
+    background-repeat: no-repeat !important;
+    background-position: right 0.75rem center !important;
+    background-size: 1rem !important;
+    padding-right: 2.5rem !important;
+    border: 1px solid #333 !important;
+    border-radius: 4px !important;
+    font-family: 'Courier New', monospace !important;
+    font-size: 0.9rem !important;
+    background-color: #1A1A1A !important;
+    color: #FFFFFF !important;
+}
+
+select:hover,
+.input-group select:hover,
+.option-group select:hover {
+    border-color: #09b5a5 !important;
+}
+
+select:focus,
+.input-group select:focus,
+.option-group select:focus {
+    outline: none !important;
+    border-color: #09b5a5 !important;
+}
+
+select option,
+.input-group select option,
+.option-group select option {
+    background: #1A1A1A !important;
+    color: #FFFFFF !important;
+    padding: 0.5rem !important;
+}
+
+/* Response Container */
+.response-container {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+}
+
+.api-request-box,
+.json-response-box {
+    border: 2px solid #333;
+    border-radius: 8px;
+    padding: 1.5rem;
+    background: #111111;
+}
+
+.api-request-box label,
+.json-response-box label {
+    display: block;
+    font-family: 'Courier New', monospace;
+    font-weight: 600;
+    color: #FFFFFF;
+    margin-bottom: 0.5rem;
+    font-size: 0.9rem;
+}
+
+.api-request-box pre,
+.json-response-box pre {
+    font-family: 'Courier New', monospace;
+    font-size: 0.85rem;
+    line-height: 1.5;
+    color: #FFFFFF;
+    background: #1A1A1A;
+    padding: 1rem;
+    border-radius: 4px;
+    overflow-x: auto;
+    white-space: pre-wrap;
+    word-break: break-all;
+}
+
+/* Results Section */
+.results-section {
+    border: 2px solid #333;
+    border-radius: 8px;
+    overflow: hidden;
+    margin-top: 2rem;
+    background: #111111;
+}
+
+.results-header {
+    background: #1A1A1A;
+    color: #FFFFFF;
+    padding: 1rem 1.5rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    border-bottom: 1px solid #333;
+}
+
+.results-header h2 {
+    font-size: 1.25rem;
+    font-weight: 600;
+    color: #FFFFFF;
+}
+
+.copy-btn {
+    background: #09b5a5;
+    color: #000000;
+    border: none;
+    padding: 0.5rem 1rem;
+    border-radius: 4px;
+    font-size: 0.9rem;
+    font-weight: 600;
+    cursor: pointer;
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    transition: background-color 0.2s ease;
+}
+
+.copy-btn:hover {
+    background: #09b5a5;
+}
+
+.results-content {
+    padding: 1.5rem;
+}
+
+.result-info {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin-bottom: 1.5rem;
+    padding: 1rem;
+    background: #1A1A1A;
+    border-radius: 4px;
+    border: 1px solid #333;
+}
+
+.info-item {
+    display: flex;
+    flex-direction: column;
+    gap: 0.25rem;
+}
+
+.info-item .label {
+    font-weight: 600;
+    color: #FFFFFF;
+    font-size: 0.9rem;
+}
+
+.info-item .value {
+    color: #CCCCCC;
+    word-break: break-all;
+}
+
+.json-display {
+    background: #1A1A1A;
+    border-radius: 4px;
+    overflow: hidden;
+    border: 1px solid #333;
+}
+
+.json-display pre {
+    color: #FFFFFF;
+    padding: 1.5rem;
+    margin: 0;
+    overflow-x: auto;
+    font-family: 'Courier New', monospace;
+    font-size: 0.9rem;
+    line-height: 1.5;
+}
+
+/* Loading State */
+.loading {
+    text-align: center;
+    padding: 3rem;
+}
+
+.spinner {
+    width: 40px;
+    height: 40px;
+    border: 3px solid #333;
+    border-top: 3px solid #09b5a5;
+    border-radius: 50%;
+    animation: spin 1s linear infinite;
+    margin: 0 auto 1rem;
+}
+
+@keyframes spin {
+    0% { transform: rotate(0deg); }
+    100% { transform: rotate(360deg); }
+}
+
+/* Models Page */
+.models-header {
+    text-align: center;
+    margin-bottom: 3rem;
+}
+
+.models-header h1 {
+    font-size: 2.5rem;
+    font-weight: 700;
+    color: #FFFFFF;
+    margin-bottom: 1rem;
+}
+
+.models-header p {
+    font-size: 1.1rem;
+    color: #CCCCCC;
+}
+
+/* API Requests Page */
+.requests-header {
+    text-align: center;
+    margin-bottom: 3rem;
+}
+
+.requests-header h1 {
+    font-size: 2.5rem;
+    font-weight: 700;
+    color: #FFFFFF;
+    margin-bottom: 1rem;
+}
+
+.requests-header p {
+    font-size: 1.1rem;
+    color: #CCCCCC;
+}
+
+.requests-container {
+    max-width: 1200px;
+    margin: 0 auto;
+}
+
+.requests-list {
+    display: grid;
+    gap: 1.5rem;
+}
+
+.request-card {
+    border: 2px solid #333;
+    border-radius: 8px;
+    padding: 1.5rem;
+    background: #111111;
+    transition: border-color 0.2s ease;
+}
+
+.request-card:hover {
+    border-color: #09b5a5;
+}
+
+.request-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+    padding-bottom: 1rem;
+    border-bottom: 1px solid #333;
+}
+
+.request-info {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.request-url {
+    font-family: 'Courier New', monospace;
+    font-weight: 600;
+    color: #09b5a5;
+    font-size: 1.1rem;
+    word-break: break-all;
+}
+
+.request-query {
+    color: #CCCCCC;
+    font-size: 0.9rem;
+    margin-top: 0.5rem;
+    word-break: break-all;
+}
+
+.request-actions {
+    display: flex;
+    gap: 0.5rem;
+}
+
+.request-curl {
+    background: #1A1A1A;
+    border: 1px solid #333;
+    border-radius: 4px;
+    padding: 1rem;
+    margin-top: 1rem;
+}
+
+.request-curl h4 {
+    color: #FFFFFF;
+    font-size: 0.9rem;
+    font-weight: 600;
+    margin-bottom: 0.5rem;
+    font-family: 'Courier New', monospace;
+}
+
+.request-curl pre {
+    color: #CCCCCC;
+    font-size: 0.8rem;
+    line-height: 1.4;
+    overflow-x: auto;
+    white-space: pre-wrap;
+    word-break: break-all;
+    background: #111111;
+    padding: 0.75rem;
+    border-radius: 4px;
+    border: 1px solid #333;
+}
+
+.models-container {
+    max-width: 800px;
+    margin: 0 auto;
+}
+
+.model-form-section {
+    border: 2px solid #333;
+    border-radius: 8px;
+    padding: 2rem;
+    margin-bottom: 2rem;
+    background: #111111;
+}
+
+.model-form-section h3 {
+    font-size: 1.25rem;
+    font-weight: 600;
+    color: #FFFFFF;
+    margin-bottom: 1.5rem;
+}
+
+.model-form {
+    display: flex;
+    flex-direction: column;
+    gap: 1.5rem;
+}
+
+.form-row {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 1rem;
+}
+
+.save-btn {
+    padding: 1rem;
+    background: #09b5a5;
+    color: #000000;
+    border: none;
+    border-radius: 4px;
+    font-size: 1rem;
+    font-weight: 600;
+    cursor: pointer;
+    transition: background-color 0.2s ease;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    gap: 0.5rem;
+}
+
+.save-btn:hover {
+    background: #09b5a5;
+}
+
+.saved-models-section h3 {
+    font-size: 1.25rem;
+    font-weight: 600;
+    color: #FFFFFF;
+    margin-bottom: 1.5rem;
+}
+
+.models-list {
+    display: grid;
+    gap: 1rem;
+}
+
+.model-card {
+    border: 2px solid #333;
+    border-radius: 8px;
+    padding: 1.5rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    transition: border-color 0.2s ease;
+    background: #111111;
+}
+
+.model-card:hover {
+    border-color: #09b5a5;
+}
+
+.model-info {
+    flex: 1;
+}
+
+.model-name {
+    font-weight: 600;
+    color: #FFFFFF;
+    font-size: 1.1rem;
+    margin-bottom: 0.5rem;
+}
+
+.model-provider {
+    color: #CCCCCC;
+    font-size: 0.9rem;
+}
+
+.model-actions {
+    display: flex;
+    gap: 0.5rem;
+}
+
+.btn-danger {
+    background: #FF4444;
+    color: #FFFFFF;
+    border: none;
+    padding: 0.5rem 1rem;
+    border-radius: 4px;
+    font-size: 0.9rem;
+    font-weight: 600;
+    cursor: pointer;
+    transition: background-color 0.2s ease;
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+}
+
+.btn-danger:hover {
+    background: #CC3333;
+}
+
+
+
+/* Toast Notifications */
+.toast-container {
+    position: fixed;
+    top: 20px;
+    right: 20px;
+    z-index: 1000;
+}
+
+.toast {
+    background: #111111;
+    border: 2px solid #333;
+    border-radius: 4px;
+    padding: 1rem 1.5rem;
+    margin-bottom: 0.5rem;
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    animation: slideIn 0.3s ease;
+    max-width: 400px;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
+    color: #FFFFFF;
+}
+
+.toast.success {
+    border-color: #09b5a5;
+    background: #0A1A1A;
+}
+
+.toast.error {
+    border-color: #FF4444;
+    background: #1A0A0A;
+}
+
+.toast.info {
+    border-color: #09b5a5;
+    background: #0A1A1A;
+}
+
+@keyframes slideIn {
+    from {
+        transform: translateX(100%);
+        opacity: 0;
+    }
+    to {
+        transform: translateX(0);
+        opacity: 1;
+    }
+}
+
+/* Responsive Design */
+@media (max-width: 768px) {
+    .header-content {
+        padding: 0 1rem;
+    }
+    
+    .main-content {
+        padding: 1rem;
+    }
+    
+    .hero-title {
+        font-size: 2rem;
+    }
+    
+    .workflow-demo {
+        grid-template-columns: 1fr;
+        gap: 1rem;
+    }
+    
+    .workflow-arrow {
+        transform: rotate(90deg);
+        margin: 1rem 0;
+    }
+    
+    .form-options {
+        grid-template-columns: 1fr;
+    }
+    
+    .form-row {
+        grid-template-columns: 1fr;
+    }
+    
+    .result-info {
+        grid-template-columns: 1fr;
+    }
+    
+    .model-card {
+        flex-direction: column;
+        gap: 1rem;
+        text-align: center;
+    }
+    
+    .model-actions {
+        width: 100%;
+        justify-content: center;
+    }
+}
\ No newline at end of file
diff --git a/docs/examples/website-to-api/test_api.py b/docs/examples/website-to-api/test_api.py
new file mode 100644
index 0000000..8fd8db5
--- /dev/null
+++ b/docs/examples/website-to-api/test_api.py
@@ -0,0 +1,28 @@
+import asyncio
+from web_scraper_lib import scrape_website
+import os
+
+async def test_library():
+    """Test the mini library directly."""
+    print("=== Testing Mini Library ===")
+    
+    # Test 1: Scrape with a custom model
+    url = "https://marketplace.mainstreet.co.in/collections/adidas-yeezy/products/adidas-yeezy-boost-350-v2-yecheil-non-reflective"
+    query = "Extract the following data: Product name, Product price, Product description, Product size. DO NOT EXTRACT ANYTHING ELSE."
+    if os.path.exists("models"):
+        model_name = os.listdir("models")[0].split(".")[0]
+    else:
+        raise Exception("No models found in models directory")
+
+    print(f"Scraping: {url}")
+    print(f"Query: {query}")
+    
+    try:
+        result = await scrape_website(url, query, model_name)
+        print("✅ Library test successful!")
+        print(f"Extracted data: {result['extracted_data']}")
+    except Exception as e:
+        print(f"❌ Library test failed: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(test_library())
\ No newline at end of file
diff --git a/docs/examples/website-to-api/test_models.py b/docs/examples/website-to-api/test_models.py
new file mode 100644
index 0000000..2de0627
--- /dev/null
+++ b/docs/examples/website-to-api/test_models.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""
+Test script for the new model management functionality.
+This script demonstrates how to save and use custom model configurations.
+"""
+
+import asyncio
+import requests
+import json
+
+# API base URL
+BASE_URL = "http://localhost:8000"
+
+def test_model_management():
+    """Test the model management endpoints."""
+    
+    print("=== Testing Model Management ===")
+    
+    # 1. List current models
+    print("\n1. Listing current models:")
+    response = requests.get(f"{BASE_URL}/models")
+    print(f"Status: {response.status_code}")
+    print(f"Response: {json.dumps(response.json(), indent=2)}")
+    
+    
+    # 2. Save another model configuration (OpenAI example)
+    print("\n2. Saving OpenAI model configuration:")
+    openai_config = {
+        "model_name": "my-openai",
+        "provider": "openai",
+        "api_token": "your-openai-api-key-here"
+    }
+    
+    response = requests.post(f"{BASE_URL}/models", json=openai_config)
+    print(f"Status: {response.status_code}")
+    print(f"Response: {json.dumps(response.json(), indent=2)}")
+    
+    # 3. List models again to see the new ones
+    print("\n3. Listing models after adding new ones:")
+    response = requests.get(f"{BASE_URL}/models")
+    print(f"Status: {response.status_code}")
+    print(f"Response: {json.dumps(response.json(), indent=2)}")
+
+    # 4. Delete a model configuration
+    print("\n4. Deleting a model configuration:")
+    response = requests.delete(f"{BASE_URL}/models/my-openai")
+    print(f"Status: {response.status_code}")
+    print(f"Response: {json.dumps(response.json(), indent=2)}")
+    
+    # 5. Final list of models
+    print("\n5. Final list of models:")
+    response = requests.get(f"{BASE_URL}/models")
+    print(f"Status: {response.status_code}")
+    print(f"Response: {json.dumps(response.json(), indent=2)}")
+
+if __name__ == "__main__":
+    print("Model Management Test Script")
+    print("Make sure the API server is running on http://localhost:8000")
+    print("=" * 50)
+    
+    try:
+        test_model_management()
+    except requests.exceptions.ConnectionError:
+        print("Error: Could not connect to the API server.")
+        print("Make sure the server is running with: python api_server.py")
+    except Exception as e:
+        print(f"Error: {e}") 
\ No newline at end of file
diff --git a/docs/examples/website-to-api/web_scraper_lib.py b/docs/examples/website-to-api/web_scraper_lib.py
new file mode 100644
index 0000000..afa58e2
--- /dev/null
+++ b/docs/examples/website-to-api/web_scraper_lib.py
@@ -0,0 +1,397 @@
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CacheMode,
+    CrawlerRunConfig,
+    LLMConfig,
+    JsonCssExtractionStrategy,
+    LLMExtractionStrategy
+)
+import os
+import json
+import hashlib
+from typing import Dict, Any, Optional, List
+from litellm import completion
+
+class ModelConfig:
+    """Configuration for LLM models."""
+    
+    def __init__(self, provider: str, api_token: str):
+        self.provider = provider
+        self.api_token = api_token
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "provider": self.provider,
+            "api_token": self.api_token
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ModelConfig':
+        return cls(
+            provider=data["provider"],
+            api_token=data["api_token"]
+        )
+
+class WebScraperAgent:
+    """
+    A mini library that converts any website into a structured data API.
+    
+    Features:
+    1. Provide a URL and tell AI what data you need in plain English
+    2. Generate: Agent reverse-engineers the site and deploys custom scraper
+    3. Integrate: Use private API endpoint to get structured data
+    4. Support for custom LLM models and API keys
+    """
+    
+    def __init__(self, schemas_dir: str = "schemas", models_dir: str = "models"):
+        self.schemas_dir = schemas_dir
+        self.models_dir = models_dir
+        os.makedirs(self.schemas_dir, exist_ok=True)
+        os.makedirs(self.models_dir, exist_ok=True)
+    
+    def _generate_schema_key(self, url: str, query: str) -> str:
+        """Generate a unique key for schema caching based on URL and query."""
+        content = f"{url}:{query}"
+        return hashlib.md5(content.encode()).hexdigest()
+    
+    def save_model_config(self, model_name: str, provider: str, api_token: str) -> bool:
+        """
+        Save a model configuration for later use.
+        
+        Args:
+            model_name: User-friendly name for the model
+            provider: LLM provider (e.g., 'gemini', 'openai', 'anthropic')
+            api_token: API token for the provider
+            
+        Returns:
+            True if saved successfully
+        """
+        try:
+            model_config = ModelConfig(provider, api_token)
+            config_path = os.path.join(self.models_dir, f"{model_name}.json")
+            
+            with open(config_path, "w") as f:
+                json.dump(model_config.to_dict(), f, indent=2)
+            
+            print(f"Model configuration saved: {model_name}")
+            return True
+        except Exception as e:
+            print(f"Failed to save model configuration: {e}")
+            return False
+    
+    def load_model_config(self, model_name: str) -> Optional[ModelConfig]:
+        """
+        Load a saved model configuration.
+        
+        Args:
+            model_name: Name of the saved model configuration
+            
+        Returns:
+            ModelConfig object or None if not found
+        """
+        try:
+            config_path = os.path.join(self.models_dir, f"{model_name}.json")
+            if not os.path.exists(config_path):
+                return None
+            
+            with open(config_path, "r") as f:
+                data = json.load(f)
+            
+            return ModelConfig.from_dict(data)
+        except Exception as e:
+            print(f"Failed to load model configuration: {e}")
+            return None
+    
+    def list_saved_models(self) -> List[str]:
+        """List all saved model configurations."""
+        models = []
+        for filename in os.listdir(self.models_dir):
+            if filename.endswith('.json'):
+                models.append(filename[:-5])  # Remove .json extension
+        return models
+    
+    def delete_model_config(self, model_name: str) -> bool:
+        """
+        Delete a saved model configuration.
+        
+        Args:
+            model_name: Name of the model configuration to delete
+            
+        Returns:
+            True if deleted successfully
+        """
+        try:
+            config_path = os.path.join(self.models_dir, f"{model_name}.json")
+            if os.path.exists(config_path):
+                os.remove(config_path)
+                print(f"Model configuration deleted: {model_name}")
+                return True
+            return False
+        except Exception as e:
+            print(f"Failed to delete model configuration: {e}")
+            return False
+    
+    async def _load_or_generate_schema(self, url: str, query: str, session_id: str = "schema_generator", model_name: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Loads schema from cache if exists, otherwise generates using AI.
+        This is the "Generate" step - our agent reverse-engineers the site.
+        
+        Args:
+            url: URL to scrape
+            query: Query for data extraction
+            session_id: Session identifier
+            model_name: Name of saved model configuration to use
+        """
+        schema_key = self._generate_schema_key(url, query)
+        schema_path = os.path.join(self.schemas_dir, f"{schema_key}.json")
+        
+        if os.path.exists(schema_path):
+            print(f"Schema found in cache for {url}")
+            with open(schema_path, "r") as f:
+                return json.load(f)
+        
+        print(f"Generating new schema for {url}")
+        print(f"Query: {query}")
+        query += """
+        IMPORTANT:
+        GENERATE THE SCHEMA WITH ONLY THE FIELDS MENTIONED IN THE QUERY. MAKE SURE THE NUMBER OF FIELDS IN THE SCHEME MATCH THE NUMBER OF FIELDS IN THE QUERY.
+        """
+        
+        # Step 1: Fetch the page HTML
+        async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(
+                    cache_mode=CacheMode.BYPASS,
+                    session_id=session_id,
+                    simulate_user=True,
+                    remove_overlay_elements=True,
+                    delay_before_return_html=5,
+                )
+            )
+            html = result.markdown.fit_html
+        
+        # Step 2: Generate schema using AI with custom model if specified
+        print("AI is analyzing the page structure...")
+        
+        # Use custom model configuration if provided
+        if model_name:
+            model_config = self.load_model_config(model_name)
+            if model_config:
+                llm_config = LLMConfig(
+                    provider=model_config.provider,
+                    api_token=model_config.api_token
+                )
+                print(f"Using custom model: {model_name}")
+            else:
+                raise ValueError(f"Model configuration '{model_name}' not found. Please add it from the Models page.")
+        else:
+            # Require a model to be specified
+            raise ValueError("No model specified. Please select a model from the dropdown or add one from the Models page.")
+        
+        schema = JsonCssExtractionStrategy.generate_schema(
+            html=html,
+            llm_config=llm_config,
+            query=query
+        )
+        
+        # Step 3: Cache the generated schema
+        print(f"Schema generated and cached: {json.dumps(schema, indent=2)}")
+        with open(schema_path, "w") as f:
+            json.dump(schema, f, indent=2)
+        
+        return schema
+    
+    def _generate_llm_schema(self, query: str, llm_config: LLMConfig) -> Dict[str, Any]:
+        """
+        Generate a schema for a given query using a custom LLM model.
+        
+        Args:
+            query: Plain English description of what data to extract
+            model_config: Model configuration to use
+        """
+        # ask the model to generate a schema for the given query in the form of a json.
+        prompt = f"""
+        IDENTIFY THE FIELDS FOR EXTRACTION MENTIONED IN THE QUERY and GENERATE A JSON SCHEMA FOR THE FIELDS.
+        eg.
+        {{
+            "name": "str",  
+            "age": "str",
+            "email": "str",
+            "product_name": "str",
+            "product_price": "str",
+            "product_description": "str",
+            "product_image": "str",
+            "product_url": "str",
+            "product_rating": "str",
+            "product_reviews": "str",
+        }}
+        Here is the query:
+        {query}
+        IMPORTANT:
+        THE RESULT SHOULD BE A JSON OBJECT.
+        MAKE SURE THE NUMBER OF FIELDS IN THE RESULT MATCH THE NUMBER OF FIELDS IN THE QUERY.
+        THE RESULT SHOULD BE A JSON OBJECT.
+        """
+        response = completion(
+            model=llm_config.provider,
+            messages=[{"role": "user", "content": prompt}],
+            api_key=llm_config.api_token,
+            result_type="json"
+        )
+
+        return response.json()["choices"][0]["message"]["content"]
+    async def scrape_data_with_llm(self, url: str, query: str, model_name: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Scrape structured data from any website using a custom LLM model.
+        
+        Args:
+            url: The website URL to scrape
+            query: Plain English description of what data to extract
+            model_name: Name of saved model configuration to use
+        """
+
+        if model_name:
+            model_config = self.load_model_config(model_name)
+            if model_config:
+                llm_config = LLMConfig(
+                    provider=model_config.provider,
+                    api_token=model_config.api_token
+                )
+                print(f"Using custom model: {model_name}")
+            else:
+                raise ValueError(f"Model configuration '{model_name}' not found. Please add it from the Models page.")
+        else:
+            # Require a model to be specified
+            raise ValueError("No model specified. Please select a model from the dropdown or add one from the Models page.")
+
+        query += """\n  
+        IMPORTANT:
+        THE RESULT SHOULD BE A JSON OBJECT WITH THE ONLY THE FIELDS MENTIONED IN THE QUERY.
+        MAKE SURE THE NUMBER OF FIELDS IN THE RESULT MATCH THE NUMBER OF FIELDS IN THE QUERY.
+        THE RESULT SHOULD BE A JSON OBJECT.
+        """
+
+        schema = self._generate_llm_schema(query, llm_config)
+
+        print(f"Schema: {schema}")
+
+        llm_extraction_strategy = LLMExtractionStrategy(
+            llm_config=llm_config,
+            instruction=query,
+            result_type="json",
+            schema=schema
+        )
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(
+                    cache_mode=CacheMode.BYPASS,
+                    simulate_user=True,
+                    extraction_strategy=llm_extraction_strategy,
+                )
+            )
+        extracted_data = result.extracted_content
+        if isinstance(extracted_data, str):
+                try:
+                    extracted_data = json.loads(extracted_data)
+                except json.JSONDecodeError:
+                    # If it's not valid JSON, keep it as string
+                    pass
+            
+        return {
+                "url": url,
+                "query": query,
+                "extracted_data": extracted_data,
+                "timestamp": result.timestamp if hasattr(result, 'timestamp') else None
+            }
+        
+    async def scrape_data(self, url: str, query: str, model_name: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Main method to scrape structured data from any website.
+        
+        Args:
+            url: The website URL to scrape
+            query: Plain English description of what data to extract
+            model_name: Name of saved model configuration to use
+            
+        Returns:
+            Structured data extracted from the website
+        """
+        # Step 1: Generate or load schema (reverse-engineer the site)
+        schema = await self._load_or_generate_schema(url=url, query=query, model_name=model_name)
+        
+        # Step 2: Deploy custom high-speed scraper
+        print(f"Deploying custom scraper for {url}")
+        browser_config = BrowserConfig(headless=True)
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            run_config = CrawlerRunConfig(
+                extraction_strategy=JsonCssExtractionStrategy(schema=schema),
+            )
+            result = await crawler.arun(url=url, config=run_config)
+            
+            # Step 3: Return structured data
+            # Parse extracted_content if it's a JSON string
+            extracted_data = result.extracted_content
+            if isinstance(extracted_data, str):
+                try:
+                    extracted_data = json.loads(extracted_data)
+                except json.JSONDecodeError:
+                    # If it's not valid JSON, keep it as string
+                    pass
+            
+            return {
+                "url": url,
+                "query": query,
+                "extracted_data": extracted_data,
+                "schema_used": schema,
+                "timestamp": result.timestamp if hasattr(result, 'timestamp') else None
+            }
+    
+    async def get_cached_schemas(self) -> Dict[str, str]:
+        """Get list of cached schemas."""
+        schemas = {}
+        for filename in os.listdir(self.schemas_dir):
+            if filename.endswith('.json'):
+                schema_key = filename[:-5]  # Remove .json extension
+                schemas[schema_key] = filename
+        return schemas
+    
+    def clear_cache(self):
+        """Clear all cached schemas."""
+        import shutil
+        if os.path.exists(self.schemas_dir):
+            shutil.rmtree(self.schemas_dir)
+        os.makedirs(self.schemas_dir, exist_ok=True)
+        print("Schema cache cleared")
+
+# Convenience function for simple usage
+async def scrape_website(url: str, query: str, model_name: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Simple function to scrape any website with plain English instructions.
+    
+    Args:
+        url: Website URL
+        query: Plain English description of what data to extract
+        model_name: Name of saved model configuration to use
+        
+    Returns:
+        Extracted structured data
+    """
+    agent = WebScraperAgent()
+    return await agent.scrape_data(url, query, model_name)
+
+async def scrape_website_with_llm(url: str, query: str, model_name: Optional[str] = None):
+    """
+    Scrape structured data from any website using a custom LLM model.
+    
+    Args:
+        url: The website URL to scrape
+        query: Plain English description of what data to extract
+        model_name: Name of saved model configuration to use
+    """
+    agent = WebScraperAgent()
+    return await agent.scrape_data_with_llm(url, query, model_name)
\ No newline at end of file
diff --git a/docs/md_v2/CONTRIBUTING.md b/docs/md_v2/CONTRIBUTING.md
new file mode 100644
index 0000000..38d22c2
--- /dev/null
+++ b/docs/md_v2/CONTRIBUTING.md
@@ -0,0 +1,102 @@
+# Contributing to Crawl4AI
+
+Welcome to the Crawl4AI project! As an open-source library for web crawling and AI integration, we value contributions from the community. This guide explains our branching strategy, how to contribute effectively, and the overall release process. Our goal is to maintain a stable, collaborative environment where bug fixes, features, and improvements can be integrated smoothly while allowing for experimental development.
+
+We follow a GitFlow-inspired workflow to ensure predictability and quality. Releases occur approximately every two weeks, with a focus on semantic versioning, comprehensive documentation, and user-friendly updates.
+
+## Core Branches
+
+- **main**: The stable branch containing production-ready code. It's always identical to the latest released version and is tagged for releases. Do not submit PRs directly here.
+- **develop**: The primary integration branch for ongoing development. This is where all contributions (bug fixes, minor features, documentation updates) are merged. Submit your pull requests targeting this branch.
+- **next**: Reserved for the lead maintainer (Unclecode) to experiment with major features, refactors, or cutting-edge changes. These are merged into `develop` when ready.
+- **release/vX.Y.Z**: Temporary branches created from `develop` for final release preparations (e.g., version bumps, demos, release notes). These are short-lived and deleted after the release.
+
+## Contributor Workflow
+
+We encourage contributions of all kinds: bug fixes, new features, documentation improvements, tests, or even Docker enhancements. Follow these steps to contribute:
+
+1. **Fork the Repository**: Create your own fork on GitHub.
+2. **Create a Branch**: Base your work on the `develop` branch.
+    
+    ```
+    git checkout develop
+    git checkout -b feature/your-feature-name  # Or bugfix/your-bugfix-name
+    
+    ```
+    
+3. **Make Changes**:
+    - Implement your feature or fix.
+    - If updating documentation (e.g., README.md, mkdocs.yml, or docs/blog/), ensure version references are consistent (e.g., update site_name in mkdocs.yml to reflect the upcoming version if relevant).
+    - For Docker-related changes (e.g., Dockerfile, docker-compose.yml, or docs/md_v2/core/docker-deployment.md), test locally and include build instructions in your PR description.
+    - Add tests if applicable (run `pytest` to verify).
+    - Follow code style guidelines (use black for formatting).
+4. **Commit and Push**:
+    - Use descriptive commit messages (e.g., "Fix: Resolve issue with async crawling").
+    - Push to your fork: `git push origin feature/your-feature-name`.
+5. **Submit a Pull Request**:
+    - Target the `develop` branch.
+    - Provide a clear description: What does it do? Link to any issues. Include screenshots or code examples if helpful.
+    - If your change affects documentation or Docker, mention how it aligns with the version (e.g., "Updates Docker docs for v0.7.0 compatibility").
+    - We'll review and merge approved PRs into `develop`.
+6. **Discuss Large Changes**: For major features or experimental ideas, open an issue first to align with the project's direction.
+
+If your PR involves breaking changes, include a migration guide in the description.
+
+## Lead Maintainer's Workflow (For Reference)
+
+- The lead maintainer (Unclecode) uses the `next` branch for isolated experimental work.
+- Features from `next` are periodically merged into `develop` (via rebase and merge) to keep everything in sync.
+- This isolation ensures your contributions aren't disrupted by ongoing major changes.
+
+## Release Process (High-Level Overview)
+
+Releases happen bi-weekly to ship improvements regularly. As a contributor, your merged changes in `develop` will be included in the next release unless specified otherwise. Here's a summary of what happens:
+
+- **Preparation**: A temporary `release/vX.Y.Z` branch is created from `develop`. Any ready features from `next` are merged here.
+- **Final Updates**:
+    - Version bump in code (e.g., `__version__.py`).
+    - Creation of a demo script in `examples/` to showcase new features.
+    - Writing release notes in `docs/blog/` (personal "I" voice from Unclecode, with code examples, impacts, and migration guides if needed).
+    - Documentation updates: README.md (highlights, version refs), mkdocs.yml (site_name with version), docs/blog/index.md (add new release), and copying notes to `docs/md_v2/blog/releases/`.
+    - Docker updates: Dockerfile (version arg), docker-compose.yml, deploy/docker/README.md, and docs/md_v2/core/docker-deployment.md. A release candidate image (e.g., `X.Y.Z-r1`) is built and tested.
+- **Testing and Merge**: Full tests run; changes committed and merged to `main` with a tag.
+- **Publication**: Tagged release on GitHub (with notes), publish to PyPI, and push Docker images (stable and `latest` after testing).
+- **Sync**: Back-merge to `develop` and reset `next` for the next cycle.
+
+Semantic versioning is used: MAJOR for breaking changes, MINOR for features, PATCH for fixes. Pre-releases (e.g., `-rc1`) may be used for testing.
+
+If your contribution requires Docker testing or affects docs, it may be part of this step—feel free to suggest updates in your PR.
+
+## Benefits of This Approach
+
+- **Stability**: `main` is always reliable for users.
+- **Collaboration**: Fixed PR target (`develop`) makes contributing straightforward.
+- **Isolation**: Experimental work in `next` doesn't block team progress.
+- **User-Focused**: Releases include demos, detailed notes, and updated docs/Docker for easy adoption.
+- **Predictability**: Bi-weekly cadence keeps the project active.
+
+## Checklist for Contributors
+
+Before submitting a PR:
+
+- [ ]  Based on and targeting `develop`.
+- [ ]  Tests pass (`pytest`).
+- [ ]  Docs updated if needed (e.g., version refs in mkdocs.yml, Docker files).
+- [ ]  No breaking changes without a migration guide.
+- [ ]  Descriptive title and description.
+
+## Common Issues
+
+- **Merge Conflicts**: Rebase your branch on latest `develop` before PR.
+- **Docker Builds**: Test multi-arch (amd64/arm64) locally if changing Dockerfile.
+- **Version Consistency**: Ensure any version mentions match semantic rules.
+
+## Communication
+
+- Open issues for discussions or bugs.
+- Join our Discord (link in README) for real-time help.
+- After releases, announcements go to GitHub, Discord, and social media.
+
+Thanks for contributing to Crawl4AI — we appreciate your help in making it better!
+
+*Last Updated: Feb 3, 2026*
diff --git a/docs/md_v2/advanced/adaptive-strategies.md b/docs/md_v2/advanced/adaptive-strategies.md
new file mode 100644
index 0000000..11c5585
--- /dev/null
+++ b/docs/md_v2/advanced/adaptive-strategies.md
@@ -0,0 +1,400 @@
+# Advanced Adaptive Strategies
+
+## Overview
+
+While the default adaptive crawling configuration works well for most use cases, understanding the underlying strategies and scoring mechanisms allows you to fine-tune the crawler for specific domains and requirements.
+
+## The Three-Layer Scoring System
+
+### 1. Coverage Score
+
+Coverage measures how comprehensively your knowledge base covers the query terms and related concepts.
+
+#### Mathematical Foundation
+
+```python
+Coverage(K, Q) = Σ(t ∈ Q) score(t, K) / |Q|
+
+where score(t, K) = doc_coverage(t) × (1 + freq_boost(t))
+```
+
+#### Components
+
+- **Document Coverage**: Percentage of documents containing the term
+- **Frequency Boost**: Logarithmic bonus for term frequency
+- **Query Decomposition**: Handles multi-word queries intelligently
+
+#### Tuning Coverage
+
+```python
+# For technical documentation with specific terminology
+config = AdaptiveConfig(
+    confidence_threshold=0.85,  # Require high coverage
+    top_k_links=5              # Cast wider net
+)
+
+# For general topics with synonyms
+config = AdaptiveConfig(
+    confidence_threshold=0.6,   # Lower threshold
+    top_k_links=2              # More focused
+)
+```
+
+### 2. Consistency Score
+
+Consistency evaluates whether the information across pages is coherent and non-contradictory.
+
+#### How It Works
+
+1. Extracts key statements from each document
+2. Compares statements across documents
+3. Measures agreement vs. contradiction
+4. Returns normalized score (0-1)
+
+#### Practical Impact
+
+- **High consistency (>0.8)**: Information is reliable and coherent
+- **Medium consistency (0.5-0.8)**: Some variation, but generally aligned
+- **Low consistency (<0.5)**: Conflicting information, need more sources
+
+### 3. Saturation Score
+
+Saturation detects when new pages stop providing novel information.
+
+#### Detection Algorithm
+
+```python
+# Tracks new unique terms per page
+new_terms_page_1 = 50
+new_terms_page_2 = 30  # 60% of first
+new_terms_page_3 = 15  # 50% of second
+new_terms_page_4 = 5   # 33% of third
+# Saturation detected: rapidly diminishing returns
+```
+
+#### Configuration
+
+```python
+config = AdaptiveConfig(
+    min_gain_threshold=0.1  # Stop if <10% new information
+)
+```
+
+## Link Ranking Algorithm
+
+### Expected Information Gain
+
+Each uncrawled link is scored based on:
+
+```python
+ExpectedGain(link) = Relevance × Novelty × Authority
+```
+
+#### 1. Relevance Scoring
+
+Uses BM25 algorithm on link preview text:
+
+```python
+relevance = BM25(link.preview_text, query)
+```
+
+Factors:
+- Term frequency in preview
+- Inverse document frequency
+- Preview length normalization
+
+#### 2. Novelty Estimation
+
+Measures how different the link appears from already-crawled content:
+
+```python
+novelty = 1 - max_similarity(preview, knowledge_base)
+```
+
+Prevents crawling duplicate or highly similar pages.
+
+#### 3. Authority Calculation
+
+URL structure and domain analysis:
+
+```python
+authority = f(domain_rank, url_depth, url_structure)
+```
+
+Factors:
+- Domain reputation
+- URL depth (fewer slashes = higher authority)
+- Clean URL structure
+
+## Domain-Specific Configurations
+
+### Technical Documentation
+
+```python
+tech_doc_config = AdaptiveConfig(
+    confidence_threshold=0.85,
+    max_pages=30,
+    top_k_links=3,
+    min_gain_threshold=0.05  # Keep crawling for small gains
+)
+```
+
+Rationale:
+- High threshold ensures comprehensive coverage
+- Lower gain threshold captures edge cases
+- Moderate link following for depth
+
+### News & Articles
+
+```python
+news_config = AdaptiveConfig(
+    confidence_threshold=0.6,
+    max_pages=10,
+    top_k_links=5,
+    min_gain_threshold=0.15  # Stop quickly on repetition
+)
+```
+
+Rationale:
+- Lower threshold (articles often repeat information)
+- Higher gain threshold (avoid duplicate stories)
+- More links per page (explore different perspectives)
+
+### E-commerce
+
+```python
+ecommerce_config = AdaptiveConfig(
+    confidence_threshold=0.7,
+    max_pages=20,
+    top_k_links=2,
+    min_gain_threshold=0.1
+)
+```
+
+Rationale:
+- Balanced threshold for product variations
+- Focused link following (avoid infinite products)
+- Standard gain threshold
+
+### Research & Academic
+
+```python
+research_config = AdaptiveConfig(
+    confidence_threshold=0.9,
+    max_pages=50,
+    top_k_links=4,
+    min_gain_threshold=0.02  # Very low - capture citations
+)
+```
+
+Rationale:
+- Very high threshold for completeness
+- Many pages allowed for thorough research
+- Very low gain threshold to capture references
+
+## Performance Optimization
+
+### Memory Management
+
+```python
+# For large crawls, use streaming
+config = AdaptiveConfig(
+    max_pages=100,
+    save_state=True,
+    state_path="large_crawl.json"
+)
+
+# Periodically clean state
+if len(state.knowledge_base) > 1000:
+    # Keep only the top 500 most relevant docs
+    top_content = adaptive.get_relevant_content(top_k=500)
+    keep_indices = {d["index"] for d in top_content}
+    state.knowledge_base = [
+        doc for i, doc in enumerate(state.knowledge_base) if i in keep_indices
+    ]
+```
+
+### Parallel Processing
+
+```python
+# Use multiple start points
+start_urls = [
+    "https://docs.example.com/intro",
+    "https://docs.example.com/api",
+    "https://docs.example.com/guides"
+]
+
+# Crawl in parallel
+tasks = [
+    adaptive.digest(url, query)
+    for url in start_urls
+]
+results = await asyncio.gather(*tasks)
+```
+
+## Debugging & Analysis
+
+### Enable Verbose Logging
+
+```python
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
+adaptive = AdaptiveCrawler(crawler, config, verbose=True)
+```
+
+### Analyze Crawl Patterns
+
+```python
+# After crawling
+state = await adaptive.digest(start_url, query)
+
+# Analyze link selection
+print("Link selection order:")
+for i, url in enumerate(state.crawl_order):
+    print(f"{i+1}. {url}")
+
+# Analyze term discovery
+print("\nTerm discovery rate:")
+for i, new_terms in enumerate(state.new_terms_history):
+    print(f"Page {i+1}: {new_terms} new terms")
+
+# Analyze score progression
+print("\nScore progression:")
+print(f"Coverage: {state.metrics['coverage_history']}")
+print(f"Saturation: {state.metrics['saturation_history']}")
+```
+
+### Export for Analysis
+
+```python
+# Export detailed metrics
+import json
+
+metrics = {
+    "query": query,
+    "total_pages": len(state.crawled_urls),
+    "confidence": adaptive.confidence,
+    "coverage_stats": adaptive.coverage_stats,
+    "crawl_order": state.crawl_order,
+    "term_frequencies": dict(state.term_frequencies),
+    "new_terms_history": state.new_terms_history
+}
+
+with open("crawl_analysis.json", "w") as f:
+    json.dump(metrics, f, indent=2)
+```
+
+## Custom Strategies
+
+### Implementing a Custom Strategy
+
+```python
+from crawl4ai.adaptive_crawler import CrawlStrategy
+
+class DomainSpecificStrategy(CrawlStrategy):
+    def calculate_coverage(self, state: CrawlState) -> float:
+        # Custom coverage calculation
+        # e.g., weight certain terms more heavily
+        pass
+    
+    def calculate_consistency(self, state: CrawlState) -> float:
+        # Custom consistency logic
+        # e.g., domain-specific validation
+        pass
+    
+    def rank_links(self, links: List[Link], state: CrawlState) -> List[Link]:
+        # Custom link ranking
+        # e.g., prioritize specific URL patterns
+        pass
+
+# Use custom strategy
+adaptive = AdaptiveCrawler(
+    crawler,
+    config=config,
+    strategy=DomainSpecificStrategy()
+)
+```
+
+### Combining Strategies
+
+```python
+class HybridStrategy(CrawlStrategy):
+    def __init__(self):
+        self.strategies = [
+            TechnicalDocStrategy(),
+            SemanticSimilarityStrategy(),
+            URLPatternStrategy()
+        ]
+    
+    def calculate_confidence(self, state: CrawlState) -> float:
+        # Weighted combination of strategies
+        scores = [s.calculate_confidence(state) for s in self.strategies]
+        weights = [0.5, 0.3, 0.2]
+        return sum(s * w for s, w in zip(scores, weights))
+```
+
+## Best Practices
+
+### 1. Start Conservative
+
+Begin with default settings and adjust based on results:
+
+```python
+# Start with defaults
+result = await adaptive.digest(url, query)
+
+# Analyze and adjust
+if adaptive.confidence < 0.7:
+    config.max_pages += 10
+    config.confidence_threshold -= 0.1
+```
+
+### 2. Monitor Resource Usage
+
+```python
+import psutil
+
+# Check memory before large crawls
+memory_percent = psutil.virtual_memory().percent
+if memory_percent > 80:
+    config.max_pages = min(config.max_pages, 20)
+```
+
+### 3. Use Domain Knowledge
+
+```python
+# For API documentation
+if "api" in start_url:
+    config.top_k_links = 2  # APIs have clear structure
+    
+# For blogs
+if "blog" in start_url:
+    config.min_gain_threshold = 0.2  # Avoid similar posts
+```
+
+### 4. Validate Results
+
+```python
+# Always validate the knowledge base
+relevant_content = adaptive.get_relevant_content(top_k=10)
+
+# Check coverage
+query_terms = set(query.lower().split())
+covered_terms = set()
+
+for doc in relevant_content:
+    content_lower = doc['content'].lower()
+    for term in query_terms:
+        if term in content_lower:
+            covered_terms.add(term)
+
+coverage_ratio = len(covered_terms) / len(query_terms)
+print(f"Query term coverage: {coverage_ratio:.0%}")
+```
+
+## Next Steps
+
+- Explore [Custom Strategy Implementation](../tutorials/custom-adaptive-strategies.md)
+- Learn about [Knowledge Base Management](../tutorials/knowledge-base-management.md)
+- See [Performance Benchmarks](../benchmarks/adaptive-performance.md)
\ No newline at end of file
diff --git a/docs/md_v2/advanced/advanced-features.md b/docs/md_v2/advanced/advanced-features.md
new file mode 100644
index 0000000..75ba1cc
--- /dev/null
+++ b/docs/md_v2/advanced/advanced-features.md
@@ -0,0 +1,446 @@
+# Overview of Some Important Advanced Features 
+(Proxy, PDF, Screenshot, SSL, Headers, & Storage State)
+
+Crawl4AI offers multiple power-user features that go beyond simple crawling. This tutorial covers:
+
+1. **Proxy Usage**  
+2. **Capturing PDFs & Screenshots**  
+3. **Handling SSL Certificates**  
+4. **Custom Headers**  
+5. **Session Persistence & Local Storage**  
+6. **Robots.txt Compliance**  
+
+> **Prerequisites**  
+> - You have a basic grasp of [AsyncWebCrawler Basics](../core/simple-crawling.md)  
+> - You know how to run or configure your Python environment with Playwright installed
+
+---
+
+## 1. Proxy Usage
+
+If you need to route your crawl traffic through a proxy—whether for IP rotation, geo-testing, or privacy—Crawl4AI supports it via `BrowserConfig.proxy_config`.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def main():
+    browser_cfg = BrowserConfig(
+        proxy_config={
+            "server": "http://proxy.example.com:8080",
+            "username": "myuser",
+            "password": "mypass",
+        },
+        headless=True
+    )
+    crawler_cfg = CrawlerRunConfig(
+        verbose=True
+    )
+
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        result = await crawler.arun(
+            url="https://www.whatismyip.com/",
+            config=crawler_cfg
+        )
+        if result.success:
+            print("[OK] Page fetched via proxy.")
+            print("Page HTML snippet:", result.html[:200])
+        else:
+            print("[ERROR]", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key Points**  
+- **`proxy_config`** expects a dict with `server` and optional auth credentials.  
+- Many commercial proxies provide an HTTP/HTTPS “gateway” server that you specify in `server`.  
+- If your proxy doesn’t need auth, omit `username`/`password`.
+
+---
+
+## 2. Capturing PDFs & Screenshots
+
+Sometimes you need a visual record of a page or a PDF “printout.” Crawl4AI can do both in one pass:
+
+```python
+import os, asyncio
+from base64 import b64decode
+from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
+
+async def main():
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        screenshot=True,
+        pdf=True
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://en.wikipedia.org/wiki/List_of_common_misconceptions",
+            config=run_config
+        )
+        if result.success:
+            print(f"Screenshot data present: {result.screenshot is not None}")
+            print(f"PDF data present: {result.pdf is not None}")
+
+            if result.screenshot:
+                print(f"[OK] Screenshot captured, size: {len(result.screenshot)} bytes")
+                with open("wikipedia_screenshot.png", "wb") as f:
+                    f.write(b64decode(result.screenshot))
+            else:
+                print("[WARN] Screenshot data is None.")
+
+            if result.pdf:
+                print(f"[OK] PDF captured, size: {len(result.pdf)} bytes")
+                with open("wikipedia_page.pdf", "wb") as f:
+                    f.write(result.pdf)
+            else:
+                print("[WARN] PDF data is None.")
+
+        else:
+            print("[ERROR]", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Why PDF + Screenshot?**  
+- Large or complex pages can be slow or error-prone with “traditional” full-page screenshots.  
+- Exporting a PDF is more reliable for very long pages. Crawl4AI automatically converts the first PDF page into an image if you request both.  
+
+**Relevant Parameters**  
+- **`pdf=True`**: Exports the current page as a PDF (base64-encoded in `result.pdf`).  
+- **`screenshot=True`**: Creates a screenshot (base64-encoded in `result.screenshot`).  
+- **`scroll_delay`**: Controls the delay (seconds) between scroll steps when taking a full-page screenshot of a tall page. Defaults to `0.2`. Increase for pages with slow-loading assets.  
+- **`scan_full_page`** or advanced hooking can further refine how the crawler captures content.
+
+---
+
+## 3. Handling SSL Certificates
+
+If you need to verify or export a site’s SSL certificate—for compliance, debugging, or data analysis—Crawl4AI can fetch it during the crawl:
+
+```python
+import asyncio, os
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+
+async def main():
+    tmp_dir = os.path.join(os.getcwd(), "tmp")
+    os.makedirs(tmp_dir, exist_ok=True)
+    
+    config = CrawlerRunConfig(
+        fetch_ssl_certificate=True,
+        cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url="https://example.com", config=config)
+        
+        if result.success and result.ssl_certificate:
+            cert = result.ssl_certificate
+            print("\nCertificate Information:")
+            print(f"Issuer (CN): {cert.issuer.get('CN', '')}")
+            print(f"Valid until: {cert.valid_until}")
+            print(f"Fingerprint: {cert.fingerprint}")
+
+            # Export in multiple formats:
+            cert.to_json(os.path.join(tmp_dir, "certificate.json"))
+            cert.to_pem(os.path.join(tmp_dir, "certificate.pem"))
+            cert.to_der(os.path.join(tmp_dir, "certificate.der"))
+            
+            print("\nCertificate exported to JSON/PEM/DER in 'tmp' folder.")
+        else:
+            print("[ERROR] No certificate or crawl failed.")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key Points**  
+- **`fetch_ssl_certificate=True`** triggers certificate retrieval.  
+- `result.ssl_certificate` includes methods (`to_json`, `to_pem`, `to_der`) for saving in various formats (handy for server config, Java keystores, etc.).
+
+---
+
+## 4. Custom Headers
+
+Sometimes you need to set custom headers (e.g., language preferences, authentication tokens, or specialized user-agent strings). You can do this in multiple ways:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+    # Option 1: Set headers at the crawler strategy level
+    crawler1 = AsyncWebCrawler(
+        # The underlying strategy can accept headers in its constructor
+        crawler_strategy=None  # We'll override below for clarity
+    )
+    crawler1.crawler_strategy.update_user_agent("MyCustomUA/1.0")
+    crawler1.crawler_strategy.set_custom_headers({
+        "Accept-Language": "fr-FR,fr;q=0.9"
+    })
+    result1 = await crawler1.arun("https://www.example.com")
+    print("Example 1 result success:", result1.success)
+
+    # Option 2: Pass headers directly to `arun()`
+    crawler2 = AsyncWebCrawler()
+    result2 = await crawler2.arun(
+        url="https://www.example.com",
+        headers={"Accept-Language": "es-ES,es;q=0.9"}
+    )
+    print("Example 2 result success:", result2.success)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Notes**  
+- Some sites may react differently to certain headers (e.g., `Accept-Language`).  
+- If you need advanced user-agent randomization or client hints, see [Identity-Based Crawling (Anti-Bot)](./identity-based-crawling.md) or use `UserAgentGenerator`.
+
+---
+
+## 5. Session Persistence & Local Storage
+
+Crawl4AI can preserve cookies and localStorage so you can continue where you left off—ideal for logging into sites or skipping repeated auth flows.
+
+### 5.1 `storage_state`
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+    storage_dict = {
+        "cookies": [
+            {
+                "name": "session",
+                "value": "abcd1234",
+                "domain": "example.com",
+                "path": "/",
+                "expires": 1699999999.0,
+                "httpOnly": False,
+                "secure": False,
+                "sameSite": "None"
+            }
+        ],
+        "origins": [
+            {
+                "origin": "https://example.com",
+                "localStorage": [
+                    {"name": "token", "value": "my_auth_token"}
+                ]
+            }
+        ]
+    }
+
+    # Provide the storage state as a dictionary to start "already logged in"
+    async with AsyncWebCrawler(
+        headless=True,
+        storage_state=storage_dict
+    ) as crawler:
+        result = await crawler.arun("https://example.com/protected")
+        if result.success:
+            print("Protected page content length:", len(result.html))
+        else:
+            print("Failed to crawl protected page")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### 5.2 Exporting & Reusing State
+
+You can sign in once, export the browser context, and reuse it later—without re-entering credentials.
+
+- **`await context.storage_state(path="my_storage.json")`**: Exports cookies, localStorage, etc. to a file.  
+- Provide `storage_state="my_storage.json"` on subsequent runs to skip the login step.
+
+**See**: [Detailed session management tutorial](./session-management.md) or [Explanations → Browser Context & Managed Browser](./identity-based-crawling.md) for more advanced scenarios (like multi-step logins, or capturing after interactive pages).
+
+---
+
+## 6. Robots.txt Compliance
+
+Crawl4AI supports respecting robots.txt rules with efficient caching:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    # Enable robots.txt checking in config
+    config = CrawlerRunConfig(
+        check_robots_txt=True  # Will check and respect robots.txt rules
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            config=config
+        )
+        
+        if not result.success and result.status_code == 403:
+            print("Access denied by robots.txt")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key Points**
+- Robots.txt files are cached locally for efficiency
+- Cache is stored in `~/.crawl4ai/robots/robots_cache.db`
+- Cache has a default TTL of 7 days
+- If robots.txt can't be fetched, crawling is allowed
+- Returns 403 status code if URL is disallowed
+
+---
+
+## Putting It All Together
+
+Here’s a snippet that combines multiple “advanced” features (proxy, PDF, screenshot, SSL, custom headers, and session reuse) into one run. Normally, you’d tailor each setting to your project’s needs.
+
+```python
+import os, asyncio
+from base64 import b64decode
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def main():
+    # 1. Browser config with proxy + headless
+    browser_cfg = BrowserConfig(
+        proxy_config={
+            "server": "http://proxy.example.com:8080",
+            "username": "myuser",
+            "password": "mypass",
+        },
+        headless=True,
+    )
+
+    # 2. Crawler config with PDF, screenshot, SSL, custom headers, and ignoring caches
+    crawler_cfg = CrawlerRunConfig(
+        pdf=True,
+        screenshot=True,
+        fetch_ssl_certificate=True,
+        cache_mode=CacheMode.BYPASS,
+        headers={"Accept-Language": "en-US,en;q=0.8"},
+        storage_state="my_storage.json",  # Reuse session from a previous sign-in
+        verbose=True,
+    )
+
+    # 3. Crawl
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        result = await crawler.arun(
+            url = "https://secure.example.com/protected", 
+            config=crawler_cfg
+        )
+        
+        if result.success:
+            print("[OK] Crawled the secure page. Links found:", len(result.links.get("internal", [])))
+            
+            # Save PDF & screenshot
+            if result.pdf:
+                with open("result.pdf", "wb") as f:
+                    f.write(b64decode(result.pdf))
+            if result.screenshot:
+                with open("result.png", "wb") as f:
+                    f.write(b64decode(result.screenshot))
+            
+            # Check SSL cert
+            if result.ssl_certificate:
+                print("SSL Issuer CN:", result.ssl_certificate.issuer.get("CN", ""))
+        else:
+            print("[ERROR]", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+---
+
+## 7. Anti-Bot Features (Stealth Mode & Undetected Browser)
+
+Crawl4AI provides two powerful features to bypass bot detection:
+
+### 7.1 Stealth Mode
+
+Stealth mode uses playwright-stealth to modify browser fingerprints and behaviors. Enable it with a simple flag:
+
+```python
+browser_config = BrowserConfig(
+    enable_stealth=True,  # Activates stealth mode
+    headless=False
+)
+```
+
+**When to use**: Sites with basic bot detection (checking navigator.webdriver, plugins, etc.)
+
+### 7.2 Undetected Browser
+
+For advanced bot detection, use the undetected browser adapter:
+
+```python
+from crawl4ai import UndetectedAdapter
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+# Create undetected adapter
+adapter = UndetectedAdapter()
+strategy = AsyncPlaywrightCrawlerStrategy(
+    browser_config=browser_config,
+    browser_adapter=adapter
+)
+
+async with AsyncWebCrawler(crawler_strategy=strategy, config=browser_config) as crawler:
+    # Your crawling code
+```
+
+**When to use**: Sites with sophisticated bot detection (Cloudflare, DataDome, etc.)
+
+### 7.3 Combining Both
+
+For maximum evasion, combine stealth mode with undetected browser:
+
+```python
+browser_config = BrowserConfig(
+    enable_stealth=True,  # Enable stealth
+    headless=False
+)
+
+adapter = UndetectedAdapter()  # Use undetected browser
+```
+
+### Choosing the Right Approach
+
+| Detection Level | Recommended Approach |
+|----------------|---------------------|
+| No protection | Regular browser |
+| Basic checks | Regular + Stealth mode |
+| Advanced protection | Undetected browser |
+| Maximum evasion | Undetected + Stealth mode |
+
+**Best Practice**: Start with regular browser + stealth mode. Only use undetected browser if needed, as it may be slightly slower.
+
+See [Undetected Browser Mode](undetected-browser.md) for detailed examples.
+
+---
+
+## Conclusion & Next Steps
+
+You've now explored several **advanced** features:
+
+- **Proxy Usage**  
+- **PDF & Screenshot** capturing for large or critical pages  
+- **SSL Certificate** retrieval & exporting  
+- **Custom Headers** for language or specialized requests  
+- **Session Persistence** via storage state
+- **Robots.txt Compliance**
+- **Anti-Bot Features** (Stealth Mode & Undetected Browser)
+
+With these power tools, you can build robust scraping workflows that mimic real user behavior, handle secure sites, capture detailed snapshots, manage sessions across multiple runs, and bypass bot detection—streamlining your entire data collection pipeline.
+
+**Note**: In future versions, we may enable stealth mode and undetected browser by default. For now, users should explicitly enable these features when needed.
+
+**Last Updated**: 2025-01-17
\ No newline at end of file
diff --git a/docs/md_v2/advanced/anti-bot-and-fallback.md b/docs/md_v2/advanced/anti-bot-and-fallback.md
new file mode 100644
index 0000000..68ed03d
--- /dev/null
+++ b/docs/md_v2/advanced/anti-bot-and-fallback.md
@@ -0,0 +1,263 @@
+# Anti-Bot Detection & Fallback
+
+When crawling sites protected by anti-bot systems (Akamai, Cloudflare, PerimeterX, DataDome, Imperva, etc.), requests often get blocked with CAPTCHAs, 403 responses, or empty pages. Crawl4AI provides a layered retry and fallback system that automatically detects blocking and escalates through multiple strategies until content is retrieved.
+
+## How Detection Works
+
+After each crawl attempt, Crawl4AI inspects the HTTP status code and HTML content for known anti-bot signals:
+
+- **HTTP 403/429** with short or empty response bodies
+- **Challenge pages** — Cloudflare "Just a moment", Akamai "Access Denied", PerimeterX block pages
+- **CAPTCHA injection** — reCAPTCHA, hCaptcha, or vendor-specific challenges on otherwise empty pages
+- **Firewall blocks** — Imperva/Incapsula resource iframes, Sucuri firewall pages, Cloudflare error codes
+
+Detection uses structural HTML markers (specific element IDs, script sources, form actions) rather than generic keywords to minimize false positives. A normal page that happens to mention "CAPTCHA" or "Cloudflare" in its content will not be flagged.
+
+When all attempts fail and blocking is still detected, the result is returned with `success=False` and `error_message` describing the block reason.
+
+## Configuration Options
+
+All anti-bot retry options live on `CrawlerRunConfig`:
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `proxy_config` | `ProxyConfig`, `list[ProxyConfig]`, or `None` | `None` | Single proxy or ordered list of proxies to try. Each retry round iterates through the full list. Use `"direct"` or `ProxyConfig.DIRECT` in a list to explicitly try without a proxy. |
+| `max_retries` | `int` | `0` | Number of retry rounds when blocking is detected. `0` = no retries. |
+| `fallback_fetch_function` | `async (str) -> str` | `None` | Async function called as last resort. Takes URL, returns raw HTML. |
+
+## Escalation Chain
+
+Each retry round tries every proxy in `proxy_config` in order. If all rounds are exhausted and the page is still blocked, the fallback fetch function is called as a last resort.
+
+```
+For each round (1 + max_retries rounds):
+    1. Try proxy_config[0] (or direct if proxy_config is None)
+    2. If blocked → try proxy_config[1]
+    3. If blocked → try proxy_config[2]
+    4. ... continue through all proxies
+    5. If any attempt succeeds → done
+
+If all rounds exhausted and still blocked:
+    6. Call fallback_fetch_function(url) → process returned HTML
+```
+
+Worst-case attempts before the fetch function: `(1 + max_retries) x len(proxy_config)`
+
+## Crawl Stats
+
+Every crawl result includes a `crawl_stats` dict with detailed attempt tracking:
+
+```python
+result.crawl_stats = {
+    "attempts": 3,                    # total browser attempts made
+    "retries": 1,                     # retry rounds used (0 = succeeded first round)
+    "proxies_used": [                 # ordered list of every attempt
+        {"proxy": None,               "status_code": 403, "blocked": True,  "reason": "Akamai block (Reference #)"},
+        {"proxy": "proxy.io:8080",    "status_code": 403, "blocked": True,  "reason": "Akamai block (Reference #)"},
+        {"proxy": "premium.io:9090",  "status_code": 200, "blocked": False, "reason": ""},
+    ],
+    "fallback_fetch_used": False,     # whether fallback_fetch_function was called
+    "resolved_by": "proxy",           # "direct" | "proxy" | "fallback_fetch" | null (all failed)
+}
+```
+
+## Usage Examples
+
+### Simple Retry (No Proxy)
+
+Retry the crawl up to 3 times when blocking is detected. Useful when blocks are intermittent or IP-based.
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+    result = await crawler.arun(
+        url="https://example.com",
+        config=CrawlerRunConfig(max_retries=3),
+    )
+```
+
+### Single Proxy
+
+Pass a single `ProxyConfig` — it's used on every attempt. Same behavior as always.
+
+```python
+from crawl4ai.async_configs import ProxyConfig
+
+config = CrawlerRunConfig(
+    max_retries=2,
+    proxy_config=ProxyConfig(
+        server="http://proxy.example.com:8080",
+        username="user",
+        password="pass",
+    ),
+)
+```
+
+### Direct-First, Then Proxies
+
+Try without a proxy first, then escalate to proxies if blocked. Use `ProxyConfig.DIRECT` (or the string `"direct"`) in the list to represent a no-proxy attempt.
+
+```python
+config = CrawlerRunConfig(
+    max_retries=1,
+    proxy_config=[
+        ProxyConfig.DIRECT,  # Try without proxy first
+        ProxyConfig(
+            server="http://datacenter-proxy.example.com:8080",
+            username="user",
+            password="pass",
+        ),
+        ProxyConfig(
+            server="http://residential-proxy.example.com:9090",
+            username="user",
+            password="pass",
+        ),
+    ],
+)
+```
+
+With this setup, each round tries direct first, then datacenter, then residential. With `max_retries=1`, worst case is 2 rounds x 3 steps = 6 attempts.
+
+### Proxy List (Escalation)
+
+Pass a list of proxies. They're tried in order — first one that works wins. Within each retry round, the entire list is tried again.
+
+```python
+config = CrawlerRunConfig(
+    max_retries=1,
+    proxy_config=[
+        ProxyConfig(
+            server="http://datacenter-proxy.example.com:8080",
+            username="user",
+            password="pass",
+        ),
+        ProxyConfig(
+            server="http://residential-proxy.example.com:9090",
+            username="user",
+            password="pass",
+        ),
+    ],
+)
+```
+
+With this setup, each round tries the datacenter proxy first, then the residential proxy. With `max_retries=1`, worst case is 2 rounds x 2 proxies = 4 attempts.
+
+### Fallback Fetch Function
+
+When all browser-based attempts fail, call a custom async function as a last resort. This function receives the URL and must return raw HTML as a string. The returned HTML is processed through the normal pipeline (markdown generation, extraction, etc.).
+
+This is useful when you have access to a scraping API, a pre-fetched cache, or any other source of HTML.
+
+```python
+import aiohttp
+
+async def my_scraping_api(url: str) -> str:
+    """Fetch HTML via an external scraping API."""
+    async with aiohttp.ClientSession() as session:
+        async with session.get(
+            "https://api.my-scraping-service.com/fetch",
+            params={"url": url, "format": "html"},
+            headers={"Authorization": "Bearer MY_TOKEN"},
+        ) as resp:
+            if resp.status == 200:
+                return await resp.text()
+            raise RuntimeError(f"API error: {resp.status}")
+
+config = CrawlerRunConfig(
+    max_retries=1,
+    fallback_fetch_function=my_scraping_api,
+)
+```
+
+The function can do anything — call an API, read from a database, return cached HTML, or make a simple HTTP request with a different library. Crawl4AI does not care how the HTML is obtained.
+
+### Full Escalation (All Features Combined)
+
+This example combines every layer: stealth mode, a list of proxies tried in order, retries, and a final fetch function.
+
+```python
+import aiohttp
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, ProxyConfig
+
+# Last-resort: fetch HTML via an external service
+async def external_fetch(url: str) -> str:
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            "https://api.my-service.com/scrape",
+            json={"url": url, "render_js": True},
+            headers={"Authorization": "Bearer MY_TOKEN"},
+        ) as resp:
+            return await resp.text()
+
+browser_config = BrowserConfig(
+    headless=True,
+    enable_stealth=True,
+)
+
+crawl_config = CrawlerRunConfig(
+    magic=True,
+    wait_until="load",
+    max_retries=2,
+
+    # Proxies tried in order — cheapest first
+    proxy_config=[
+        ProxyConfig(
+            server="http://datacenter-proxy.example.com:8080",
+            username="user",
+            password="pass",
+        ),
+        ProxyConfig(
+            server="http://residential-proxy.example.com:9090",
+            username="user",
+            password="pass",
+        ),
+    ],
+
+    # Last resort — called after all retries and proxies are exhausted
+    fallback_fetch_function=external_fetch,
+)
+
+async with AsyncWebCrawler(config=browser_config) as crawler:
+    result = await crawler.arun(
+        url="https://protected-site.com/products",
+        config=crawl_config,
+    )
+
+    if result.success:
+        print(f"Got {len(result.markdown.raw_markdown)} chars of markdown")
+        print(f"Resolved by: {result.crawl_stats['resolved_by']}")
+        print(f"Attempts: {result.crawl_stats['attempts']}")
+    else:
+        print(f"All attempts failed: {result.error_message}")
+```
+
+**What happens step by step:**
+
+| Round | Attempt | What runs |
+|---|---|---|
+| 1 | 1 | Datacenter proxy — blocked |
+| 1 | 2 | Residential proxy — blocked |
+| 2 | 1 | Datacenter proxy — blocked |
+| 2 | 2 | Residential proxy — blocked |
+| 3 | 1 | Datacenter proxy — blocked |
+| 3 | 2 | Residential proxy — blocked |
+| - | - | `external_fetch(url)` called — returns HTML |
+
+That's up to 6 browser attempts + 1 function call before giving up.
+
+## Tips
+
+- **Start with `max_retries=0`** and a `fallback_fetch_function` if you just want a safety net without burning time on retries.
+- **Order proxies cheapest-first** — datacenter proxies before residential, residential before premium.
+- **Combine with stealth mode** — `BrowserConfig(enable_stealth=True)` and `CrawlerRunConfig(magic=True)` reduce the chance of being blocked in the first place.
+- **`wait_until="load"`** is important for anti-bot sites — the default `domcontentloaded` can return before the anti-bot sensor finishes.
+- **Check `crawl_stats`** to understand what happened — how many attempts, which proxy worked, whether the fallback function was needed.
+
+## See Also
+
+- [Proxy & Security](proxy-security.md) — Proxy setup, authentication, and rotation
+- [Undetected Browser](undetected-browser.md) — Stealth mode and browser fingerprint evasion
+- [Session Management](session-management.md) — Maintaining sessions across requests
diff --git a/docs/md_v2/advanced/crawl-dispatcher.md b/docs/md_v2/advanced/crawl-dispatcher.md
new file mode 100644
index 0000000..e4059f2
--- /dev/null
+++ b/docs/md_v2/advanced/crawl-dispatcher.md
@@ -0,0 +1,12 @@
+# Crawl Dispatcher
+
+We’re excited to announce a **Crawl Dispatcher** module that can handle **thousands** of crawling tasks simultaneously. By efficiently managing system resources (memory, CPU, network), this dispatcher ensures high-performance data extraction at scale. It also provides **real-time monitoring** of each crawler’s status, memory usage, and overall progress.
+
+Stay tuned—this feature is **coming soon** in an upcoming release of Crawl4AI! For the latest news, keep an eye on our changelogs and follow [@unclecode](https://twitter.com/unclecode) on X.
+
+Below is a **sample** of how the dispatcher’s performance monitor might look in action:
+
+![Crawl Dispatcher Performance Monitor](../assets/images/dispatcher.png)
+
+
+We can’t wait to bring you this streamlined, **scalable** approach to multi-URL crawling—**watch this space** for updates!
\ No newline at end of file
diff --git a/docs/md_v2/advanced/file-downloading.md b/docs/md_v2/advanced/file-downloading.md
new file mode 100644
index 0000000..2fa3759
--- /dev/null
+++ b/docs/md_v2/advanced/file-downloading.md
@@ -0,0 +1,118 @@
+# Download Handling in Crawl4AI
+
+This guide explains how to use Crawl4AI to handle file downloads during crawling. You'll learn how to trigger downloads, specify download locations, and access downloaded files.
+
+## Enabling Downloads
+
+To enable downloads, set the `accept_downloads` parameter in the `BrowserConfig` object and pass it to the crawler.
+
+```python
+from crawl4ai.async_configs import BrowserConfig, AsyncWebCrawler
+
+async def main():
+    config = BrowserConfig(accept_downloads=True)  # Enable downloads globally
+    async with AsyncWebCrawler(config=config) as crawler:
+        # ... your crawling logic ...
+
+asyncio.run(main())
+```
+
+## Specifying Download Location
+
+Specify the download directory using the `downloads_path` attribute in the `BrowserConfig` object. If not provided, Crawl4AI defaults to creating a "downloads" directory inside the `.crawl4ai` folder in your home directory.
+
+```python
+from crawl4ai.async_configs import BrowserConfig
+import os
+
+downloads_path = os.path.join(os.getcwd(), "my_downloads")  # Custom download path
+os.makedirs(downloads_path, exist_ok=True)
+
+config = BrowserConfig(accept_downloads=True, downloads_path=downloads_path)
+
+async def main():
+    async with AsyncWebCrawler(config=config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        # ...
+```
+
+## Triggering Downloads
+
+Downloads are typically triggered by user interactions on a web page, such as clicking a download button. Use `js_code` in `CrawlerRunConfig` to simulate these actions and `wait_for` to allow sufficient time for downloads to start.
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+
+config = CrawlerRunConfig(
+    js_code="""
+        const downloadLink = document.querySelector('a[href$=".exe"]');
+        if (downloadLink) {
+            downloadLink.click();
+        }
+    """,
+    wait_for=5  # Wait 5 seconds for the download to start
+)
+
+result = await crawler.arun(url="https://www.python.org/downloads/", config=config)
+```
+
+## Accessing Downloaded Files
+
+The `downloaded_files` attribute of the `CrawlResult` object contains paths to downloaded files.
+
+```python
+if result.downloaded_files:
+    print("Downloaded files:")
+    for file_path in result.downloaded_files:
+        print(f"- {file_path}")
+        file_size = os.path.getsize(file_path)
+        print(f"- File size: {file_size} bytes")
+else:
+    print("No files downloaded.")
+```
+
+## Example: Downloading Multiple Files
+
+```python
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+import os
+from pathlib import Path
+
+async def download_multiple_files(url: str, download_path: str):
+    config = BrowserConfig(accept_downloads=True, downloads_path=download_path)
+    async with AsyncWebCrawler(config=config) as crawler:
+        run_config = CrawlerRunConfig(
+            js_code="""
+                const downloadLinks = document.querySelectorAll('a[download]');
+                for (const link of downloadLinks) {
+                    link.click();
+                    // Delay between clicks
+                    await new Promise(r => setTimeout(r, 2000));  
+                }
+            """,
+            wait_for=10  # Wait for all downloads to start
+        )
+        result = await crawler.arun(url=url, config=run_config)
+
+        if result.downloaded_files:
+            print("Downloaded files:")
+            for file in result.downloaded_files:
+                print(f"- {file}")
+        else:
+            print("No files downloaded.")
+
+# Usage
+download_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
+os.makedirs(download_path, exist_ok=True)
+
+asyncio.run(download_multiple_files("https://www.python.org/downloads/windows/", download_path))
+```
+
+## Important Considerations
+
+- **Browser Context:** Downloads are managed within the browser context. Ensure `js_code` correctly targets the download triggers on the webpage.
+- **Timing:** Use `wait_for` in `CrawlerRunConfig` to manage download timing.
+- **Error Handling:** Handle errors to manage failed downloads or incorrect paths gracefully.
+- **Security:** Scan downloaded files for potential security threats before use.
+
+This revised guide ensures consistency with the `Crawl4AI` codebase by using `BrowserConfig` and `CrawlerRunConfig` for all download-related configurations. Let me know if further adjustments are needed!
\ No newline at end of file
diff --git a/docs/md_v2/advanced/hooks-auth.md b/docs/md_v2/advanced/hooks-auth.md
new file mode 100644
index 0000000..6787abd
--- /dev/null
+++ b/docs/md_v2/advanced/hooks-auth.md
@@ -0,0 +1,254 @@
+# Hooks & Auth in AsyncWebCrawler
+
+Crawl4AI’s **hooks** let you customize the crawler at specific points in the pipeline:
+
+1. **`on_browser_created`** – After browser creation.  
+2. **`on_page_context_created`** – After a new context & page are created.  
+3. **`before_goto`** – Just before navigating to a page.  
+4. **`after_goto`** – Right after navigation completes.  
+5. **`on_user_agent_updated`** – Whenever the user agent changes.  
+6. **`on_execution_started`** – Once custom JavaScript execution begins.  
+7. **`before_retrieve_html`** – Just before the crawler retrieves final HTML.  
+8. **`before_return_html`** – Right before returning the HTML content.
+
+**Important**: Avoid heavy tasks in `on_browser_created` since you don’t yet have a page context. If you need to *log in*, do so in **`on_page_context_created`**.
+
+> note "Important Hook Usage Warning"
+    **Avoid Misusing Hooks**: Do not manipulate page objects in the wrong hook or at the wrong time, as it can crash the pipeline or produce incorrect results. A common mistake is attempting to handle authentication prematurely—such as creating or closing pages in `on_browser_created`. 
+
+>   **Use the Right Hook for Auth**: If you need to log in or set tokens, use `on_page_context_created`. This ensures you have a valid page/context to work with, without disrupting the main crawling flow.
+
+>    **Identity-Based Crawling**: For robust auth, consider identity-based crawling (or passing a session ID) to preserve state. Run your initial login steps in a separate, well-defined process, then feed that session to your main crawl—rather than shoehorning complex authentication into early hooks. Check out [Identity-Based Crawling](../advanced/identity-based-crawling.md) for more details.
+
+>    **Be Cautious**: Overwriting or removing elements in the wrong hook can compromise the final crawl. Keep hooks focused on smaller tasks (like route filters, custom headers), and let your main logic (crawling, data extraction) proceed normally.
+
+
+Below is an example demonstration.
+
+---
+
+## Example: Using Hooks in AsyncWebCrawler
+
+```python
+import asyncio
+import json
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from playwright.async_api import Page, BrowserContext
+
+async def main():
+    print("🔗 Hooks Example: Demonstrating recommended usage")
+
+    # 1) Configure the browser
+    browser_config = BrowserConfig(
+        headless=True,
+        verbose=True
+    )
+
+    # 2) Configure the crawler run
+    crawler_run_config = CrawlerRunConfig(
+        js_code="window.scrollTo(0, document.body.scrollHeight);",
+        wait_for="body",
+        cache_mode=CacheMode.BYPASS
+    )
+
+    # 3) Create the crawler instance
+    crawler = AsyncWebCrawler(config=browser_config)
+
+    #
+    # Define Hook Functions
+    #
+
+    async def on_browser_created(browser, **kwargs):
+        # Called once the browser instance is created (but no pages or contexts yet)
+        print("[HOOK] on_browser_created - Browser created successfully!")
+        # Typically, do minimal setup here if needed
+        return browser
+
+    async def on_page_context_created(page: Page, context: BrowserContext, **kwargs):
+        # Called right after a new page + context are created (ideal for auth or route config).
+        print("[HOOK] on_page_context_created - Setting up page & context.")
+        
+        # Example 1: Route filtering (e.g., block images)
+        async def route_filter(route):
+            if route.request.resource_type == "image":
+                print(f"[HOOK] Blocking image request: {route.request.url}")
+                await route.abort()
+            else:
+                await route.continue_()
+
+        await context.route("**", route_filter)
+
+        # Example 2: (Optional) Simulate a login scenario
+        # (We do NOT create or close pages here, just do quick steps if needed)
+        # e.g., await page.goto("https://example.com/login")
+        # e.g., await page.fill("input[name='username']", "testuser")
+        # e.g., await page.fill("input[name='password']", "password123")
+        # e.g., await page.click("button[type='submit']")
+        # e.g., await page.wait_for_selector("#welcome")
+        # e.g., await context.add_cookies([...])
+        # Then continue
+
+        # Example 3: Adjust the viewport
+        await page.set_viewport_size({"width": 1080, "height": 600})
+        return page
+
+    async def before_goto(
+        page: Page, context: BrowserContext, url: str, **kwargs
+    ):
+        # Called before navigating to each URL.
+        print(f"[HOOK] before_goto - About to navigate: {url}")
+        # e.g., inject custom headers
+        await page.set_extra_http_headers({
+            "Custom-Header": "my-value"
+        })
+        return page
+
+    async def after_goto(
+        page: Page, context: BrowserContext, 
+        url: str, response, **kwargs
+    ):
+        # Called after navigation completes.
+        print(f"[HOOK] after_goto - Successfully loaded: {url}")
+        # e.g., wait for a certain element if we want to verify
+        try:
+            await page.wait_for_selector('.content', timeout=1000)
+            print("[HOOK] Found .content element!")
+        except:
+            print("[HOOK] .content not found, continuing anyway.")
+        return page
+
+    async def on_user_agent_updated(
+        page: Page, context: BrowserContext, 
+        user_agent: str, **kwargs
+    ):
+        # Called whenever the user agent updates.
+        print(f"[HOOK] on_user_agent_updated - New user agent: {user_agent}")
+        return page
+
+    async def on_execution_started(page: Page, context: BrowserContext, **kwargs):
+        # Called after custom JavaScript execution begins.
+        print("[HOOK] on_execution_started - JS code is running!")
+        return page
+
+    async def before_retrieve_html(page: Page, context: BrowserContext, **kwargs):
+        # Called before final HTML retrieval.
+        print("[HOOK] before_retrieve_html - We can do final actions")
+        # Example: Scroll again
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+        return page
+
+    async def before_return_html(
+        page: Page, context: BrowserContext, html: str, **kwargs
+    ):
+        # Called just before returning the HTML in the result.
+        print(f"[HOOK] before_return_html - HTML length: {len(html)}")
+        return page
+
+    #
+    # Attach Hooks
+    #
+
+    crawler.crawler_strategy.set_hook("on_browser_created", on_browser_created)
+    crawler.crawler_strategy.set_hook(
+        "on_page_context_created", on_page_context_created
+    )
+    crawler.crawler_strategy.set_hook("before_goto", before_goto)
+    crawler.crawler_strategy.set_hook("after_goto", after_goto)
+    crawler.crawler_strategy.set_hook(
+        "on_user_agent_updated", on_user_agent_updated
+    )
+    crawler.crawler_strategy.set_hook(
+        "on_execution_started", on_execution_started
+    )
+    crawler.crawler_strategy.set_hook(
+        "before_retrieve_html", before_retrieve_html
+    )
+    crawler.crawler_strategy.set_hook(
+        "before_return_html", before_return_html
+    )
+
+    await crawler.start()
+
+    # 4) Run the crawler on an example page
+    url = "https://example.com"
+    result = await crawler.arun(url, config=crawler_run_config)
+    
+    if result.success:
+        print("\nCrawled URL:", result.url)
+        print("HTML length:", len(result.html))
+    else:
+        print("Error:", result.error_message)
+
+    await crawler.close()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+## Hook Lifecycle Summary
+
+1. **`on_browser_created`**:  
+   - Browser is up, but **no** pages or contexts yet.  
+   - Light setup only—don’t try to open or close pages here (that belongs in `on_page_context_created`).
+
+2. **`on_page_context_created`**:  
+   - Perfect for advanced **auth** or route blocking.  
+   - You have a **page** + **context** ready but haven’t navigated to the target URL yet.
+
+3. **`before_goto`**:  
+   - Right before navigation. Typically used for setting **custom headers** or logging the target URL.
+
+4. **`after_goto`**:  
+   - After page navigation is done. Good place for verifying content or waiting on essential elements. 
+
+5. **`on_user_agent_updated`**:  
+   - Whenever the user agent changes (for stealth or different UA modes).
+
+6. **`on_execution_started`**:  
+   - If you set `js_code` or run custom scripts, this runs once your JS is about to start.
+
+7. **`before_retrieve_html`**:  
+   - Just before the final HTML snapshot is taken. Often you do a final scroll or lazy-load triggers here.
+
+8. **`before_return_html`**:  
+   - The last hook before returning HTML to the `CrawlResult`. Good for logging HTML length or minor modifications.
+
+---
+
+## When to Handle Authentication
+
+**Recommended**: Use **`on_page_context_created`** if you need to:
+
+- Navigate to a login page or fill forms
+- Set cookies or localStorage tokens
+- Block resource routes to avoid ads
+
+This ensures the newly created context is under your control **before** `arun()` navigates to the main URL.
+
+---
+
+## Additional Considerations
+
+- **Session Management**: If you want multiple `arun()` calls to reuse a single session, pass `session_id=` in your `CrawlerRunConfig`. Hooks remain the same.  
+- **Performance**: Hooks can slow down crawling if they do heavy tasks. Keep them concise.  
+- **Error Handling**: If a hook fails, the overall crawl might fail. Catch exceptions or handle them gracefully.  
+- **Concurrency**: If you run `arun_many()`, each URL triggers these hooks in parallel. Ensure your hooks are thread/async-safe.
+
+---
+
+## Conclusion
+
+Hooks provide **fine-grained** control over:
+
+- **Browser** creation (light tasks only)
+- **Page** and **context** creation (auth, route blocking)
+- **Navigation** phases
+- **Final HTML** retrieval
+
+Follow the recommended usage:
+- **Login** or advanced tasks in `on_page_context_created`  
+- **Custom headers** or logs in `before_goto` / `after_goto`  
+- **Scrolling** or final checks in `before_retrieve_html` / `before_return_html`
+
diff --git a/docs/md_v2/advanced/identity-based-crawling.md b/docs/md_v2/advanced/identity-based-crawling.md
new file mode 100644
index 0000000..2b15585
--- /dev/null
+++ b/docs/md_v2/advanced/identity-based-crawling.md
@@ -0,0 +1,413 @@
+# Preserve Your Identity with Crawl4AI
+
+Crawl4AI empowers you to navigate and interact with the web using your **authentic digital identity**, ensuring you’re recognized as a human and not mistaken for a bot. This tutorial covers:
+
+1. **Managed Browsers** – The recommended approach for persistent profiles and identity-based crawling.  
+2. **Magic Mode** – A simplified fallback solution for quick automation without persistent identity.
+
+---
+
+## 1. Managed Browsers: Your Digital Identity Solution
+
+**Managed Browsers** let developers create and use **persistent browser profiles**. These profiles store local storage, cookies, and other session data, letting you browse as your **real self**—complete with logins, preferences, and cookies.
+
+### Key Benefits
+
+- **Authentic Browsing Experience**: Retain session data and browser fingerprints as though you’re a normal user.  
+- **Effortless Configuration**: Once you log in or solve CAPTCHAs in your chosen data directory, you can re-run crawls without repeating those steps.  
+- **Empowered Data Access**: If you can see the data in your own browser, you can automate its retrieval with your genuine identity.
+
+---
+
+Below is a **partial update** to your **Managed Browsers** tutorial, specifically the section about **creating a user-data directory** using **Playwright’s Chromium** binary rather than a system-wide Chrome/Edge. We’ll show how to **locate** that binary and launch it with a `--user-data-dir` argument to set up your profile. You can then point `BrowserConfig.user_data_dir` to that folder for subsequent crawls.
+
+---
+
+### Creating a User Data Directory (Command-Line Approach via Playwright)
+
+If you installed Crawl4AI (which installs Playwright under the hood), you already have a Playwright-managed Chromium on your system. Follow these steps to launch that **Chromium** from your command line, specifying a **custom** data directory:
+
+1. **Find** the Playwright Chromium binary:
+   - On most systems, installed browsers go under a `~/.cache/ms-playwright/` folder or similar path.  
+   - To see an overview of installed browsers, run:
+     ```bash
+     python -m playwright install --dry-run
+     ```
+     or
+     ```bash
+     playwright install --dry-run
+     ```
+     (depending on your environment). This shows where Playwright keeps Chromium.
+
+   - For instance, you might see a path like:
+     ```
+     ~/.cache/ms-playwright/chromium-1234/chrome-linux/chrome
+     ```
+     on Linux, or a corresponding folder on macOS/Windows.
+
+2. **Launch** the Playwright Chromium binary with a **custom** user-data directory:
+   ```bash
+   # Linux example
+   ~/.cache/ms-playwright/chromium-1234/chrome-linux/chrome \
+       --user-data-dir=/home//my_chrome_profile
+   ```
+   ```bash
+   # macOS example (Playwright’s internal binary)
+   ~/Library/Caches/ms-playwright/chromium-1234/chrome-mac/Chromium.app/Contents/MacOS/Chromium \
+       --user-data-dir=/Users//my_chrome_profile
+   ```
+   ```powershell
+   # Windows example (PowerShell/cmd)
+   "C:\Users\\AppData\Local\ms-playwright\chromium-1234\chrome-win\chrome.exe" ^
+       --user-data-dir="C:\Users\\my_chrome_profile"
+   ```
+   
+   **Replace** the path with the actual subfolder indicated in your `ms-playwright` cache structure.  
+   - This **opens** a fresh Chromium with your new or existing data folder.  
+   - **Log into** any sites or configure your browser the way you want.  
+   - **Close** when done—your profile data is saved in that folder.
+
+3. **Use** that folder in **`BrowserConfig.user_data_dir`**:
+   ```python
+   from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+   browser_config = BrowserConfig(
+       headless=True,
+       use_managed_browser=True,
+       user_data_dir="/home//my_chrome_profile",
+       browser_type="chromium"
+   )
+   ```
+   - Next time you run your code, it reuses that folder—**preserving** your session data, cookies, local storage, etc.
+
+---
+
+### Creating a Profile Using the Crawl4AI CLI (Easiest)
+
+If you prefer a guided, interactive setup, use the built-in CLI to create and manage persistent browser profiles.
+
+1.⠀Launch the profile manager:
+   ```bash
+   crwl profiles
+   ```
+
+2.⠀Choose "Create new profile" and enter a profile name. A Chromium window opens so you can log in to sites and configure settings. When finished, return to the terminal and press `q` to save the profile.
+
+3.⠀Profiles are saved under `~/.crawl4ai/profiles/` (for example: `/home//.crawl4ai/profiles/test_profile_1`) along with a `storage_state.json` for cookies and session data.
+
+4.⠀Optionally, choose "List profiles" in the CLI to view available profiles and their paths.
+
+5.⠀Use the saved path with `BrowserConfig.user_data_dir`:
+   ```python
+   from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+   profile_path = "/home//.crawl4ai/profiles/test_profile_1"
+
+   browser_config = BrowserConfig(
+       headless=True,
+       use_managed_browser=True,
+       user_data_dir=profile_path,
+       browser_type="chromium",
+   )
+
+   async with AsyncWebCrawler(config=browser_config) as crawler:
+       result = await crawler.arun(url="https://example.com/private")
+   ```
+
+The CLI also supports listing and deleting profiles, and even testing a crawl directly from the menu.
+
+---
+
+## 3. Using Managed Browsers in Crawl4AI
+
+Once you have a data directory with your session data, pass it to **`BrowserConfig`**:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def main():
+    # 1) Reference your persistent data directory
+    browser_config = BrowserConfig(
+        headless=True,             # 'True' for automated runs
+        verbose=True,
+        use_managed_browser=True,  # Enables persistent browser strategy
+        browser_type="chromium",
+        user_data_dir="/path/to/my-chrome-profile"
+    )
+
+    # 2) Standard crawl config
+    crawl_config = CrawlerRunConfig(
+        wait_for="css:.logged-in-content"
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com/private", config=crawl_config)
+        if result.success:
+            print("Successfully accessed private data with your identity!")
+        else:
+            print("Error:", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### Workflow
+
+1. **Login** externally (via CLI or your normal Chrome with `--user-data-dir=...`).  
+2. **Close** that browser.  
+3. **Use** the same folder in `user_data_dir=` in Crawl4AI.  
+4. **Crawl** – The site sees your identity as if you’re the same user who just logged in.
+
+---
+
+## 4. Magic Mode: Simplified Automation
+
+If you **don’t** need a persistent profile or identity-based approach, **Magic Mode** offers a quick way to simulate human-like browsing without storing long-term data.
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://example.com",
+        config=CrawlerRunConfig(
+            magic=True,  # Simplifies a lot of interaction
+            remove_overlay_elements=True,
+            page_timeout=60000
+        )
+    )
+```
+
+**Magic Mode**:
+
+- Simulates a user-like experience  
+- Randomizes user agent & navigator
+- Randomizes interactions & timings  
+- Masks automation signals  
+- Attempts pop-up handling  
+
+**But** it’s no substitute for **true** user-based sessions if you want a fully legitimate identity-based solution.
+
+---
+
+## 5. Comparing Managed Browsers vs. Magic Mode
+
+| Feature                    | **Managed Browsers**                                           | **Magic Mode**                                     |
+|----------------------------|---------------------------------------------------------------|-----------------------------------------------------|
+| **Session Persistence**    | Full localStorage/cookies retained in user_data_dir           | No persistent data (fresh each run)                |
+| **Genuine Identity**       | Real user profile with full rights & preferences              | Emulated user-like patterns, but no actual identity |
+| **Complex Sites**          | Best for login-gated sites or heavy config                    | Simple tasks, minimal login or config needed        |
+| **Setup**                  | External creation of user_data_dir, then use in Crawl4AI       | Single-line approach (`magic=True`)                 |
+| **Reliability**            | Extremely consistent (same data across runs)                  | Good for smaller tasks, can be less stable          |
+
+---
+
+## 6. Using the BrowserProfiler Class
+
+Crawl4AI provides a dedicated `BrowserProfiler` class for managing browser profiles, making it easy to create, list, and delete profiles for identity-based browsing.
+
+### Creating and Managing Profiles with BrowserProfiler
+
+The `BrowserProfiler` class offers a comprehensive API for browser profile management:
+
+```python
+import asyncio
+from crawl4ai import BrowserProfiler
+
+async def manage_profiles():
+    # Create a profiler instance
+    profiler = BrowserProfiler()
+    
+    # Create a profile interactively - opens a browser window
+    profile_path = await profiler.create_profile(
+        profile_name="my-login-profile"  # Optional: name your profile
+    )
+    
+    print(f"Profile saved at: {profile_path}")
+    
+    # List all available profiles
+    profiles = profiler.list_profiles()
+    
+    for profile in profiles:
+        print(f"Profile: {profile['name']}")
+        print(f"  Path: {profile['path']}")
+        print(f"  Created: {profile['created']}")
+        print(f"  Browser type: {profile['type']}")
+    
+    # Get a specific profile path by name
+    specific_profile = profiler.get_profile_path("my-login-profile")
+    
+    # Delete a profile when no longer needed
+    success = profiler.delete_profile("old-profile-name")
+    
+asyncio.run(manage_profiles())
+```
+
+**How profile creation works:**
+1. A browser window opens for you to interact with
+2. You log in to websites, set preferences, etc.
+3. When you're done, press 'q' in the terminal to close the browser
+4. The profile is saved in the Crawl4AI profiles directory
+5. You can use the returned path with `BrowserConfig.user_data_dir`
+
+### Interactive Profile Management
+
+The `BrowserProfiler` also offers an interactive management console that guides you through profile creation, listing, and deletion:
+
+```python
+import asyncio
+from crawl4ai import BrowserProfiler, AsyncWebCrawler, BrowserConfig
+
+# Define a function to use a profile for crawling
+async def crawl_with_profile(profile_path, url):
+    browser_config = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        user_data_dir=profile_path
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url)
+        return result
+
+async def main():
+    # Create a profiler instance
+    profiler = BrowserProfiler()
+    
+    # Launch the interactive profile manager
+    # Passing the crawl function as a callback adds a "crawl with profile" option
+    await profiler.interactive_manager(crawl_callback=crawl_with_profile)
+    
+asyncio.run(main())
+```
+
+### Legacy Methods
+
+For backward compatibility, the previous methods on `ManagedBrowser` are still available, but they delegate to the new `BrowserProfiler` class:
+
+```python
+from crawl4ai.browser_manager import ManagedBrowser
+
+# These methods still work but use BrowserProfiler internally
+profiles = ManagedBrowser.list_profiles()
+```
+
+### Complete Example
+
+See the full example in `docs/examples/identity_based_browsing.py` for a complete demonstration of creating and using profiles for authenticated browsing using the new `BrowserProfiler` class.
+
+---
+
+## 7. Locale, Timezone, and Geolocation Control
+
+In addition to using persistent profiles, Crawl4AI supports customizing your browser's locale, timezone, and geolocation settings. These features enhance your identity-based browsing experience by allowing you to control how websites perceive your location and regional settings.
+
+### Setting Locale and Timezone
+
+You can set the browser's locale and timezone through `CrawlerRunConfig`:
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://example.com",
+        config=CrawlerRunConfig(
+            # Set browser locale (language and region formatting)
+            locale="fr-FR",  # French (France)
+            
+            # Set browser timezone
+            timezone_id="Europe/Paris",
+            
+            # Other normal options...
+            magic=True,
+            page_timeout=60000
+        )
+    )
+```
+
+**How it works:**
+- `locale` affects language preferences, date formats, number formats, etc.
+- `timezone_id` affects JavaScript's Date object and time-related functionality
+- These settings are applied when creating the browser context and maintained throughout the session
+
+### Configuring Geolocation
+
+Control the GPS coordinates reported by the browser's geolocation API:
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, GeolocationConfig
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://maps.google.com",  # Or any location-aware site
+        config=CrawlerRunConfig(
+            # Configure precise GPS coordinates
+            geolocation=GeolocationConfig(
+                latitude=48.8566,   # Paris coordinates
+                longitude=2.3522,
+                accuracy=100        # Accuracy in meters (optional)
+            ),
+            
+            # This site will see you as being in Paris
+            page_timeout=60000
+        )
+    )
+```
+
+**Important notes:**
+- When `geolocation` is specified, the browser is automatically granted permission to access location
+- Websites using the Geolocation API will receive the exact coordinates you specify
+- This affects map services, store locators, delivery services, etc.
+- Combined with the appropriate `locale` and `timezone_id`, you can create a fully consistent location profile
+
+### Combining with Managed Browsers
+
+These settings work perfectly with managed browsers for a complete identity solution:
+
+```python
+from crawl4ai import (
+    AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, 
+    GeolocationConfig
+)
+
+browser_config = BrowserConfig(
+    use_managed_browser=True,
+    user_data_dir="/path/to/my-profile",
+    browser_type="chromium"
+)
+
+crawl_config = CrawlerRunConfig(
+    # Location settings
+    locale="es-MX",                  # Spanish (Mexico)
+    timezone_id="America/Mexico_City",
+    geolocation=GeolocationConfig(
+        latitude=19.4326,            # Mexico City
+        longitude=-99.1332
+    )
+)
+
+async with AsyncWebCrawler(config=browser_config) as crawler:
+    result = await crawler.arun(url="https://example.com", config=crawl_config)
+```
+
+Combining persistent profiles with precise geolocation and region settings gives you complete control over your digital identity.
+
+## 8. Summary
+
+- **Create** your user-data directory either:
+  - By launching Chrome/Chromium externally with `--user-data-dir=/some/path` 
+  - Or by using the built-in `BrowserProfiler.create_profile()` method
+  - Or through the interactive interface with `profiler.interactive_manager()`
+- **Log in** or configure sites as needed, then close the browser
+- **Reference** that folder in `BrowserConfig(user_data_dir="...")` + `use_managed_browser=True`
+- **Customize** identity aspects with `locale`, `timezone_id`, and `geolocation`
+- **List and reuse** profiles with `BrowserProfiler.list_profiles()`
+- **Manage** your profiles with the dedicated `BrowserProfiler` class
+- Enjoy **persistent** sessions that reflect your real identity
+- If you only need quick, ephemeral automation, **Magic Mode** might suffice
+
+**Recommended**: Always prefer a **Managed Browser** for robust, identity-based crawling and simpler interactions with complex sites. Use **Magic Mode** for quick tasks or prototypes where persistent data is unnecessary.
+
+With these approaches, you preserve your **authentic** browsing environment, ensuring the site sees you exactly as a normal user—no repeated logins or wasted time.
\ No newline at end of file
diff --git a/docs/md_v2/advanced/lazy-loading.md b/docs/md_v2/advanced/lazy-loading.md
new file mode 100644
index 0000000..2db9531
--- /dev/null
+++ b/docs/md_v2/advanced/lazy-loading.md
@@ -0,0 +1,104 @@
+## Handling Lazy-Loaded Images
+
+Many websites now load images **lazily** as you scroll. If you need to ensure they appear in your final crawl (and in `result.media`), consider:
+
+1. **`wait_for_images=True`** – Wait for images to fully load.  
+2. **`scan_full_page`** – Force the crawler to scroll the entire page, triggering lazy loads.  
+3. **`scroll_delay`** – Add small delays between scroll steps.  
+
+**Note**: If the site requires multiple “Load More” triggers or complex interactions, see the [Page Interaction docs](../core/page-interaction.md). For sites with virtual scrolling (Twitter/Instagram style), see the [Virtual Scroll docs](virtual-scroll.md).
+
+### Example: Ensuring Lazy Images Appear
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+from crawl4ai.async_configs import CacheMode
+
+async def main():
+    config = CrawlerRunConfig(
+        # Force the crawler to wait until images are fully loaded
+        wait_for_images=True,
+
+        # Option 1: If you want to automatically scroll the page to load images
+        scan_full_page=True,  # Tells the crawler to try scrolling the entire page
+        scroll_delay=0.5,     # Delay (seconds) between scroll steps
+
+        # Option 2: If the site uses a 'Load More' or JS triggers for images,
+        # you can also specify js_code or wait_for logic here.
+
+        cache_mode=CacheMode.BYPASS,
+        verbose=True
+    )
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        result = await crawler.arun("https://www.example.com/gallery", config=config)
+        
+        if result.success:
+            images = result.media.get("images", [])
+            print("Images found:", len(images))
+            for i, img in enumerate(images[:5]):
+                print(f"[Image {i}] URL: {img['src']}, Score: {img.get('score','N/A')}")
+        else:
+            print("Error:", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Explanation**:
+
+- **`wait_for_images=True`**  
+  The crawler tries to ensure images have finished loading before finalizing the HTML.  
+- **`scan_full_page=True`**  
+  Tells the crawler to attempt scrolling from top to bottom. Each scroll step helps trigger lazy loading.  
+- **`scroll_delay=0.5`**  
+  Pause half a second between each scroll step. Helps the site load images before continuing.
+
+**When to Use**:
+
+- **Lazy-Loading**: If images appear only when the user scrolls into view, `scan_full_page` + `scroll_delay` helps the crawler see them.  
+- **Heavier Pages**: If a page is extremely long, be mindful that scanning the entire page can be slow. Adjust `scroll_delay` or the max scroll steps as needed.
+
+---
+
+## Combining with Other Link & Media Filters
+
+You can still combine **lazy-load** logic with the usual **exclude_external_images**, **exclude_domains**, or link filtration:
+
+```python
+config = CrawlerRunConfig(
+    wait_for_images=True,
+    scan_full_page=True,
+    scroll_delay=0.5,
+
+    # Filter out external images if you only want local ones
+    exclude_external_images=True,
+
+    # Exclude certain domains for links
+    exclude_domains=["spammycdn.com"],
+)
+```
+
+This approach ensures you see **all** images from the main domain while ignoring external ones, and the crawler physically scrolls the entire page so that lazy-loading triggers.
+
+---
+
+## Tips & Troubleshooting
+
+1. **Long Pages**  
+   - Setting `scan_full_page=True` on extremely long or infinite-scroll pages can be resource-intensive.  
+   - Consider using [hooks](../core/page-interaction.md) or specialized logic to load specific sections or “Load More” triggers repeatedly.
+
+2. **Mixed Image Behavior**  
+   - Some sites load images in batches as you scroll. If you’re missing images, increase your `scroll_delay` or call multiple partial scrolls in a loop with JS code or hooks.
+
+3. **Combining with Dynamic Wait**  
+   - If the site has a placeholder that only changes to a real image after a certain event, you might do `wait_for="css:img.loaded"` or a custom JS `wait_for`.
+
+4. **Caching**  
+   - If `cache_mode` is enabled, repeated crawls might skip some network fetches. If you suspect caching is missing new images, set `cache_mode=CacheMode.BYPASS` for fresh fetches.
+
+---
+
+With **lazy-loading** support, **wait_for_images**, and **scan_full_page** settings, you can capture the entire gallery or feed of images you expect—even if the site only loads them as the user scrolls. Combine these with the standard media filtering and domain exclusion for a complete link & media handling strategy.
\ No newline at end of file
diff --git a/docs/md_v2/advanced/multi-url-crawling.md b/docs/md_v2/advanced/multi-url-crawling.md
new file mode 100644
index 0000000..2c924ef
--- /dev/null
+++ b/docs/md_v2/advanced/multi-url-crawling.md
@@ -0,0 +1,604 @@
+# Advanced Multi-URL Crawling with Dispatchers
+
+> **Heads Up**: Crawl4AI supports advanced dispatchers for **parallel** or **throttled** crawling, providing dynamic rate limiting and memory usage checks. The built-in `arun_many()` function uses these dispatchers to handle concurrency efficiently.
+
+## 1. Introduction
+
+When crawling many URLs:
+
+- **Basic**: Use `arun()` in a loop (simple but less efficient)
+- **Better**: Use `arun_many()`, which efficiently handles multiple URLs with proper concurrency control
+- **Best**: Customize dispatcher behavior for your specific needs (memory management, rate limits, etc.)
+
+**Why Dispatchers?**  
+
+- **Adaptive**: Memory-based dispatchers can pause or slow down based on system resources
+- **Rate-limiting**: Built-in rate limiting with exponential backoff for 429/503 responses
+- **Real-time Monitoring**: Live dashboard of ongoing tasks, memory usage, and performance
+- **Flexibility**: Choose between memory-adaptive or semaphore-based concurrency
+
+---
+
+## 2. Core Components
+
+### 2.1 Rate Limiter
+
+```python
+class RateLimiter:
+    def __init__(
+        # Random delay range between requests
+        base_delay: Tuple[float, float] = (1.0, 3.0),  
+        
+        # Maximum backoff delay
+        max_delay: float = 60.0,                        
+        
+        # Retries before giving up
+        max_retries: int = 3,                          
+        
+        # Status codes triggering backoff
+        rate_limit_codes: List[int] = [429, 503]        
+    )
+```
+
+Here’s the revised and simplified explanation of the **RateLimiter**, focusing on constructor parameters and adhering to your markdown style and mkDocs guidelines.
+
+#### RateLimiter Constructor Parameters
+
+The **RateLimiter** is a utility that helps manage the pace of requests to avoid overloading servers or getting blocked due to rate limits. It operates internally to delay requests and handle retries but can be configured using its constructor parameters.
+
+**Parameters of the `RateLimiter` constructor:**
+
+1. **`base_delay`** (`Tuple[float, float]`, default: `(1.0, 3.0)`)  
+  The range for a random delay (in seconds) between consecutive requests to the same domain.
+
+- A random delay is chosen between `base_delay[0]` and `base_delay[1]` for each request.  
+- This prevents sending requests at a predictable frequency, reducing the chances of triggering rate limits.
+
+**Example:**  
+If `base_delay = (2.0, 5.0)`, delays could be randomly chosen as `2.3s`, `4.1s`, etc.
+
+---
+
+2. **`max_delay`** (`float`, default: `60.0`)  
+  The maximum allowable delay when rate-limiting errors occur.
+
+- When servers return rate-limit responses (e.g., 429 or 503), the delay increases exponentially with jitter.  
+- The `max_delay` ensures the delay doesn’t grow unreasonably high, capping it at this value.
+
+**Example:**  
+For a `max_delay = 30.0`, even if backoff calculations suggest a delay of `45s`, it will cap at `30s`.
+
+---
+
+3. **`max_retries`** (`int`, default: `3`)  
+  The maximum number of retries for a request if rate-limiting errors occur.
+
+- After encountering a rate-limit response, the `RateLimiter` retries the request up to this number of times.  
+- If all retries fail, the request is marked as failed, and the process continues.
+
+**Example:**  
+If `max_retries = 3`, the system retries a failed request three times before giving up.
+
+---
+
+4. **`rate_limit_codes`** (`List[int]`, default: `[429, 503]`)  
+  A list of HTTP status codes that trigger the rate-limiting logic.
+
+- These status codes indicate the server is overwhelmed or actively limiting requests.  
+- You can customize this list to include other codes based on specific server behavior.
+
+**Example:**  
+If `rate_limit_codes = [429, 503, 504]`, the crawler will back off on these three error codes.
+
+---
+
+**How to Use the `RateLimiter`:**
+
+Here’s an example of initializing and using a `RateLimiter` in your project:
+
+```python
+from crawl4ai import RateLimiter
+
+# Create a RateLimiter with custom settings
+rate_limiter = RateLimiter(
+    base_delay=(2.0, 4.0),  # Random delay between 2-4 seconds
+    max_delay=30.0,         # Cap delay at 30 seconds
+    max_retries=5,          # Retry up to 5 times on rate-limiting errors
+    rate_limit_codes=[429, 503]  # Handle these HTTP status codes
+)
+
+# RateLimiter will handle delays and retries internally
+# No additional setup is required for its operation
+```
+
+The `RateLimiter` integrates seamlessly with dispatchers like `MemoryAdaptiveDispatcher` and `SemaphoreDispatcher`, ensuring requests are paced correctly without user intervention. Its internal mechanisms manage delays and retries to avoid overwhelming servers while maximizing efficiency.
+
+
+### 2.2 Crawler Monitor
+
+The CrawlerMonitor provides real-time visibility into crawling operations:
+
+```python
+from crawl4ai import CrawlerMonitor, DisplayMode
+monitor = CrawlerMonitor(
+    # Maximum rows in live display
+    max_visible_rows=15,          
+
+    # DETAILED or AGGREGATED view
+    display_mode=DisplayMode.DETAILED  
+)
+```
+
+**Display Modes**:
+
+1. **DETAILED**: Shows individual task status, memory usage, and timing
+2. **AGGREGATED**: Displays summary statistics and overall progress
+
+---
+
+## 3. Available Dispatchers
+
+### 3.1 MemoryAdaptiveDispatcher (Default)
+
+Automatically manages concurrency based on system memory usage:
+
+```python
+from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher
+
+dispatcher = MemoryAdaptiveDispatcher(
+    memory_threshold_percent=90.0,  # Pause if memory exceeds this
+    check_interval=1.0,             # How often to check memory
+    max_session_permit=10,          # Maximum concurrent tasks
+    rate_limiter=RateLimiter(       # Optional rate limiting
+        base_delay=(1.0, 2.0),
+        max_delay=30.0,
+        max_retries=2
+    ),
+    monitor=CrawlerMonitor(         # Optional monitoring
+        max_visible_rows=15,
+        display_mode=DisplayMode.DETAILED
+    )
+)
+```
+
+**Constructor Parameters:**
+
+1. **`memory_threshold_percent`** (`float`, default: `90.0`)  
+  Specifies the memory usage threshold (as a percentage). If system memory usage exceeds this value, the dispatcher pauses crawling to prevent system overload.
+
+2. **`check_interval`** (`float`, default: `1.0`)  
+  The interval (in seconds) at which the dispatcher checks system memory usage.
+
+3. **`max_session_permit`** (`int`, default: `10`)  
+  The maximum number of concurrent crawling tasks allowed. This ensures resource limits are respected while maintaining concurrency.
+
+4. **`memory_wait_timeout`** (`float`, default: `600.0`)
+  Optional timeout (in seconds). If memory usage exceeds `memory_threshold_percent` for longer than this duration, a `MemoryError` is raised.
+
+5. **`rate_limiter`** (`RateLimiter`, default: `None`)  
+  Optional rate-limiting logic to avoid server-side blocking (e.g., for handling 429 or 503 errors). See **RateLimiter** for details.
+
+6. **`monitor`** (`CrawlerMonitor`, default: `None`)  
+  Optional monitoring for real-time task tracking and performance insights. See **CrawlerMonitor** for details.
+
+---
+
+### 3.2 SemaphoreDispatcher
+
+Provides simple concurrency control with a fixed limit:
+
+```python
+from crawl4ai.async_dispatcher import SemaphoreDispatcher
+
+dispatcher = SemaphoreDispatcher(
+    max_session_permit=20,         # Maximum concurrent tasks
+    rate_limiter=RateLimiter(      # Optional rate limiting
+        base_delay=(0.5, 1.0),
+        max_delay=10.0
+    ),
+    monitor=CrawlerMonitor(        # Optional monitoring
+        max_visible_rows=15,
+        display_mode=DisplayMode.DETAILED
+    )
+)
+```
+
+**Constructor Parameters:**
+
+1. **`max_session_permit`** (`int`, default: `20`)  
+  The maximum number of concurrent crawling tasks allowed, irrespective of semaphore slots.
+
+2. **`rate_limiter`** (`RateLimiter`, default: `None`)  
+  Optional rate-limiting logic to avoid overwhelming servers. See **RateLimiter** for details.
+
+3. **`monitor`** (`CrawlerMonitor`, default: `None`)  
+  Optional monitoring for tracking task progress and resource usage. See **CrawlerMonitor** for details.
+
+---
+
+## 4. Usage Examples
+
+### 4.1 Batch Processing (Default)
+
+```python
+async def crawl_batch():
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        stream=False  # Default: get all results at once
+    )
+    
+    dispatcher = MemoryAdaptiveDispatcher(
+        memory_threshold_percent=70.0,
+        check_interval=1.0,
+        max_session_permit=10,
+        monitor=CrawlerMonitor(
+            display_mode=DisplayMode.DETAILED
+        )
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Get all results at once
+        results = await crawler.arun_many(
+            urls=urls,
+            config=run_config,
+            dispatcher=dispatcher
+        )
+        
+        # Process all results after completion
+        for result in results:
+            if result.success:
+                await process_result(result)
+            else:
+                print(f"Failed to crawl {result.url}: {result.error_message}")
+```
+
+**Review:**  
+- **Purpose:** Executes a batch crawl with all URLs processed together after crawling is complete.  
+- **Dispatcher:** Uses `MemoryAdaptiveDispatcher` to manage concurrency and system memory.  
+- **Stream:** Disabled (`stream=False`), so all results are collected at once for post-processing.  
+- **Best Use Case:** When you need to analyze results in bulk rather than individually during the crawl.
+
+---
+
+### 4.2 Streaming Mode
+
+```python
+async def crawl_streaming():
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        stream=True  # Enable streaming mode
+    )
+    
+    dispatcher = MemoryAdaptiveDispatcher(
+        memory_threshold_percent=70.0,
+        check_interval=1.0,
+        max_session_permit=10,
+        monitor=CrawlerMonitor(
+            display_mode=DisplayMode.DETAILED
+        )
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Process results as they become available
+        async for result in await crawler.arun_many(
+            urls=urls,
+            config=run_config,
+            dispatcher=dispatcher
+        ):
+            if result.success:
+                # Process each result immediately
+                await process_result(result)
+            else:
+                print(f"Failed to crawl {result.url}: {result.error_message}")
+```
+
+**Review:**  
+- **Purpose:** Enables streaming to process results as soon as they’re available.  
+- **Dispatcher:** Uses `MemoryAdaptiveDispatcher` for concurrency and memory management.  
+- **Stream:** Enabled (`stream=True`), allowing real-time processing during crawling.  
+- **Best Use Case:** When you need to act on results immediately, such as for real-time analytics or progressive data storage.
+
+---
+
+### 4.3 Semaphore-based Crawling
+
+```python
+async def crawl_with_semaphore(urls):
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    
+    dispatcher = SemaphoreDispatcher(
+        semaphore_count=5,
+        rate_limiter=RateLimiter(
+            base_delay=(0.5, 1.0),
+            max_delay=10.0
+        ),
+        monitor=CrawlerMonitor(
+            max_visible_rows=15,
+            display_mode=DisplayMode.DETAILED
+        )
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        results = await crawler.arun_many(
+            urls, 
+            config=run_config,
+            dispatcher=dispatcher
+        )
+        return results
+```
+
+**Review:**  
+- **Purpose:** Uses `SemaphoreDispatcher` to limit concurrency with a fixed number of slots.  
+- **Dispatcher:** Configured with a semaphore to control parallel crawling tasks.  
+- **Rate Limiter:** Prevents servers from being overwhelmed by pacing requests.  
+- **Best Use Case:** When you want precise control over the number of concurrent requests, independent of system memory.
+
+---
+
+### 4.4 Robots.txt Consideration
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+
+async def main():
+    urls = [
+        "https://example1.com",
+        "https://example2.com",
+        "https://example3.com"
+    ]
+    
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.ENABLED,
+        check_robots_txt=True,  # Will respect robots.txt for each URL
+        semaphore_count=3      # Max concurrent requests
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        async for result in crawler.arun_many(urls, config=config):
+            if result.success:
+                print(f"Successfully crawled {result.url}")
+            elif result.status_code == 403 and "robots.txt" in result.error_message:
+                print(f"Skipped {result.url} - blocked by robots.txt")
+            else:
+                print(f"Failed to crawl {result.url}: {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Review:**  
+- **Purpose:** Ensures compliance with `robots.txt` rules for ethical and legal web crawling.  
+- **Configuration:** Set `check_robots_txt=True` to validate each URL against `robots.txt` before crawling.  
+- **Dispatcher:** Handles requests with concurrency limits (`semaphore_count=3`).  
+- **Best Use Case:** When crawling websites that strictly enforce robots.txt policies or for responsible crawling practices.
+
+---
+
+## 5. Dispatch Results
+
+Each crawl result includes dispatch information:
+
+```python
+@dataclass
+class DispatchResult:
+    task_id: str
+    memory_usage: float
+    peak_memory: float
+    start_time: datetime
+    end_time: datetime
+    error_message: str = ""
+```
+
+Access via `result.dispatch_result`:
+
+```python
+for result in results:
+    if result.success:
+        dr = result.dispatch_result
+        print(f"URL: {result.url}")
+        print(f"Memory: {dr.memory_usage:.1f}MB")
+        print(f"Duration: {dr.end_time - dr.start_time}")
+```
+
+## 6. URL-Specific Configurations
+
+When crawling diverse content types, you often need different configurations for different URLs. For example:
+- PDFs need specialized extraction
+- Blog pages benefit from content filtering
+- Dynamic sites need JavaScript execution
+- API endpoints need JSON parsing
+
+### 6.1 Basic URL Pattern Matching
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, MatchMode
+from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def crawl_mixed_content():
+    # Configure different strategies for different content
+    configs = [
+        # PDF files - specialized extraction
+        CrawlerRunConfig(
+            url_matcher="*.pdf",
+            scraping_strategy=PDFContentScrapingStrategy()
+        ),
+        
+        # Blog/article pages - content filtering
+        CrawlerRunConfig(
+            url_matcher=["*/blog/*", "*/article/*"],
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(threshold=0.48)
+            )
+        ),
+        
+        # Dynamic pages - JavaScript execution
+        CrawlerRunConfig(
+            url_matcher=lambda url: 'github.com' in url,
+            js_code="window.scrollTo(0, 500);"
+        ),
+        
+        # API endpoints - JSON extraction
+        CrawlerRunConfig(
+            url_matcher=lambda url: 'api' in url or url.endswith('.json'),
+            # Custome settings for JSON extraction
+        ),
+        
+        # Default config for everything else
+        CrawlerRunConfig()  # No url_matcher means it matches ALL URLs (fallback)
+    ]
+    
+    # Mixed URLs
+    urls = [
+        "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
+        "https://blog.python.org/",
+        "https://github.com/microsoft/playwright",
+        "https://httpbin.org/json",
+        "https://example.com/"
+    ]
+    
+    async with AsyncWebCrawler() as crawler:
+        results = await crawler.arun_many(
+            urls=urls,
+            config=configs  # Pass list of configs
+        )
+        
+        for result in results:
+            print(f"{result.url}: {len(result.markdown)} chars")
+```
+
+### 6.2 Advanced Pattern Matching
+
+**Important**: A `CrawlerRunConfig` without `url_matcher` (or with `url_matcher=None`) matches ALL URLs. This makes it perfect as a default/fallback configuration.
+
+The `url_matcher` parameter supports three types of patterns:
+
+#### Glob Patterns (Strings)
+```python
+# Simple patterns
+"*.pdf"                    # Any PDF file
+"*/api/*"                  # Any URL with /api/ in path
+"https://*.example.com/*"  # Subdomain matching
+"*://example.com/blog/*"   # Any protocol
+```
+
+#### Custom Functions
+```python
+# Complex logic with lambdas
+lambda url: url.startswith('https://') and 'secure' in url
+lambda url: len(url) > 50 and url.count('/') > 5
+lambda url: any(domain in url for domain in ['api.', 'data.', 'feed.'])
+```
+
+#### Mixed Lists with AND/OR Logic
+```python
+# Combine multiple conditions
+CrawlerRunConfig(
+    url_matcher=[
+        "https://*",                        # Must be HTTPS
+        lambda url: 'internal' in url,      # Must contain 'internal'
+        lambda url: not url.endswith('.pdf') # Must not be PDF
+    ],
+    match_mode=MatchMode.AND  # ALL conditions must match
+)
+```
+
+### 6.3 Practical Example: News Site Crawler
+
+```python
+async def crawl_news_site():
+    dispatcher = MemoryAdaptiveDispatcher(
+        memory_threshold_percent=70.0,
+        rate_limiter=RateLimiter(base_delay=(1.0, 2.0))
+    )
+    
+    configs = [
+        # Homepage - light extraction
+        CrawlerRunConfig(
+            url_matcher=lambda url: url.rstrip('/') == 'https://news.ycombinator.com',
+            css_selector="nav, .headline",
+            extraction_strategy=None
+        ),
+        
+        # Article pages - full extraction
+        CrawlerRunConfig(
+            url_matcher="*/article/*",
+            extraction_strategy=CosineStrategy(
+                semantic_filter="article content",
+                word_count_threshold=100
+            ),
+            screenshot=True,
+            excluded_tags=["nav", "aside", "footer"]
+        ),
+        
+        # Author pages - metadata focus
+        CrawlerRunConfig(
+            url_matcher="*/author/*",
+            extraction_strategy=JsonCssExtractionStrategy({
+                "name": "h1.author-name",
+                "bio": ".author-bio",
+                "articles": "article.post-card h2"
+            })
+        ),
+        
+        # Everything else
+        CrawlerRunConfig()
+    ]
+    
+    async with AsyncWebCrawler() as crawler:
+        results = await crawler.arun_many(
+            urls=news_urls,
+            config=configs,
+            dispatcher=dispatcher
+        )
+```
+
+### 6.4 Best Practices
+
+1. **Order Matters**: Configs are evaluated in order - put specific patterns before general ones
+2. **Default Config Behavior**: 
+   - A config without `url_matcher` matches ALL URLs
+   - Always include a default config as the last item if you want to handle all URLs
+   - Without a default config, unmatched URLs will fail with "No matching configuration found"
+3. **Test Your Patterns**: Use the config's `is_match()` method to test patterns:
+   ```python
+   config = CrawlerRunConfig(url_matcher="*.pdf")
+   print(config.is_match("https://example.com/doc.pdf"))  # True
+   
+   default_config = CrawlerRunConfig()  # No url_matcher
+   print(default_config.is_match("https://any-url.com"))  # True - matches everything!
+   ```
+4. **Optimize for Performance**: 
+   - Disable JS for static content
+   - Skip screenshots for data APIs
+   - Use appropriate extraction strategies
+
+## 7. Summary
+
+1. **Two Dispatcher Types**:
+
+   - MemoryAdaptiveDispatcher (default): Dynamic concurrency based on memory
+   - SemaphoreDispatcher: Fixed concurrency limit
+
+2. **Optional Components**:
+
+   - RateLimiter: Smart request pacing and backoff
+   - CrawlerMonitor: Real-time progress visualization
+
+3. **Key Benefits**:
+
+   - Automatic memory management
+   - Built-in rate limiting
+   - Live progress monitoring
+   - Flexible concurrency control
+
+Choose the dispatcher that best fits your needs:
+
+- **MemoryAdaptiveDispatcher**: For large crawls or limited resources
+- **SemaphoreDispatcher**: For simple, fixed-concurrency scenarios
diff --git a/docs/md_v2/advanced/network-console-capture.md b/docs/md_v2/advanced/network-console-capture.md
new file mode 100644
index 0000000..4305a25
--- /dev/null
+++ b/docs/md_v2/advanced/network-console-capture.md
@@ -0,0 +1,205 @@
+# Network Requests & Console Message Capturing
+
+Crawl4AI can capture all network requests and browser console messages during a crawl, which is invaluable for debugging, security analysis, or understanding page behavior.
+
+## Configuration
+
+To enable network and console capturing, use these configuration options:
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+# Enable both network request capture and console message capture
+config = CrawlerRunConfig(
+    capture_network_requests=True,  # Capture all network requests and responses
+    capture_console_messages=True   # Capture all browser console output
+)
+```
+
+## Example Usage
+
+```python
+import asyncio
+import json
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    # Enable both network request capture and console message capture
+    config = CrawlerRunConfig(
+        capture_network_requests=True,
+        capture_console_messages=True
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=config
+        )
+        
+        if result.success:
+            # Analyze network requests
+            if result.network_requests:
+                print(f"Captured {len(result.network_requests)} network events")
+                
+                # Count request types
+                request_count = len([r for r in result.network_requests if r.get("event_type") == "request"])
+                response_count = len([r for r in result.network_requests if r.get("event_type") == "response"])
+                failed_count = len([r for r in result.network_requests if r.get("event_type") == "request_failed"])
+                
+                print(f"Requests: {request_count}, Responses: {response_count}, Failed: {failed_count}")
+                
+                # Find API calls
+                api_calls = [r for r in result.network_requests 
+                            if r.get("event_type") == "request" and "api" in r.get("url", "")]
+                if api_calls:
+                    print(f"Detected {len(api_calls)} API calls:")
+                    for call in api_calls[:3]:  # Show first 3
+                        print(f"  - {call.get('method')} {call.get('url')}")
+            
+            # Analyze console messages
+            if result.console_messages:
+                print(f"Captured {len(result.console_messages)} console messages")
+                
+                # Group by type
+                message_types = {}
+                for msg in result.console_messages:
+                    msg_type = msg.get("type", "unknown")
+                    message_types[msg_type] = message_types.get(msg_type, 0) + 1
+                
+                print("Message types:", message_types)
+                
+                # Show errors (often the most important)
+                errors = [msg for msg in result.console_messages if msg.get("type") == "error"]
+                if errors:
+                    print(f"Found {len(errors)} console errors:")
+                    for err in errors[:2]:  # Show first 2
+                        print(f"  - {err.get('text', '')[:100]}")
+            
+            # Export all captured data to a file for detailed analysis
+            with open("network_capture.json", "w") as f:
+                json.dump({
+                    "url": result.url,
+                    "network_requests": result.network_requests or [],
+                    "console_messages": result.console_messages or []
+                }, f, indent=2)
+            
+            print("Exported detailed capture data to network_capture.json")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Captured Data Structure
+
+### Network Requests
+
+The `result.network_requests` contains a list of dictionaries, each representing a network event with these common fields:
+
+| Field | Description |
+|-------|-------------|
+| `event_type` | Type of event: `"request"`, `"response"`, or `"request_failed"` |
+| `url` | The URL of the request |
+| `timestamp` | Unix timestamp when the event was captured |
+
+#### Request Event Fields
+
+```json
+{
+  "event_type": "request",
+  "url": "https://example.com/api/data.json",
+  "method": "GET",
+  "headers": {"User-Agent": "...", "Accept": "..."},
+  "post_data": "key=value&otherkey=value",
+  "resource_type": "fetch",
+  "is_navigation_request": false,
+  "timestamp": 1633456789.123
+}
+```
+
+#### Response Event Fields
+
+```json
+{
+  "event_type": "response",
+  "url": "https://example.com/api/data.json",
+  "status": 200,
+  "status_text": "OK",
+  "headers": {"Content-Type": "application/json", "Cache-Control": "..."},
+  "from_service_worker": false,
+  "request_timing": {"requestTime": 1234.56, "receiveHeadersEnd": 1234.78},
+  "timestamp": 1633456789.456
+}
+```
+
+#### Failed Request Event Fields
+
+```json
+{
+  "event_type": "request_failed",
+  "url": "https://example.com/missing.png",
+  "method": "GET",
+  "resource_type": "image",
+  "failure_text": "net::ERR_ABORTED 404",
+  "timestamp": 1633456789.789
+}
+```
+
+### Console Messages
+
+The `result.console_messages` contains a list of dictionaries, each representing a console message with these common fields:
+
+| Field | Description |
+|-------|-------------|
+| `type` | Message type: `"log"`, `"error"`, `"warning"`, `"info"`, etc. |
+| `text` | The message text |
+| `timestamp` | Unix timestamp when the message was captured |
+
+#### Console Message Example
+
+```json
+{
+  "type": "error",
+  "text": "Uncaught TypeError: Cannot read property 'length' of undefined",
+  "location": "https://example.com/script.js:123:45",
+  "timestamp": 1633456790.123
+}
+```
+
+## Key Benefits
+
+- **Full Request Visibility**: Capture all network activity including:
+  - Requests (URLs, methods, headers, post data)
+  - Responses (status codes, headers, timing)
+  - Failed requests (with error messages)
+  
+- **Console Message Access**: View all JavaScript console output:
+  - Log messages
+  - Warnings
+  - Errors with stack traces
+  - Developer debugging information
+
+- **Debugging Power**: Identify issues such as:
+  - Failed API calls or resource loading
+  - JavaScript errors affecting page functionality
+  - CORS or other security issues
+  - Hidden API endpoints and data flows
+
+- **Security Analysis**: Detect:
+  - Unexpected third-party requests
+  - Data leakage in request payloads
+  - Suspicious script behavior
+
+- **Performance Insights**: Analyze:
+  - Request timing data
+  - Resource loading patterns
+  - Potential bottlenecks
+
+## Use Cases
+
+1. **API Discovery**: Identify hidden endpoints and data flows in single-page applications
+2. **Debugging**: Track down JavaScript errors affecting page functionality
+3. **Security Auditing**: Detect unwanted third-party requests or data leakage
+4. **Performance Analysis**: Identify slow-loading resources
+5. **Ad/Tracker Analysis**: Detect and catalog advertising or tracking calls
+
+This capability is especially valuable for complex sites with heavy JavaScript, single-page applications, or when you need to understand the exact communication happening between a browser and servers.
\ No newline at end of file
diff --git a/docs/md_v2/advanced/pdf-parsing.md b/docs/md_v2/advanced/pdf-parsing.md
new file mode 100644
index 0000000..909c0dd
--- /dev/null
+++ b/docs/md_v2/advanced/pdf-parsing.md
@@ -0,0 +1,201 @@
+# PDF Processing Strategies
+
+Crawl4AI provides specialized strategies for handling and extracting content from PDF files. These strategies allow you to seamlessly integrate PDF processing into your crawling workflows, whether the PDFs are hosted online or stored locally.
+
+## `PDFCrawlerStrategy`
+
+### Overview
+`PDFCrawlerStrategy` is an implementation of `AsyncCrawlerStrategy` designed specifically for PDF documents. Instead of interpreting the input URL as an HTML webpage, this strategy treats it as a pointer to a PDF file. It doesn't perform deep crawling or HTML parsing itself but rather prepares the PDF source for a dedicated PDF scraping strategy. Its primary role is to identify the PDF source (web URL or local file) and pass it along the processing pipeline in a way that `AsyncWebCrawler` can handle.
+
+### When to Use
+Use `PDFCrawlerStrategy` when you need to:
+- Process PDF files using the `AsyncWebCrawler`.
+- Handle PDFs from both web URLs (e.g., `https://example.com/document.pdf`) and local file paths (e.g., `file:///path/to/your/document.pdf`).
+- Integrate PDF content extraction into a unified `CrawlResult` object, allowing consistent handling of PDF data alongside web page data.
+
+### Key Methods and Their Behavior
+-   **`__init__(self, logger: AsyncLogger = None)`**:
+    -   Initializes the strategy.
+    -   `logger`: An optional `AsyncLogger` instance (from `crawl4ai.async_logger`) for logging purposes.
+-   **`async crawl(self, url: str, **kwargs) -> AsyncCrawlResponse`**:
+    -   This method is called by the `AsyncWebCrawler` during the `arun` process.
+    -   It takes the `url` (which should point to a PDF) and creates a minimal `AsyncCrawlResponse`.
+    -   The `html` attribute of this response is typically empty or a placeholder, as the actual PDF content processing is deferred to the `PDFContentScrapingStrategy` (or a similar PDF-aware scraping strategy).
+    -   It sets `response_headers` to indicate "application/pdf" and `status_code` to 200.
+-   **`async close(self)`**:
+    -   A method for cleaning up any resources used by the strategy. For `PDFCrawlerStrategy`, this is usually minimal.
+-   **`async __aenter__(self)` / `async __aexit__(self, exc_type, exc_val, exc_tb)`**:
+    -   Enables asynchronous context management for the strategy, allowing it to be used with `async with`.
+
+### Example Usage
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.processors.pdf import PDFCrawlerStrategy, PDFContentScrapingStrategy
+
+async def main():
+    # Initialize the PDF crawler strategy
+    pdf_crawler_strategy = PDFCrawlerStrategy()
+
+    # PDFCrawlerStrategy is typically used in conjunction with PDFContentScrapingStrategy
+    # The scraping strategy handles the actual PDF content extraction
+    pdf_scraping_strategy = PDFContentScrapingStrategy()
+    run_config = CrawlerRunConfig(scraping_strategy=pdf_scraping_strategy)
+
+    async with AsyncWebCrawler(crawler_strategy=pdf_crawler_strategy) as crawler:
+        # Example with a remote PDF URL
+        pdf_url = "https://arxiv.org/pdf/2310.06825.pdf" # A public PDF from arXiv
+        
+        print(f"Attempting to process PDF: {pdf_url}")
+        result = await crawler.arun(url=pdf_url, config=run_config)
+
+        if result.success:
+            print(f"Successfully processed PDF: {result.url}")
+            print(f"Metadata Title: {result.metadata.get('title', 'N/A')}")
+            # Further processing of result.markdown, result.media, etc.
+            # would be done here, based on what PDFContentScrapingStrategy extracts.
+            if result.markdown and hasattr(result.markdown, 'raw_markdown'):
+                print(f"Extracted text (first 200 chars): {result.markdown.raw_markdown[:200]}...")
+            else:
+                print("No markdown (text) content extracted.")
+        else:
+            print(f"Failed to process PDF: {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### Pros and Cons
+**Pros:**
+-   Enables `AsyncWebCrawler` to handle PDF sources directly using familiar `arun` calls.
+-   Provides a consistent interface for specifying PDF sources (URLs or local paths).
+-   Abstracts the source handling, allowing a separate scraping strategy to focus on PDF content parsing.
+
+**Cons:**
+-   Does not perform any PDF data extraction itself; it strictly relies on a compatible scraping strategy (like `PDFContentScrapingStrategy`) to process the PDF.
+-   Has limited utility on its own; most of its value comes from being paired with a PDF-specific content scraping strategy.
+
+---
+
+## `PDFContentScrapingStrategy`
+
+### Overview
+`PDFContentScrapingStrategy` is an implementation of `ContentScrapingStrategy` designed to extract text, metadata, and optionally images from PDF documents. It is intended to be used in conjunction with a crawler strategy that can provide it with a PDF source, such as `PDFCrawlerStrategy`. This strategy uses the `NaivePDFProcessorStrategy` internally to perform the low-level PDF parsing.
+
+### When to Use
+Use `PDFContentScrapingStrategy` when your `AsyncWebCrawler` (often configured with `PDFCrawlerStrategy`) needs to:
+-   Extract textual content page by page from a PDF document.
+-   Retrieve standard metadata embedded within the PDF (e.g., title, author, subject, creation date, page count).
+-   Optionally, extract images contained within the PDF pages. These images can be saved to a local directory or made available for further processing.
+-   Produce a `ScrapingResult` that can be converted into a `CrawlResult`, making PDF content accessible in a manner similar to HTML web content (e.g., text in `result.markdown`, metadata in `result.metadata`).
+
+### Key Configuration Attributes
+When initializing `PDFContentScrapingStrategy`, you can configure its behavior using the following attributes:
+-   **`extract_images: bool = False`**: If `True`, the strategy will attempt to extract images from the PDF.
+-   **`save_images_locally: bool = False`**: If `True` (and `extract_images` is also `True`), extracted images will be saved to disk in the `image_save_dir`. If `False`, image data might be available in another form (e.g., base64, depending on the underlying processor) but not saved as separate files by this strategy.
+-   **`image_save_dir: str = None`**: Specifies the directory where extracted images should be saved if `save_images_locally` is `True`. If `None`, a default or temporary directory might be used.
+-   **`batch_size: int = 4`**: Defines how many PDF pages are processed in a single batch. This can be useful for managing memory when dealing with very large PDF documents.
+-   **`logger: AsyncLogger = None`**: An optional `AsyncLogger` instance for logging.
+
+### Key Methods and Their Behavior
+-   **`__init__(self, save_images_locally: bool = False, extract_images: bool = False, image_save_dir: str = None, batch_size: int = 4, logger: AsyncLogger = None)`**:
+    -   Initializes the strategy with configurations for image handling, batch processing, and logging. It sets up an internal `NaivePDFProcessorStrategy` instance which performs the actual PDF parsing.
+-   **`scrap(self, url: str, html: str, **params) -> ScrapingResult`**:
+    -   This is the primary synchronous method called by the crawler (via `ascrap`) to process the PDF.
+    -   `url`: The path or URL to the PDF file (provided by `PDFCrawlerStrategy` or similar).
+    -   `html`: Typically an empty string when used with `PDFCrawlerStrategy`, as the content is a PDF, not HTML.
+    -   It first ensures the PDF is accessible locally (downloads it to a temporary file if `url` is remote).
+    -   It then uses its internal PDF processor to extract text, metadata, and images (if configured).
+    -   The extracted information is compiled into a `ScrapingResult` object:
+        -   `cleaned_html`: Contains an HTML-like representation of the PDF, where each page's content is often wrapped in a `
` with page number information.
+        -   `media`: A dictionary where `media["images"]` will contain information about extracted images if `extract_images` was `True`.
+        -   `links`: A dictionary where `links["urls"]` can contain URLs found within the PDF content.
+        -   `metadata`: A dictionary holding PDF metadata (e.g., title, author, num_pages).
+-   **`async ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult`**:
+    -   The asynchronous version of `scrap`. Under the hood, it typically runs the synchronous `scrap` method in a separate thread using `asyncio.to_thread` to avoid blocking the event loop.
+-   **`_get_pdf_path(self, url: str) -> str`**:
+    -   A private helper method to manage PDF file access. If the `url` is remote (http/https), it downloads the PDF to a temporary local file and returns its path. If `url` indicates a local file (`file://` or a direct path), it resolves and returns the local path.
+
+### Example Usage
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.processors.pdf import PDFCrawlerStrategy, PDFContentScrapingStrategy
+import os # For creating image directory
+
+async def main():
+    # Define the directory for saving extracted images
+    image_output_dir = "./my_pdf_images"
+    os.makedirs(image_output_dir, exist_ok=True)
+
+    # Configure the PDF content scraping strategy
+    # Enable image extraction and specify where to save them
+    pdf_scraping_cfg = PDFContentScrapingStrategy(
+        extract_images=True,
+        save_images_locally=True,
+        image_save_dir=image_output_dir,
+        batch_size=2 # Process 2 pages at a time for demonstration
+    )
+
+    # The PDFCrawlerStrategy is needed to tell AsyncWebCrawler how to "crawl" a PDF
+    pdf_crawler_cfg = PDFCrawlerStrategy()
+
+    # Configure the overall crawl run
+    run_cfg = CrawlerRunConfig(
+        scraping_strategy=pdf_scraping_cfg # Use our PDF scraping strategy
+    )
+
+    # Initialize the crawler with the PDF-specific crawler strategy
+    async with AsyncWebCrawler(crawler_strategy=pdf_crawler_cfg) as crawler:
+        pdf_url = "https://arxiv.org/pdf/2310.06825.pdf" # Example PDF
+        
+        print(f"Starting PDF processing for: {pdf_url}")
+        result = await crawler.arun(url=pdf_url, config=run_cfg)
+
+        if result.success:
+            print("\n--- PDF Processing Successful ---")
+            print(f"Processed URL: {result.url}")
+            
+            print("\n--- Metadata ---")
+            for key, value in result.metadata.items():
+                print(f"  {key.replace('_', ' ').title()}: {value}")
+
+            if result.markdown and hasattr(result.markdown, 'raw_markdown'):
+                print(f"\n--- Extracted Text (Markdown Snippet) ---")
+                print(result.markdown.raw_markdown[:500].strip() + "...")
+            else:
+                print("\nNo text (markdown) content extracted.")
+
+            if result.media and result.media.get("images"):
+                print(f"\n--- Image Extraction ---")
+                print(f"Extracted {len(result.media['images'])} image(s).")
+                for i, img_info in enumerate(result.media["images"][:2]): # Show info for first 2 images
+                    print(f"  Image {i+1}:")
+                    print(f"    Page: {img_info.get('page')}")
+                    print(f"    Format: {img_info.get('format', 'N/A')}")
+                    if img_info.get('path'):
+                        print(f"    Saved at: {img_info.get('path')}")
+            else:
+                print("\nNo images were extracted (or extract_images was False).")
+        else:
+            print(f"\n--- PDF Processing Failed ---")
+            print(f"Error: {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### Pros and Cons
+
+**Pros:**
+-   Provides a comprehensive way to extract text, metadata, and (optionally) images from PDF documents.
+-   Handles both remote PDFs (via URL) and local PDF files.
+-   Configurable image extraction allows saving images to disk or accessing their data.
+-   Integrates smoothly with the `CrawlResult` object structure, making PDF-derived data accessible in a way consistent with web-scraped data.
+-   The `batch_size` parameter can help in managing memory consumption when processing large or numerous PDF pages.
+
+**Cons:**
+-   Extraction quality and performance can vary significantly depending on the PDF's complexity, encoding, and whether it's image-based (scanned) or text-based.
+-   Image extraction can be resource-intensive (both CPU and disk space if `save_images_locally` is true).
+-   Relies on `NaivePDFProcessorStrategy` internally, which might have limitations with very complex layouts, encrypted PDFs, or forms compared to more sophisticated PDF parsing libraries. Scanned PDFs will not yield text unless an OCR step is performed (which is not part of this strategy by default).
+-   Link extraction from PDFs can be basic and depends on how hyperlinks are embedded in the document.
diff --git a/docs/md_v2/advanced/proxy-security.md b/docs/md_v2/advanced/proxy-security.md
new file mode 100644
index 0000000..39c5d7a
--- /dev/null
+++ b/docs/md_v2/advanced/proxy-security.md
@@ -0,0 +1,308 @@
+# Proxy & Security
+
+This guide covers proxy configuration and security features in Crawl4AI, including SSL certificate analysis and proxy rotation strategies.
+
+## Understanding Proxy Configuration
+
+Crawl4AI recommends configuring proxies per request through `CrawlerRunConfig.proxy_config`. This gives you precise control, enables rotation strategies, and keeps examples simple enough to copy, paste, and run.
+
+## Basic Proxy Setup
+
+Configure proxies that apply to each crawl operation:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, ProxyConfig
+
+run_config = CrawlerRunConfig(proxy_config=ProxyConfig(server="http://proxy.example.com:8080"))
+# run_config = CrawlerRunConfig(proxy_config={"server": "http://proxy.example.com:8080"})
+# run_config = CrawlerRunConfig(proxy_config="http://proxy.example.com:8080")
+
+
+async def main():
+    browser_config = BrowserConfig()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com", config=run_config)
+        print(f"Success: {result.success} -> {result.url}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+!!! note "Why request-level?"
+    `CrawlerRunConfig.proxy_config` keeps each request self-contained, so swapping proxies or rotation strategies is just a matter of building a new run configuration.
+
+## Supported Proxy Formats
+
+The `ProxyConfig.from_string()` method supports multiple formats:
+
+```python
+from crawl4ai import ProxyConfig
+
+# HTTP proxy with authentication
+proxy1 = ProxyConfig.from_string("http://user:pass@192.168.1.1:8080")
+
+# HTTPS proxy
+proxy2 = ProxyConfig.from_string("https://proxy.example.com:8080")
+
+# SOCKS5 proxy
+proxy3 = ProxyConfig.from_string("socks5://proxy.example.com:1080")
+
+# Simple IP:port format
+proxy4 = ProxyConfig.from_string("192.168.1.1:8080")
+
+# IP:port:user:pass format
+proxy5 = ProxyConfig.from_string("192.168.1.1:8080:user:pass")
+```
+
+## Authenticated Proxies
+
+For proxies requiring authentication:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler,BrowserConfig, CrawlerRunConfig, ProxyConfig
+
+run_config = CrawlerRunConfig(
+    proxy_config=ProxyConfig(
+        server="http://proxy.example.com:8080",
+        username="your_username",
+        password="your_password",
+    )
+)
+# Or dictionary style:
+# run_config = CrawlerRunConfig(proxy_config={
+#     "server": "http://proxy.example.com:8080",
+#     "username": "your_username",
+#     "password": "your_password",
+# })
+
+
+async def main():
+    browser_config = BrowserConfig()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com", config=run_config)
+        print(f"Success: {result.success} -> {result.url}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Environment Variable Configuration
+
+Load proxies from environment variables for easy configuration:
+
+```python
+import os
+from crawl4ai import ProxyConfig, CrawlerRunConfig
+
+# Set environment variable
+os.environ["PROXIES"] = "ip1:port1:user1:pass1,ip2:port2:user2:pass2,ip3:port3"
+
+# Load all proxies
+proxies = ProxyConfig.from_env()
+print(f"Loaded {len(proxies)} proxies")
+
+# Use first proxy
+if proxies:
+    run_config = CrawlerRunConfig(proxy_config=proxies[0])
+```
+
+## Rotating Proxies
+
+Crawl4AI supports automatic proxy rotation to distribute requests across multiple proxy servers. Rotation is applied per request using a rotation strategy on `CrawlerRunConfig`.
+
+### Proxy Rotation (recommended)
+```python
+import asyncio
+import re
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, ProxyConfig
+from crawl4ai.proxy_strategy import RoundRobinProxyStrategy
+
+async def main():
+    # Load proxies from environment
+    proxies = ProxyConfig.from_env()
+    if not proxies:
+        print("No proxies found! Set PROXIES environment variable.")
+        return
+
+    # Create rotation strategy
+    proxy_strategy = RoundRobinProxyStrategy(proxies)
+
+    # Configure per-request with proxy rotation
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        proxy_rotation_strategy=proxy_strategy,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        urls = ["https://httpbin.org/ip"] * (len(proxies) * 2)  # Test each proxy twice
+
+        print(f"🚀 Testing {len(proxies)} proxies with rotation...")
+        results = await crawler.arun_many(urls=urls, config=run_config)
+
+        for i, result in enumerate(results):
+            if result.success:
+                # Extract IP from response
+                ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
+                if ip_match:
+                    detected_ip = ip_match.group(0)
+                    proxy_index = i % len(proxies)
+                    expected_ip = proxies[proxy_index].ip
+
+                    print(f"✅ Request {i+1}: Proxy {proxy_index+1} -> IP {detected_ip}")
+                    if detected_ip == expected_ip:
+                        print("   🎯 IP matches proxy configuration")
+                    else:
+                        print(f"   ⚠️  IP mismatch (expected {expected_ip})")
+                else:
+                    print(f"❌ Request {i+1}: Could not extract IP from response")
+            else:
+                print(f"❌ Request {i+1}: Failed - {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## SSL Certificate Analysis
+
+Combine proxy usage with SSL certificate inspection for enhanced security analysis. SSL certificate fetching is configured per request via `CrawlerRunConfig`.
+
+### Per-Request SSL Certificate Analysis
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+run_config = CrawlerRunConfig(
+    proxy_config={
+        "server": "http://proxy.example.com:8080",
+        "username": "user",
+        "password": "pass",
+    },
+    fetch_ssl_certificate=True,  # Enable SSL certificate analysis for this request
+)
+
+
+async def main():
+    browser_config = BrowserConfig()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com", config=run_config)
+
+        if result.success:
+            print(f"✅ Crawled via proxy: {result.url}")
+
+            # Analyze SSL certificate
+            if result.ssl_certificate:
+                cert = result.ssl_certificate
+                print("🔒 SSL Certificate Info:")
+                print(f"   Issuer: {cert.issuer}")
+                print(f"   Subject: {cert.subject}")
+                print(f"   Valid until: {cert.valid_until}")
+                print(f"   Fingerprint: {cert.fingerprint}")
+
+                # Export certificate
+                cert.to_json("certificate.json")
+                print("💾 Certificate exported to certificate.json")
+            else:
+                print("⚠️  No SSL certificate information available")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Security Best Practices
+
+### 1. Proxy Rotation for Anonymity
+```python
+from crawl4ai import CrawlerRunConfig, ProxyConfig
+from crawl4ai.proxy_strategy import RoundRobinProxyStrategy
+
+# Use multiple proxies to avoid IP blocking
+proxies = ProxyConfig.from_env("PROXIES")
+strategy = RoundRobinProxyStrategy(proxies)
+
+# Configure rotation per request (recommended)
+run_config = CrawlerRunConfig(proxy_rotation_strategy=strategy)
+
+# For a fixed proxy across all requests, just reuse the same run_config instance
+static_run_config = run_config
+```
+
+### 2. SSL Certificate Verification
+```python
+from crawl4ai import CrawlerRunConfig
+
+# Always verify SSL certificates when possible
+# Per-request (affects specific requests)
+run_config = CrawlerRunConfig(fetch_ssl_certificate=True)
+```
+
+### 3. Environment Variable Security
+```bash
+# Use environment variables for sensitive proxy credentials
+# Avoid hardcoding usernames/passwords in code
+export PROXIES="ip1:port1:user1:pass1,ip2:port2:user2:pass2"
+```
+
+### 4. SOCKS5 for Enhanced Security
+```python
+from crawl4ai import CrawlerRunConfig
+
+# Prefer SOCKS5 proxies for better protocol support
+run_config = CrawlerRunConfig(proxy_config="socks5://proxy.example.com:1080")
+```
+
+## Migration from Deprecated `proxy` Parameter
+
+- "Deprecation Notice"
+    The legacy `proxy` argument on `BrowserConfig` is deprecated. Configure proxies through `CrawlerRunConfig.proxy_config` so each request fully describes its network settings.
+
+```python
+# Old (deprecated) approach
+# from crawl4ai import BrowserConfig
+# browser_config = BrowserConfig(proxy_config="http://proxy.example.com:8080")
+
+# New (preferred) approach
+from crawl4ai import CrawlerRunConfig
+run_config = CrawlerRunConfig(proxy_config="http://proxy.example.com:8080")
+```
+
+### Safe Logging of Proxies
+```python
+from crawl4ai import ProxyConfig
+
+def safe_proxy_repr(proxy: ProxyConfig):
+    if getattr(proxy, "username", None):
+        return f"{proxy.server} (auth: ****)"
+    return proxy.server
+```
+
+## Troubleshooting
+
+### Common Issues
+
+- "Proxy connection failed"
+    - Verify the proxy server is reachable from your network.
+    - Double-check authentication credentials.
+    - Ensure the protocol matches (`http`, `https`, or `socks5`).
+
+- "SSL certificate errors"
+    - Some proxies break SSL inspection; switch proxies if you see repeated failures.
+    - Consider temporarily disabling certificate fetching to isolate the issue.
+
+- "Environment variables not loading"
+    - Confirm `PROXIES` (or your custom env var) is set before running the script.
+    - Check formatting: `ip:port:user:pass,ip:port:user:pass`.
+
+- "Proxy rotation not working"
+    - Ensure `ProxyConfig.from_env()` actually loaded entries (`len(proxies) > 0`).
+    - Attach `proxy_rotation_strategy` to `CrawlerRunConfig`.
+    - Validate the proxy definitions you pass into the strategy.
+
+## See Also
+
+- [Anti-Bot Detection & Fallback](anti-bot-and-fallback.md) — Automatic retry with proxy escalation and fallback functions when anti-bot blocking is detected
diff --git a/docs/md_v2/advanced/session-management.md b/docs/md_v2/advanced/session-management.md
new file mode 100644
index 0000000..1007a60
--- /dev/null
+++ b/docs/md_v2/advanced/session-management.md
@@ -0,0 +1,267 @@
+# Session Management
+
+Session management in Crawl4AI is a powerful feature that allows you to maintain state across multiple requests, making it particularly suitable for handling complex multi-step crawling tasks. It enables you to reuse the same browser tab (or page object) across sequential actions and crawls, which is beneficial for:
+
+- **Performing JavaScript actions before and after crawling.**
+- **Executing multiple sequential crawls faster** without needing to reopen tabs or allocate memory repeatedly.
+
+**Note:** This feature is designed for sequential workflows and is not suitable for parallel operations.
+
+---
+
+#### Basic Session Usage
+
+Use `BrowserConfig` and `CrawlerRunConfig` to maintain state with a `session_id`:
+
+```python
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+async with AsyncWebCrawler() as crawler:
+    session_id = "my_session"
+
+    # Define configurations
+    config1 = CrawlerRunConfig(
+        url="https://example.com/page1", session_id=session_id
+    )
+    config2 = CrawlerRunConfig(
+        url="https://example.com/page2", session_id=session_id
+    )
+
+    # First request
+    result1 = await crawler.arun(config=config1)
+
+    # Subsequent request using the same session
+    result2 = await crawler.arun(config=config2)
+
+    # Clean up when done
+    await crawler.crawler_strategy.kill_session(session_id)
+```
+
+---
+
+#### Dynamic Content with Sessions
+
+Here's an example of crawling GitHub commits across multiple pages while preserving session state:
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai import JsonCssExtractionStrategy
+from crawl4ai.cache_context import CacheMode
+
+async def crawl_dynamic_content():
+    url = "https://github.com/microsoft/TypeScript/commits/main"
+    session_id = "wait_for_session"
+    all_commits = []
+
+    js_next_page = """
+    const commits = document.querySelectorAll('li[data-testid="commit-row-item"] h4');
+    if (commits.length > 0) {
+        window.lastCommit = commits[0].textContent.trim();
+    }
+    const button = document.querySelector('a[data-testid="pagination-next-button"]');
+    if (button) {button.click(); console.log('button clicked') }
+    """
+
+    wait_for = """() => {
+        const commits = document.querySelectorAll('li[data-testid="commit-row-item"] h4');
+        if (commits.length === 0) return false;
+        const firstCommit = commits[0].textContent.trim();
+        return firstCommit !== window.lastCommit;
+    }"""
+    
+    schema = {
+        "name": "Commit Extractor",
+        "baseSelector": "li[data-testid='commit-row-item']",
+        "fields": [
+            {
+                "name": "title",
+                "selector": "h4 a",
+                "type": "text",
+                "transform": "strip",
+            },
+        ],
+    }
+    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+    
+    
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+    )
+        
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        for page in range(3):
+            crawler_config = CrawlerRunConfig(
+                session_id=session_id,
+                css_selector="li[data-testid='commit-row-item']",
+                extraction_strategy=extraction_strategy,
+                js_code=js_next_page if page > 0 else None,
+                wait_for=wait_for if page > 0 else None,
+                js_only=page > 0,
+                cache_mode=CacheMode.BYPASS,
+                capture_console_messages=True,
+            )
+            
+            result = await crawler.arun(url=url, config=crawler_config)
+            
+            if result.console_messages:
+                print(f"Page {page + 1} console messages:", result.console_messages)
+            
+            if result.extracted_content:
+                # print(f"Page {page + 1} result:", result.extracted_content)
+                commits = json.loads(result.extracted_content)
+                all_commits.extend(commits)
+                print(f"Page {page + 1}: Found {len(commits)} commits")
+            else:
+                print(f"Page {page + 1}: No content extracted")
+
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+        # Clean up session
+        await crawler.crawler_strategy.kill_session(session_id)
+```
+
+---
+
+## Example 1: Basic Session-Based Crawling
+
+A simple example using session-based crawling:
+
+```python
+import asyncio
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.cache_context import CacheMode
+
+async def basic_session_crawl():
+    async with AsyncWebCrawler() as crawler:
+        session_id = "dynamic_content_session"
+        url = "https://example.com/dynamic-content"
+
+        for page in range(3):
+            config = CrawlerRunConfig(
+                url=url,
+                session_id=session_id,
+                js_code="document.querySelector('.load-more-button').click();" if page > 0 else None,
+                css_selector=".content-item",
+                cache_mode=CacheMode.BYPASS
+            )
+            
+            result = await crawler.arun(config=config)
+            print(f"Page {page + 1}: Found {result.extracted_content.count('.content-item')} items")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+
+asyncio.run(basic_session_crawl())
+```
+
+This example shows:
+1. Reusing the same `session_id` across multiple requests.
+2. Executing JavaScript to load more content dynamically.
+3. Properly closing the session to free resources.
+
+---
+
+## Advanced Technique 1: Custom Execution Hooks
+
+> Warning: You might feel confused by the end of the next few examples 😅, so make sure you are comfortable with the order of the parts before you start this.
+
+Use custom hooks to handle complex scenarios, such as waiting for content to load dynamically:
+
+```python
+async def advanced_session_crawl_with_hooks():
+    first_commit = ""
+
+    async def on_execution_started(page):
+        nonlocal first_commit
+        try:
+            while True:
+                await page.wait_for_selector("li.commit-item h4")
+                commit = await page.query_selector("li.commit-item h4")
+                commit = await commit.evaluate("(element) => element.textContent").strip()
+                if commit and commit != first_commit:
+                    first_commit = commit
+                    break
+                await asyncio.sleep(0.5)
+        except Exception as e:
+            print(f"Warning: New content didn't appear: {e}")
+
+    async with AsyncWebCrawler() as crawler:
+        session_id = "commit_session"
+        url = "https://github.com/example/repo/commits/main"
+        crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+
+        js_next_page = """document.querySelector('a.pagination-next').click();"""
+
+        for page in range(3):
+            config = CrawlerRunConfig(
+                url=url,
+                session_id=session_id,
+                js_code=js_next_page if page > 0 else None,
+                css_selector="li.commit-item",
+                js_only=page > 0,
+                cache_mode=CacheMode.BYPASS
+            )
+
+            result = await crawler.arun(config=config)
+            print(f"Page {page + 1}: Found {len(result.extracted_content)} commits")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+
+asyncio.run(advanced_session_crawl_with_hooks())
+```
+
+This technique ensures new content loads before the next action.
+
+---
+
+## Advanced Technique 2: Integrated JavaScript Execution and Waiting
+
+Combine JavaScript execution and waiting logic for concise handling of dynamic content:
+
+```python
+async def integrated_js_and_wait_crawl():
+    async with AsyncWebCrawler() as crawler:
+        session_id = "integrated_session"
+        url = "https://github.com/example/repo/commits/main"
+
+        js_next_page_and_wait = """
+        (async () => {
+            const getCurrentCommit = () => document.querySelector('li.commit-item h4').textContent.trim();
+            const initialCommit = getCurrentCommit();
+            document.querySelector('a.pagination-next').click();
+            while (getCurrentCommit() === initialCommit) {
+                await new Promise(resolve => setTimeout(resolve, 100));
+            }
+        })();
+        """
+
+        for page in range(3):
+            config = CrawlerRunConfig(
+                url=url,
+                session_id=session_id,
+                js_code=js_next_page_and_wait if page > 0 else None,
+                css_selector="li.commit-item",
+                js_only=page > 0,
+                cache_mode=CacheMode.BYPASS
+            )
+
+            result = await crawler.arun(config=config)
+            print(f"Page {page + 1}: Found {len(result.extracted_content)} commits")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+
+asyncio.run(integrated_js_and_wait_crawl())
+```
+
+---
+
+#### Common Use Cases for Sessions
+
+1. **Authentication Flows**: Login and interact with secured pages.
+
+2. **Pagination Handling**: Navigate through multiple pages.
+
+3. **Form Submissions**: Fill forms, submit, and process results.
+
+4. **Multi-step Processes**: Complete workflows that span multiple actions.
+
+5. **Dynamic Content Navigation**: Handle JavaScript-rendered or event-triggered content.
diff --git a/docs/md_v2/advanced/ssl-certificate.md b/docs/md_v2/advanced/ssl-certificate.md
new file mode 100644
index 0000000..fa04716
--- /dev/null
+++ b/docs/md_v2/advanced/ssl-certificate.md
@@ -0,0 +1,179 @@
+# `SSLCertificate` Reference
+
+The **`SSLCertificate`** class encapsulates an SSL certificate’s data and allows exporting it in various formats (PEM, DER, JSON, or text). It’s used within **Crawl4AI** whenever you set **`fetch_ssl_certificate=True`** in your **`CrawlerRunConfig`**.  
+
+## 1. Overview
+
+**Location**: `crawl4ai/ssl_certificate.py`
+
+```python
+class SSLCertificate:
+    """
+    Represents an SSL certificate with methods to export in various formats.
+
+    Main Methods:
+    - from_url(url, timeout=10)
+    - from_file(file_path)
+    - from_binary(binary_data)
+    - to_json(filepath=None)
+    - to_pem(filepath=None)
+    - to_der(filepath=None)
+    ...
+
+    Common Properties:
+    - issuer
+    - subject
+    - valid_from
+    - valid_until
+    - fingerprint
+    """
+```
+
+### Typical Use Case
+1. You **enable** certificate fetching in your crawl by:
+   ```python
+   CrawlerRunConfig(fetch_ssl_certificate=True, ...)
+   ```
+2. After `arun()`, if `result.ssl_certificate` is present, it’s an instance of **`SSLCertificate`**.  
+3. You can **read** basic properties (issuer, subject, validity) or **export** them in multiple formats.
+
+---
+
+## 2. Construction & Fetching
+
+### 2.1 **`from_url(url, timeout=10)`**
+Manually load an SSL certificate from a given URL (port 443). Typically used internally, but you can call it directly if you want:
+
+```python
+cert = SSLCertificate.from_url("https://example.com")
+if cert:
+    print("Fingerprint:", cert.fingerprint)
+```
+
+### 2.2 **`from_file(file_path)`**
+Load from a file containing certificate data in ASN.1 or DER. Rarely needed unless you have local cert files:
+
+```python
+cert = SSLCertificate.from_file("/path/to/cert.der")
+```
+
+### 2.3 **`from_binary(binary_data)`**
+Initialize from raw binary. E.g., if you captured it from a socket or another source:
+
+```python
+cert = SSLCertificate.from_binary(raw_bytes)
+```
+
+---
+
+## 3. Common Properties
+
+After obtaining a **`SSLCertificate`** instance (e.g. `result.ssl_certificate` from a crawl), you can read:
+
+1. **`issuer`** *(dict)*  
+   - E.g. `{"CN": "My Root CA", "O": "..."}`
+2. **`subject`** *(dict)*  
+   - E.g. `{"CN": "example.com", "O": "ExampleOrg"}`
+3. **`valid_from`** *(str)*  
+   - NotBefore date/time. Often in ASN.1/UTC format.
+4. **`valid_until`** *(str)*  
+   - NotAfter date/time.
+5. **`fingerprint`** *(str)*  
+   - The SHA-256 digest (lowercase hex).  
+   - E.g. `"d14d2e..."`
+
+---
+
+## 4. Export Methods
+
+Once you have a **`SSLCertificate`** object, you can **export** or **inspect** it:
+
+### 4.1 **`to_json(filepath=None)` → `Optional[str]`**
+- Returns a JSON string containing the parsed certificate fields.  
+- If `filepath` is provided, saves it to disk instead, returning `None`.
+
+**Usage**:
+```python
+json_data = cert.to_json()  # returns JSON string
+cert.to_json("certificate.json")  # writes file, returns None
+```
+
+### 4.2 **`to_pem(filepath=None)` → `Optional[str]`**
+- Returns a PEM-encoded string (common for web servers).  
+- If `filepath` is provided, saves it to disk instead.
+
+```python
+pem_str = cert.to_pem()              # in-memory PEM string
+cert.to_pem("/path/to/cert.pem")     # saved to file
+```
+
+### 4.3 **`to_der(filepath=None)` → `Optional[bytes]`**
+- Returns the original DER (binary ASN.1) bytes.  
+- If `filepath` is specified, writes the bytes there instead.
+
+```python
+der_bytes = cert.to_der()
+cert.to_der("certificate.der")
+```
+
+### 4.4 (Optional) **`export_as_text()`**
+- If you see a method like `export_as_text()`, it typically returns an OpenSSL-style textual representation.  
+- Not always needed, but can help for debugging or manual inspection.
+
+---
+
+## 5. Example Usage in Crawl4AI
+
+Below is a minimal sample showing how the crawler obtains an SSL cert from a site, then reads or exports it. The code snippet:
+
+```python
+import asyncio
+import os
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+
+async def main():
+    tmp_dir = "tmp"
+    os.makedirs(tmp_dir, exist_ok=True)
+
+    config = CrawlerRunConfig(
+        fetch_ssl_certificate=True,
+        cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com", config=config)
+        if result.success and result.ssl_certificate:
+            cert = result.ssl_certificate
+            # 1. Basic Info
+            print("Issuer CN:", cert.issuer.get("CN", ""))
+            print("Valid until:", cert.valid_until)
+            print("Fingerprint:", cert.fingerprint)
+            
+            # 2. Export
+            cert.to_json(os.path.join(tmp_dir, "certificate.json"))
+            cert.to_pem(os.path.join(tmp_dir, "certificate.pem"))
+            cert.to_der(os.path.join(tmp_dir, "certificate.der"))
+    
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+## 6. Notes & Best Practices
+
+1. **Timeout**: `SSLCertificate.from_url` internally uses a default **10s** socket connect and wraps SSL.  
+2. **Binary Form**: The certificate is loaded in ASN.1 (DER) form, then re-parsed by `OpenSSL.crypto`.  
+3. **Validation**: This does **not** validate the certificate chain or trust store. It only fetches and parses.  
+4. **Integration**: Within Crawl4AI, you typically just set `fetch_ssl_certificate=True` in `CrawlerRunConfig`; the final result’s `ssl_certificate` is automatically built.  
+5. **Export**: If you need to store or analyze a cert, the `to_json` and `to_pem` are quite universal.
+
+---
+
+### Summary
+
+- **`SSLCertificate`** is a convenience class for capturing and exporting the **TLS certificate** from your crawled site(s).  
+- Common usage is in the **`CrawlResult.ssl_certificate`** field, accessible after setting `fetch_ssl_certificate=True`.  
+- Offers quick access to essential certificate details (`issuer`, `subject`, `fingerprint`) and is easy to export (PEM, DER, JSON) for further analysis or server usage.
+
+Use it whenever you need **insight** into a site’s certificate or require some form of cryptographic or compliance check.
\ No newline at end of file
diff --git a/docs/md_v2/advanced/undetected-browser.md b/docs/md_v2/advanced/undetected-browser.md
new file mode 100644
index 0000000..32b73fc
--- /dev/null
+++ b/docs/md_v2/advanced/undetected-browser.md
@@ -0,0 +1,395 @@
+# Undetected Browser Mode
+
+## Overview
+
+Crawl4AI offers two powerful anti-bot features to help you access websites with bot detection:
+
+1. **Stealth Mode** - Uses playwright-stealth to modify browser fingerprints and behaviors
+2. **Undetected Browser Mode** - Advanced browser adapter with deep-level patches for sophisticated bot detection
+
+This guide covers both features and helps you choose the right approach for your needs.
+
+## Anti-Bot Features Comparison
+
+| Feature | Regular Browser | Stealth Mode | Undetected Browser |
+|---------|----------------|--------------|-------------------|
+| WebDriver Detection | ❌ | ✅ | ✅ |
+| Navigator Properties | ❌ | ✅ | ✅ |
+| Plugin Emulation | ❌ | ✅ | ✅ |
+| CDP Detection | ❌ | Partial | ✅ |
+| Deep Browser Patches | ❌ | ❌ | ✅ |
+| Performance Impact | None | Minimal | Moderate |
+| Setup Complexity | None | None | Minimal |
+
+## When to Use Each Approach
+
+### Use Regular Browser + Stealth Mode When:
+- Sites have basic bot detection (checking navigator.webdriver, plugins, etc.)
+- You need good performance with basic protection
+- Sites check for common automation indicators
+
+### Use Undetected Browser When:
+- Sites employ sophisticated bot detection services (Cloudflare, DataDome, etc.)
+- Stealth mode alone isn't sufficient
+- You're willing to trade some performance for better evasion
+
+### Best Practice: Progressive Enhancement
+1. **Start with**: Regular browser + Stealth mode
+2. **If blocked**: Switch to Undetected browser
+3. **If still blocked**: Combine Undetected browser + Stealth mode
+
+## Stealth Mode
+
+Stealth mode is the simpler anti-bot solution that works with both regular and undetected browsers:
+
+```python
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+# Enable stealth mode with regular browser
+browser_config = BrowserConfig(
+    enable_stealth=True,  # Simple flag to enable
+    headless=False       # Better for avoiding detection
+)
+
+async with AsyncWebCrawler(config=browser_config) as crawler:
+    result = await crawler.arun("https://example.com")
+```
+
+### What Stealth Mode Does:
+- Removes `navigator.webdriver` flag
+- Modifies browser fingerprints
+- Emulates realistic plugin behavior
+- Adjusts navigator properties
+- Fixes common automation leaks
+
+## Undetected Browser Mode
+
+For sites with sophisticated bot detection that stealth mode can't bypass, use the undetected browser adapter:
+
+### Key Features
+
+- **Drop-in Replacement**: Uses the same API as regular browser mode
+- **Enhanced Stealth**: Built-in patches to evade common detection methods
+- **Browser Adapter Pattern**: Seamlessly switch between regular and undetected modes
+- **Automatic Installation**: `crawl4ai-setup` installs all necessary browser dependencies
+
+### Quick Start
+
+```python
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler, 
+    BrowserConfig, 
+    CrawlerRunConfig,
+    UndetectedAdapter
+)
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+async def main():
+    # Create the undetected adapter
+    undetected_adapter = UndetectedAdapter()
+    
+    # Create browser config
+    browser_config = BrowserConfig(
+        headless=False,  # Headless mode can be detected easier
+        verbose=True,
+    )
+    
+    # Create the crawler strategy with undetected adapter
+    crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=undetected_adapter
+    )
+    
+    # Create the crawler with our custom strategy
+    async with AsyncWebCrawler(
+        crawler_strategy=crawler_strategy,
+        config=browser_config
+    ) as crawler:
+        # Your crawling code here
+        result = await crawler.arun(
+            url="https://example.com",
+            config=CrawlerRunConfig()
+        )
+        print(result.markdown[:500])
+
+asyncio.run(main())
+```
+
+## Combining Both Features
+
+For maximum evasion, combine stealth mode with undetected browser:
+
+```python
+from crawl4ai import AsyncWebCrawler, BrowserConfig, UndetectedAdapter
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+# Create browser config with stealth enabled
+browser_config = BrowserConfig(
+    enable_stealth=True,  # Enable stealth mode
+    headless=False
+)
+
+# Create undetected adapter
+adapter = UndetectedAdapter()
+
+# Create strategy with both features
+strategy = AsyncPlaywrightCrawlerStrategy(
+    browser_config=browser_config,
+    browser_adapter=adapter
+)
+
+async with AsyncWebCrawler(
+    crawler_strategy=strategy,
+    config=browser_config
+) as crawler:
+    result = await crawler.arun("https://protected-site.com")
+```
+
+## Examples
+
+### Example 1: Basic Stealth Mode
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def test_stealth_mode():
+    # Simple stealth mode configuration
+    browser_config = BrowserConfig(
+        enable_stealth=True,
+        headless=False
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://bot.sannysoft.com",
+            config=CrawlerRunConfig(screenshot=True)
+        )
+        
+        if result.success:
+            print("✓ Successfully accessed bot detection test site")
+            # Save screenshot to verify detection results
+            if result.screenshot:
+                import base64
+                with open("stealth_test.png", "wb") as f:
+                    f.write(base64.b64decode(result.screenshot))
+                print("✓ Screenshot saved - check for green (passed) tests")
+
+asyncio.run(test_stealth_mode())
+```
+
+### Example 2: Undetected Browser Mode
+
+```python
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    UndetectedAdapter
+)
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+
+async def main():
+    # Create browser config
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True,
+    )
+    
+    # Create the undetected adapter
+    undetected_adapter = UndetectedAdapter()
+    
+    # Create the crawler strategy with the undetected adapter
+    crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=undetected_adapter
+    )
+    
+    # Create the crawler with our custom strategy
+    async with AsyncWebCrawler(
+        crawler_strategy=crawler_strategy,
+        config=browser_config
+    ) as crawler:
+        # Configure the crawl
+        crawler_config = CrawlerRunConfig(
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter()
+            ),
+            capture_console_messages=True,  # Test adapter console capture
+        )
+        
+        # Test on a site that typically detects bots
+        print("Testing undetected adapter...")
+        result: CrawlResult = await crawler.arun(
+            url="https://www.helloworld.org", 
+            config=crawler_config
+        )
+        
+        print(f"Status: {result.status_code}")
+        print(f"Success: {result.success}")
+        print(f"Console messages captured: {len(result.console_messages or [])}")
+        print(f"Markdown content (first 500 chars):\n{result.markdown.raw_markdown[:500]}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Browser Adapter Pattern
+
+The undetected browser support is implemented using an adapter pattern, allowing seamless switching between different browser implementations:
+
+```python
+# Regular browser adapter (default)
+from crawl4ai import PlaywrightAdapter
+regular_adapter = PlaywrightAdapter()
+
+# Undetected browser adapter
+from crawl4ai import UndetectedAdapter
+undetected_adapter = UndetectedAdapter()
+```
+
+The adapter handles:
+- JavaScript execution
+- Console message capture
+- Error handling
+- Browser-specific optimizations
+
+## Best Practices
+
+1. **Avoid Headless Mode**: Detection is easier in headless mode
+   ```python
+   browser_config = BrowserConfig(headless=False)
+   ```
+
+2. **Use Reasonable Delays**: Don't rush through pages
+   ```python
+   crawler_config = CrawlerRunConfig(
+       wait_time=3.0,  # Wait 3 seconds after page load
+       delay_before_return_html=2.0  # Additional delay
+   )
+   ```
+
+3. **Rotate User Agents**: You can customize user agents
+   ```python
+   browser_config = BrowserConfig(
+       headers={"User-Agent": "your-user-agent"}
+   )
+   ```
+
+4. **Handle Failures Gracefully**: Some sites may still detect and block
+   ```python
+   if not result.success:
+       print(f"Crawl failed: {result.error_message}")
+   ```
+
+## Advanced Usage Tips
+
+### Progressive Detection Handling
+
+```python
+async def crawl_with_progressive_evasion(url):
+    # Step 1: Try regular browser with stealth
+    browser_config = BrowserConfig(
+        enable_stealth=True,
+        headless=False
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url)
+        if result.success and "Access Denied" not in result.html:
+            return result
+    
+    # Step 2: If blocked, try undetected browser
+    print("Regular + stealth blocked, trying undetected browser...")
+    
+    adapter = UndetectedAdapter()
+    strategy = AsyncPlaywrightCrawlerStrategy(
+        browser_config=browser_config,
+        browser_adapter=adapter
+    )
+    
+    async with AsyncWebCrawler(
+        crawler_strategy=strategy,
+        config=browser_config
+    ) as crawler:
+        result = await crawler.arun(url)
+        return result
+```
+
+## Installation
+
+The undetected browser dependencies are automatically installed when you run:
+
+```bash
+crawl4ai-setup
+```
+
+This command installs all necessary browser dependencies for both regular and undetected modes.
+
+## Limitations
+
+- **Performance**: Slightly slower than regular mode due to additional patches
+- **Headless Detection**: Some sites can still detect headless mode
+- **Resource Usage**: May use more resources than regular mode
+- **Not 100% Guaranteed**: Advanced anti-bot services are constantly evolving
+
+## Troubleshooting
+
+### Browser Not Found
+
+Run the setup command:
+```bash
+crawl4ai-setup
+```
+
+### Detection Still Occurring
+
+Try combining with other features:
+```python
+crawler_config = CrawlerRunConfig(
+    simulate_user=True,  # Add user simulation
+    magic=True,  # Enable magic mode
+    wait_time=5.0,  # Longer waits
+)
+```
+
+### Performance Issues
+
+If experiencing slow performance:
+```python
+# Use selective undetected mode only for protected sites
+if is_protected_site(url):
+    adapter = UndetectedAdapter()
+else:
+    adapter = PlaywrightAdapter()  # Default adapter
+```
+
+## Future Plans
+
+**Note**: In future versions of Crawl4AI, we may enable stealth mode and undetected browser by default to provide better out-of-the-box success rates. For now, users should explicitly enable these features when needed.
+
+## Conclusion
+
+Crawl4AI provides flexible anti-bot solutions:
+
+1. **Start Simple**: Use regular browser + stealth mode for most sites
+2. **Escalate if Needed**: Switch to undetected browser for sophisticated protection
+3. **Combine for Maximum Effect**: Use both features together when facing the toughest challenges
+
+Remember:
+- Always respect robots.txt and website terms of service
+- Use appropriate delays to avoid overwhelming servers
+- Consider the performance trade-offs of each approach
+- Test progressively to find the minimum necessary evasion level
+
+## See Also
+
+- [Advanced Features](advanced-features.md) - Overview of all advanced features
+- [Proxy & Security](proxy-security.md) - Using proxies with anti-bot features
+- [Session Management](session-management.md) - Maintaining sessions across requests
+- [Identity Based Crawling](identity-based-crawling.md) - Additional anti-detection strategies
+- [Anti-Bot Detection & Fallback](anti-bot-and-fallback.md) - Automatic retry and proxy escalation when blocking is detected
\ No newline at end of file
diff --git a/docs/md_v2/advanced/virtual-scroll.md b/docs/md_v2/advanced/virtual-scroll.md
new file mode 100644
index 0000000..271a956
--- /dev/null
+++ b/docs/md_v2/advanced/virtual-scroll.md
@@ -0,0 +1,309 @@
+# Virtual Scroll
+
+Modern websites increasingly use **virtual scrolling** (also called windowed rendering or viewport rendering) to handle large datasets efficiently. This technique only renders visible items in the DOM, replacing content as users scroll. Popular examples include Twitter's timeline, Instagram's feed, and many data tables.
+
+Crawl4AI's Virtual Scroll feature automatically detects and handles these scenarios, ensuring you capture **all content**, not just what's initially visible.
+
+## Understanding Virtual Scroll
+
+### The Problem
+
+Traditional infinite scroll **appends** new content to existing content. Virtual scroll **replaces** content to maintain performance:
+
+```
+Traditional Scroll:          Virtual Scroll:
+┌─────────────┐             ┌─────────────┐
+│ Item 1      │             │ Item 11     │  <- Items 1-10 removed
+│ Item 2      │             │ Item 12     │  <- Only visible items
+│ ...         │             │ Item 13     │     in DOM
+│ Item 10     │             │ Item 14     │
+│ Item 11 NEW │             │ Item 15     │
+│ Item 12 NEW │             └─────────────┘
+└─────────────┘             
+DOM keeps growing           DOM size stays constant
+```
+
+Without proper handling, crawlers only capture the currently visible items, missing the rest of the content.
+
+### Three Scrolling Scenarios
+
+Crawl4AI's Virtual Scroll detects and handles three scenarios:
+
+1. **No Change** - Content doesn't update on scroll (static page or end reached)
+2. **Content Appended** - New items added to existing ones (traditional infinite scroll)  
+3. **Content Replaced** - Items replaced with new ones (true virtual scroll)
+
+Only scenario 3 requires special handling, which Virtual Scroll automates.
+
+## Basic Usage
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, VirtualScrollConfig
+
+# Configure virtual scroll
+virtual_config = VirtualScrollConfig(
+    container_selector="#feed",      # CSS selector for scrollable container
+    scroll_count=20,                 # Number of scrolls to perform
+    scroll_by="container_height",    # How much to scroll each time
+    wait_after_scroll=0.5           # Wait time (seconds) after each scroll
+)
+
+# Use in crawler configuration
+config = CrawlerRunConfig(
+    virtual_scroll_config=virtual_config
+)
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(url="https://example.com", config=config)
+    # result.html contains ALL items from the virtual scroll
+```
+
+## Configuration Parameters
+
+### VirtualScrollConfig
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `container_selector` | `str` | Required | CSS selector for the scrollable container |
+| `scroll_count` | `int` | `10` | Maximum number of scrolls to perform |
+| `scroll_by` | `str` or `int` | `"container_height"` | Scroll amount per step |
+| `wait_after_scroll` | `float` | `0.5` | Seconds to wait after each scroll |
+
+### Scroll By Options
+
+- `"container_height"` - Scroll by the container's visible height
+- `"page_height"` - Scroll by the viewport height
+- `500` (integer) - Scroll by exact pixel amount
+
+## Real-World Examples
+
+### Twitter-like Timeline
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, VirtualScrollConfig, BrowserConfig
+
+async def crawl_twitter_timeline():
+    # Twitter replaces tweets as you scroll
+    virtual_config = VirtualScrollConfig(
+        container_selector="[data-testid='primaryColumn']",
+        scroll_count=30,
+        scroll_by="container_height",
+        wait_after_scroll=1.0  # Twitter needs time to load
+    )
+    
+    browser_config = BrowserConfig(headless=True)  # Set to False to watch it work
+    config = CrawlerRunConfig(
+        virtual_scroll_config=virtual_config
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://twitter.com/search?q=AI",
+            config=config
+        )
+        
+        # Extract tweet count
+        import re
+        tweets = re.findall(r'data-testid="tweet"', result.html)
+        print(f"Captured {len(tweets)} tweets")
+```
+
+### Instagram Grid
+
+```python
+async def crawl_instagram_grid():
+    # Instagram uses virtualized grid for performance
+    virtual_config = VirtualScrollConfig(
+        container_selector="article",  # Main feed container
+        scroll_count=50,               # More scrolls for grid layout
+        scroll_by=800,                 # Fixed pixel scrolling
+        wait_after_scroll=0.8
+    )
+    
+    config = CrawlerRunConfig(
+        virtual_scroll_config=virtual_config,
+        screenshot=True  # Capture final state
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.instagram.com/explore/tags/photography/",
+            config=config
+        )
+        
+        # Count posts
+        posts = result.html.count('class="post"')
+        print(f"Captured {posts} posts from virtualized grid")
+```
+
+### Mixed Content (News Feed)
+
+Some sites mix static and virtualized content:
+
+```python
+async def crawl_mixed_feed():
+    # Featured articles stay, regular articles virtualize
+    virtual_config = VirtualScrollConfig(
+        container_selector=".main-feed",
+        scroll_count=25,
+        scroll_by="container_height",
+        wait_after_scroll=0.5
+    )
+    
+    config = CrawlerRunConfig(
+        virtual_scroll_config=virtual_config
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://news.example.com",
+            config=config
+        )
+        
+        # Featured articles remain throughout
+        featured = result.html.count('class="featured-article"')
+        regular = result.html.count('class="regular-article"')
+        
+        print(f"Featured (static): {featured}")
+        print(f"Regular (virtualized): {regular}")
+```
+
+## Virtual Scroll vs scan_full_page
+
+Both features handle dynamic content, but serve different purposes:
+
+| Feature | Virtual Scroll | scan_full_page |
+|---------|---------------|----------------|
+| **Purpose** | Capture content that's replaced during scroll | Load content that's appended during scroll |
+| **Use Case** | Twitter, Instagram, virtual tables | Traditional infinite scroll, lazy-loaded images |
+| **DOM Behavior** | Replaces elements | Adds elements |
+| **Memory Usage** | Efficient (merges content) | Can grow large |
+| **Configuration** | Requires container selector | Works on full page |
+
+### When to Use Which?
+
+Use **Virtual Scroll** when:
+- Content disappears as you scroll (Twitter timeline)
+- DOM element count stays relatively constant
+- You need ALL items from a virtualized list
+- Container-based scrolling (not full page)
+
+Use **scan_full_page** when:
+- Content accumulates as you scroll
+- Images load lazily
+- Simple "load more" behavior
+- Full page scrolling
+
+## Combining with Extraction
+
+Virtual Scroll works seamlessly with extraction strategies:
+
+```python
+from crawl4ai import LLMExtractionStrategy, LLMConfig
+
+# Define extraction schema
+schema = {
+    "type": "array",
+    "items": {
+        "type": "object", 
+        "properties": {
+            "author": {"type": "string"},
+            "content": {"type": "string"},
+            "timestamp": {"type": "string"}
+        }
+    }
+}
+
+# Configure both virtual scroll and extraction
+config = CrawlerRunConfig(
+    virtual_scroll_config=VirtualScrollConfig(
+        container_selector="#timeline",
+        scroll_count=20
+    ),
+    extraction_strategy=LLMExtractionStrategy(
+        llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+        schema=schema
+    )
+)
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(url="...", config=config)
+    
+    # Extracted data from ALL scrolled content
+    import json
+    posts = json.loads(result.extracted_content)
+    print(f"Extracted {len(posts)} posts from virtual scroll")
+```
+
+## Performance Tips
+
+1. **Container Selection**: Be specific with selectors. Using the correct container improves performance.
+
+2. **Scroll Count**: Start conservative and increase as needed:
+   ```python
+   # Start with fewer scrolls
+   virtual_config = VirtualScrollConfig(
+       container_selector="#feed",
+       scroll_count=10  # Test with 10, increase if needed
+   )
+   ```
+
+3. **Wait Times**: Adjust based on site speed:
+   ```python
+   # Fast sites
+   wait_after_scroll=0.2
+   
+   # Slower sites or heavy content
+   wait_after_scroll=1.5
+   ```
+
+4. **Debug Mode**: Set `headless=False` to watch scrolling:
+   ```python
+   browser_config = BrowserConfig(headless=False)
+   async with AsyncWebCrawler(config=browser_config) as crawler:
+       # Watch the scrolling happen
+   ```
+
+## How It Works Internally
+
+1. **Detection Phase**: Scrolls and compares HTML to detect behavior
+2. **Capture Phase**: For replaced content, stores HTML chunks at each position
+3. **Merge Phase**: Combines all chunks, removing duplicates based on text content
+4. **Result**: Complete HTML with all unique items
+
+The deduplication uses normalized text (lowercase, no spaces/symbols) to ensure accurate merging without false positives.
+
+## Error Handling
+
+Virtual Scroll handles errors gracefully:
+
+```python
+# If container not found or scrolling fails
+result = await crawler.arun(url="...", config=config)
+
+if result.success:
+    # Virtual scroll worked or wasn't needed
+    print(f"Captured {len(result.html)} characters")
+else:
+    # Crawl failed entirely
+    print(f"Error: {result.error_message}")
+```
+
+If the container isn't found, crawling continues normally without virtual scroll.
+
+## Complete Example
+
+See our [comprehensive example](/docs/examples/virtual_scroll_example.py) that demonstrates:
+- Twitter-like feeds
+- Instagram grids  
+- Traditional infinite scroll
+- Mixed content scenarios
+- Performance comparisons
+
+```bash
+# Run the examples
+cd docs/examples
+python virtual_scroll_example.py
+```
+
+The example includes a local test server with different scrolling behaviors for experimentation.
\ No newline at end of file
diff --git a/docs/md_v2/api/adaptive-crawler.md b/docs/md_v2/api/adaptive-crawler.md
new file mode 100644
index 0000000..5bd5bf4
--- /dev/null
+++ b/docs/md_v2/api/adaptive-crawler.md
@@ -0,0 +1,244 @@
+# AdaptiveCrawler
+
+The `AdaptiveCrawler` class implements intelligent web crawling that automatically determines when sufficient information has been gathered to answer a query. It uses a three-layer scoring system to evaluate coverage, consistency, and saturation.
+
+## Constructor
+
+```python
+AdaptiveCrawler(
+    crawler: AsyncWebCrawler,
+    config: Optional[AdaptiveConfig] = None
+)
+```
+
+### Parameters
+
+- **crawler** (`AsyncWebCrawler`): The underlying web crawler instance to use for fetching pages
+- **config** (`Optional[AdaptiveConfig]`): Configuration settings for adaptive crawling behavior. If not provided, uses default settings.
+
+## Primary Method
+
+### digest()
+
+The main method that performs adaptive crawling starting from a URL with a specific query.
+
+```python
+async def digest(
+    start_url: str,
+    query: str,
+    resume_from: Optional[Union[str, Path]] = None
+) -> CrawlState
+```
+
+#### Parameters
+
+- **start_url** (`str`): The starting URL for crawling
+- **query** (`str`): The search query that guides the crawling process
+- **resume_from** (`Optional[Union[str, Path]]`): Path to a saved state file to resume from
+
+#### Returns
+
+- **CrawlState**: The final crawl state containing all crawled URLs, knowledge base, and metrics
+
+#### Example
+
+```python
+async with AsyncWebCrawler() as crawler:
+    adaptive = AdaptiveCrawler(crawler)
+    state = await adaptive.digest(
+        start_url="https://docs.python.org",
+        query="async context managers"
+    )
+```
+
+## Properties
+
+### confidence
+
+Current confidence score (0-1) indicating information sufficiency.
+
+```python
+@property
+def confidence(self) -> float
+```
+
+### coverage_stats
+
+Dictionary containing detailed coverage statistics.
+
+```python
+@property  
+def coverage_stats(self) -> Dict[str, float]
+```
+
+Returns:
+- **coverage**: Query term coverage score
+- **consistency**: Information consistency score  
+- **saturation**: Content saturation score
+- **confidence**: Overall confidence score
+
+### is_sufficient
+
+Boolean indicating whether sufficient information has been gathered.
+
+```python
+@property
+def is_sufficient(self) -> bool
+```
+
+### state
+
+Access to the current crawl state.
+
+```python
+@property
+def state(self) -> CrawlState
+```
+
+## Methods
+
+### get_relevant_content()
+
+Retrieve the most relevant content from the knowledge base.
+
+```python
+def get_relevant_content(
+    self,
+    top_k: int = 5
+) -> List[Dict[str, Any]]
+```
+
+#### Parameters
+
+- **top_k** (`int`): Number of top relevant documents to return (default: 5)
+
+#### Returns
+
+List of dictionaries containing:
+- **url**: The URL of the page
+- **content**: The page content
+- **score**: Relevance score
+- **metadata**: Additional page metadata
+
+### print_stats()
+
+Display crawl statistics in formatted output.
+
+```python
+def print_stats(
+    self,
+    detailed: bool = False
+) -> None
+```
+
+#### Parameters
+
+- **detailed** (`bool`): If True, shows detailed metrics with colors. If False, shows summary table.
+
+### export_knowledge_base()
+
+Export the collected knowledge base to a JSONL file.
+
+```python
+def export_knowledge_base(
+    self,
+    path: Union[str, Path]
+) -> None
+```
+
+#### Parameters
+
+- **path** (`Union[str, Path]`): Output file path for JSONL export
+
+#### Example
+
+```python
+adaptive.export_knowledge_base("my_knowledge.jsonl")
+```
+
+### import_knowledge_base()
+
+Import a previously exported knowledge base.
+
+```python
+async def import_knowledge_base(
+    self,
+    path: Union[str, Path]
+) -> None
+```
+
+#### Parameters
+
+- **path** (`Union[str, Path]`): Path to JSONL file to import
+
+## Configuration
+
+The `AdaptiveConfig` class controls the behavior of adaptive crawling:
+
+```python
+@dataclass
+class AdaptiveConfig:
+    confidence_threshold: float = 0.8      # Stop when confidence reaches this
+    max_pages: int = 50                    # Maximum pages to crawl
+    top_k_links: int = 5                   # Links to follow per page
+    min_gain_threshold: float = 0.1        # Minimum expected gain to continue
+    save_state: bool = False               # Auto-save crawl state
+    state_path: Optional[str] = None       # Path for state persistence
+```
+
+### Example with Custom Config
+
+```python
+config = AdaptiveConfig(
+    confidence_threshold=0.7,
+    max_pages=20,
+    top_k_links=3
+)
+
+adaptive = AdaptiveCrawler(crawler, config=config)
+```
+
+## Complete Example
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
+
+async def main():
+    # Configure adaptive crawling
+    config = AdaptiveConfig(
+        confidence_threshold=0.75,
+        max_pages=15,
+        save_state=True,
+        state_path="my_crawl.json"
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        adaptive = AdaptiveCrawler(crawler, config)
+        
+        # Start crawling
+        state = await adaptive.digest(
+            start_url="https://example.com/docs",
+            query="authentication oauth2 jwt"
+        )
+        
+        # Check results
+        print(f"Confidence achieved: {adaptive.confidence:.0%}")
+        adaptive.print_stats()
+        
+        # Get most relevant pages
+        for page in adaptive.get_relevant_content(top_k=3):
+            print(f"- {page['url']} (score: {page['score']:.2f})")
+        
+        # Export for later use
+        adaptive.export_knowledge_base("auth_knowledge.jsonl")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## See Also
+
+- [digest() Method Reference](digest.md)
+- [Adaptive Crawling Guide](../core/adaptive-crawling.md)
+- [Advanced Adaptive Strategies](../advanced/adaptive-strategies.md)
\ No newline at end of file
diff --git a/docs/md_v2/api/arun.md b/docs/md_v2/api/arun.md
new file mode 100644
index 0000000..4fc0cb5
--- /dev/null
+++ b/docs/md_v2/api/arun.md
@@ -0,0 +1,309 @@
+# `arun()` Parameter Guide (New Approach)
+
+In Crawl4AI’s **latest** configuration model, nearly all parameters that once went directly to `arun()` are now part of **`CrawlerRunConfig`**. When calling `arun()`, you provide:
+
+```python
+await crawler.arun(
+    url="https://example.com",  
+    config=my_run_config
+)
+```
+
+Below is an organized look at the parameters that can go inside `CrawlerRunConfig`, divided by their functional areas. For **Browser** settings (e.g., `headless`, `browser_type`), see [BrowserConfig](./parameters.md).
+
+---
+
+## 1. Core Usage
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+
+async def main():
+    run_config = CrawlerRunConfig(
+        verbose=True,            # Detailed logging
+        cache_mode=CacheMode.ENABLED,  # Use normal read/write cache
+        check_robots_txt=True,   # Respect robots.txt rules
+        # ... other parameters
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=run_config
+        )
+        
+        # Check if blocked by robots.txt
+        if not result.success and result.status_code == 403:
+            print(f"Error: {result.error_message}")
+```
+
+**Key Fields**:
+- `verbose=True` logs each crawl step.  
+- `cache_mode` decides how to read/write the local crawl cache.
+
+---
+
+## 2. Cache Control
+
+**`cache_mode`** (default: `CacheMode.ENABLED`)  
+Use a built-in enum from `CacheMode`:
+
+- `ENABLED`: Normal caching—reads if available, writes if missing.
+- `DISABLED`: No caching—always refetch pages.
+- `READ_ONLY`: Reads from cache only; no new writes.
+- `WRITE_ONLY`: Writes to cache but doesn’t read existing data.
+- `BYPASS`: Skips reading cache for this crawl (though it might still write if set up that way).
+
+```python
+run_config = CrawlerRunConfig(
+    cache_mode=CacheMode.BYPASS
+)
+```
+
+**Cache modes**:
+
+- `CacheMode.BYPASS` — Skip cache entirely; always fetch fresh, write result to cache.
+- `CacheMode.DISABLED` — No caching at all; don't read or write.
+- `CacheMode.WRITE_ONLY` — Never read from cache, but write results.
+- `CacheMode.READ_ONLY` — Read from cache if available, never write.
+
+---
+
+## 3. Content Processing & Selection
+
+### 3.1 Text Processing
+
+```python
+run_config = CrawlerRunConfig(
+    word_count_threshold=10,   # Ignore text blocks <10 words
+    only_text=False,           # If True, tries to remove non-text elements
+    keep_data_attributes=False # Keep or discard data-* attributes
+)
+```
+
+### 3.2 Content Selection
+
+```python
+run_config = CrawlerRunConfig(
+    css_selector=".main-content",  # Focus on .main-content region only
+    excluded_tags=["form", "nav"], # Remove entire tag blocks
+    remove_forms=True,             # Specifically strip 
 elements
+    remove_overlay_elements=True,  # Attempt to remove modals/popups
+)
+```
+
+### 3.3 Link Handling
+
+```python
+run_config = CrawlerRunConfig(
+    exclude_external_links=True,         # Remove external links from final content
+    exclude_social_media_links=True,     # Remove links to known social sites
+    exclude_domains=["ads.example.com"], # Exclude links to these domains
+    exclude_social_media_domains=["facebook.com","twitter.com"], # Extend the default list
+)
+```
+
+### 3.4 Media Filtering
+
+```python
+run_config = CrawlerRunConfig(
+    exclude_external_images=True  # Strip images from other domains
+)
+```
+
+---
+
+## 4. Page Navigation & Timing
+
+### 4.1 Basic Browser Flow
+
+```python
+run_config = CrawlerRunConfig(
+    wait_for="css:.dynamic-content", # Wait for .dynamic-content
+    delay_before_return_html=2.0,    # Wait 2s before capturing final HTML
+    page_timeout=60000,             # Navigation & script timeout (ms)
+)
+```
+
+**Key Fields**:
+
+- `wait_for`:  
+  - `"css:selector"` or  
+  - `"js:() => boolean"`  
+  e.g. `js:() => document.querySelectorAll('.item').length > 10`.
+
+- `mean_delay` & `max_range`: define random delays for `arun_many()` calls.  
+- `semaphore_count`: concurrency limit when crawling multiple URLs.
+
+### 4.2 JavaScript Execution
+
+```python
+run_config = CrawlerRunConfig(
+    js_code=[
+        "window.scrollTo(0, document.body.scrollHeight);",
+        "document.querySelector('.load-more')?.click();"
+    ],
+    js_only=False
+)
+```
+
+- `js_code` can be a single string or a list of strings.  
+- `js_only=True` means “I’m continuing in the same session with new JS steps, no new full navigation.”
+
+### 4.3 Anti-Bot
+
+```python
+run_config = CrawlerRunConfig(
+    magic=True,
+    simulate_user=True,
+    override_navigator=True
+)
+```
+- `magic=True` tries multiple stealth features.  
+- `simulate_user=True` mimics mouse movements or random delays.  
+- `override_navigator=True` fakes some navigator properties (like user agent checks).
+
+---
+
+## 5. Session Management
+
+**`session_id`**: 
+```python
+run_config = CrawlerRunConfig(
+    session_id="my_session123"
+)
+```
+If re-used in subsequent `arun()` calls, the same tab/page context is continued (helpful for multi-step tasks or stateful browsing).
+
+---
+
+## 6. Screenshot, PDF & Media Options
+
+```python
+run_config = CrawlerRunConfig(
+    screenshot=True,             # Grab a screenshot as base64
+    screenshot_wait_for=1.0,     # Wait 1s before capturing
+    pdf=True,                    # Also produce a PDF
+    image_description_min_word_threshold=5,  # If analyzing alt text
+    image_score_threshold=3,                # Filter out low-score images
+)
+```
+**Where they appear**:
+- `result.screenshot` → Base64 screenshot string.
+- `result.pdf` → Byte array with PDF data.
+
+---
+
+## 7. Extraction Strategy
+
+**For advanced data extraction** (CSS/LLM-based), set `extraction_strategy`:
+
+```python
+run_config = CrawlerRunConfig(
+    extraction_strategy=my_css_or_llm_strategy
+)
+```
+
+The extracted data will appear in `result.extracted_content`.
+
+---
+
+## 8. Comprehensive Example
+
+Below is a snippet combining many parameters:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai import JsonCssExtractionStrategy
+
+async def main():
+    # Example schema
+    schema = {
+        "name": "Articles",
+        "baseSelector": "article.post",
+        "fields": [
+            {"name": "title", "selector": "h2", "type": "text"},
+            {"name": "link",  "selector": "a",  "type": "attribute", "attribute": "href"}
+        ]
+    }
+
+    run_config = CrawlerRunConfig(
+        # Core
+        verbose=True,
+        cache_mode=CacheMode.ENABLED,
+        check_robots_txt=True,   # Respect robots.txt rules
+        
+        # Content
+        word_count_threshold=10,
+        css_selector="main.content",
+        excluded_tags=["nav", "footer"],
+        exclude_external_links=True,
+        
+        # Page & JS
+        js_code="document.querySelector('.show-more')?.click();",
+        wait_for="css:.loaded-block",
+        page_timeout=30000,
+        
+        # Extraction
+        extraction_strategy=JsonCssExtractionStrategy(schema),
+        
+        # Session
+        session_id="persistent_session",
+        
+        # Media
+        screenshot=True,
+        pdf=True,
+        
+        # Anti-bot
+        simulate_user=True,
+        magic=True,
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com/posts", config=run_config)
+        if result.success:
+            print("HTML length:", len(result.cleaned_html))
+            print("Extraction JSON:", result.extracted_content)
+            if result.screenshot:
+                print("Screenshot length:", len(result.screenshot))
+            if result.pdf:
+                print("PDF bytes length:", len(result.pdf))
+        else:
+            print("Error:", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**What we covered**:
+
+1. **Crawling** the main content region, ignoring external links.  
+2. Running **JavaScript** to click “.show-more”.  
+3. **Waiting** for “.loaded-block” to appear.  
+4. Generating a **screenshot** & **PDF** of the final page.  
+5. Extracting repeated “article.post” elements with a **CSS-based** extraction strategy.
+
+---
+
+## 9. Best Practices
+
+1. **Use `BrowserConfig` for global browser** settings (headless, user agent).  
+2. **Use `CrawlerRunConfig`** to handle the **specific** crawl needs: content filtering, caching, JS, screenshot, extraction, etc.  
+3. Keep your **parameters consistent** in run configs—especially if you’re part of a large codebase with multiple crawls.  
+4. **Limit** large concurrency (`semaphore_count`) if the site or your system can’t handle it.  
+5. For dynamic pages, set `js_code` or `scan_full_page` so you load all content.
+
+---
+
+## 10. Conclusion
+
+All parameters that used to be direct arguments to `arun()` now belong in **`CrawlerRunConfig`**. This approach:
+
+- Makes code **clearer** and **more maintainable**.  
+- Minimizes confusion about which arguments affect global vs. per-crawl behavior.  
+- Allows you to create **reusable** config objects for different pages or tasks.
+
+For a **full** reference, check out the [CrawlerRunConfig Docs](./parameters.md). 
+
+Happy crawling with your **structured, flexible** config approach!
\ No newline at end of file
diff --git a/docs/md_v2/api/arun_many.md b/docs/md_v2/api/arun_many.md
new file mode 100644
index 0000000..740dcc2
--- /dev/null
+++ b/docs/md_v2/api/arun_many.md
@@ -0,0 +1,192 @@
+# `arun_many(...)` Reference
+
+> **Note**: This function is very similar to [`arun()`](./arun.md) but focused on **concurrent** or **batch** crawling. If you’re unfamiliar with `arun()` usage, please read that doc first, then review this for differences.
+
+## Function Signature
+
+```python
+async def arun_many(
+    urls: Union[List[str], List[Any]],
+    config: Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]] = None,
+    dispatcher: Optional[BaseDispatcher] = None,
+    ...
+) -> RunManyReturn:
+    """
+    Crawl multiple URLs concurrently or in batches.
+
+    :param urls: A list of URLs (or tasks) to crawl.
+    :param config: (Optional) Either:
+        - A single `CrawlerRunConfig` applying to all URLs
+        - A list of `CrawlerRunConfig` objects with url_matcher patterns
+    :param dispatcher: (Optional) A concurrency controller (e.g. MemoryAdaptiveDispatcher).
+    ...
+    :return: RunManyReturn containing either a list of `CrawlResult` objects or an async generator if streaming is enabled.
+    """
+```
+
+## Differences from `arun()`
+
+1. **Multiple URLs**:  
+   
+   - Instead of crawling a single URL, you pass a list of them (strings or tasks).  
+   - The function returns `RunManyReturn` which contains either a **list** of `CrawlResult` or an **async generator** if streaming is enabled.
+
+2. **Concurrency & Dispatchers**:  
+
+   - **`dispatcher`** param allows advanced concurrency control.  
+   - If omitted, a default dispatcher (like `MemoryAdaptiveDispatcher`) is used internally.  
+   - Dispatchers handle concurrency, rate limiting, and memory-based adaptive throttling (see [Multi-URL Crawling](../advanced/multi-url-crawling.md)).
+
+3. **Streaming Support**:  
+
+   - Enable streaming by setting `stream=True` in your `CrawlerRunConfig`.
+   - When streaming, use `async for` to process results as they become available.
+   - Ideal for processing large numbers of URLs without waiting for all to complete.
+
+4. **Parallel** Execution**:  
+
+   - `arun_many()` can run multiple requests concurrently under the hood.  
+   - Each `CrawlResult` might also include a **`dispatch_result`** with concurrency details (like memory usage, start/end times).
+
+### Basic Example (Batch Mode)
+
+```python
+# Minimal usage: The default dispatcher will be used
+results = await crawler.arun_many(
+    urls=["https://site1.com", "https://site2.com"],
+    config=CrawlerRunConfig(stream=False)  # Default behavior
+)
+
+for res in results:
+    if res.success:
+        print(res.url, "crawled OK!")
+    else:
+        print("Failed:", res.url, "-", res.error_message)
+```
+
+### Streaming Example
+
+```python
+config = CrawlerRunConfig(
+    stream=True,  # Enable streaming mode
+    cache_mode=CacheMode.BYPASS
+)
+
+# Process results as they complete
+async for result in await crawler.arun_many(
+    urls=["https://site1.com", "https://site2.com", "https://site3.com"],
+    config=config
+):
+    if result.success:
+        print(f"Just completed: {result.url}")
+        # Process each result immediately
+        process_result(result)
+```
+
+### With a Custom Dispatcher
+
+```python
+dispatcher = MemoryAdaptiveDispatcher(
+    memory_threshold_percent=70.0,
+    max_session_permit=10
+)
+results = await crawler.arun_many(
+    urls=["https://site1.com", "https://site2.com", "https://site3.com"],
+    config=my_run_config,
+    dispatcher=dispatcher
+)
+```
+
+### URL-Specific Configurations
+
+Instead of using one config for all URLs, provide a list of configs with `url_matcher` patterns:
+
+```python
+from crawl4ai import CrawlerRunConfig, MatchMode
+from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+# PDF files - specialized extraction
+pdf_config = CrawlerRunConfig(
+    url_matcher="*.pdf",
+    scraping_strategy=PDFContentScrapingStrategy()
+)
+
+# Blog/article pages - content filtering
+blog_config = CrawlerRunConfig(
+    url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
+    markdown_generator=DefaultMarkdownGenerator(
+        content_filter=PruningContentFilter(threshold=0.48)
+    )
+)
+
+# Dynamic pages - JavaScript execution
+github_config = CrawlerRunConfig(
+    url_matcher=lambda url: 'github.com' in url,
+    js_code="window.scrollTo(0, 500);"
+)
+
+# API endpoints - JSON extraction
+api_config = CrawlerRunConfig(
+    url_matcher=lambda url: 'api' in url or url.endswith('.json'),
+    # Custome settings for JSON extraction
+)
+
+# Default fallback config
+default_config = CrawlerRunConfig()  # No url_matcher means it never matches except as fallback
+
+# Pass the list of configs - first match wins!
+results = await crawler.arun_many(
+    urls=[
+        "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",  # → pdf_config
+        "https://blog.python.org/",  # → blog_config
+        "https://github.com/microsoft/playwright",  # → github_config
+        "https://httpbin.org/json",  # → api_config
+        "https://example.com/"  # → default_config
+    ],
+    config=[pdf_config, blog_config, github_config, api_config, default_config]
+)
+```
+
+**URL Matching Features**:
+- **String patterns**: `"*.pdf"`, `"*/blog/*"`, `"*python.org*"`
+- **Function matchers**: `lambda url: 'api' in url`
+- **Mixed patterns**: Combine strings and functions with `MatchMode.OR` or `MatchMode.AND`
+- **First match wins**: Configs are evaluated in order
+
+**Key Points**:
+- Each URL is processed by the same or separate sessions, depending on the dispatcher’s strategy.
+- `dispatch_result` in each `CrawlResult` (if using concurrency) can hold memory and timing info.  
+- If you need to handle authentication or session IDs, pass them in each individual task or within your run config.
+- **Important**: Always include a default config (without `url_matcher`) as the last item if you want to handle all URLs. Otherwise, unmatched URLs will fail.
+
+### Return Value
+
+Returns a **`RunManyReturn`** object which contains either a **list** of [`CrawlResult`](./crawl-result.md) objects, or an **async generator** if streaming is enabled. You can iterate to check `result.success` or read each item’s `extracted_content`, `markdown`, or `dispatch_result`.
+
+---
+
+## Dispatcher Reference
+
+- **`MemoryAdaptiveDispatcher`**: Dynamically manages concurrency based on system memory usage.  
+- **`SemaphoreDispatcher`**: Fixed concurrency limit, simpler but less adaptive.  
+
+For advanced usage or custom settings, see [Multi-URL Crawling with Dispatchers](../advanced/multi-url-crawling.md).
+
+---
+
+## Common Pitfalls
+
+1. **Large Lists**: If you pass thousands of URLs, be mindful of memory or rate-limits. A dispatcher can help.  
+
+2. **Session Reuse**: If you need specialized logins or persistent contexts, ensure your dispatcher or tasks handle sessions accordingly.  
+
+3. **Error Handling**: Each `CrawlResult` might fail for different reasons—always check `result.success` or the `error_message` before proceeding.
+
+---
+
+## Conclusion
+
+Use `arun_many()` when you want to **crawl multiple URLs** simultaneously or in controlled parallel tasks. If you need advanced concurrency features (like memory-based adaptive throttling or complex rate-limiting), provide a **dispatcher**. Each result is a standard `CrawlResult`, possibly augmented with concurrency stats (`dispatch_result`) for deeper inspection. For more details on concurrency logic and dispatchers, see the [Advanced Multi-URL Crawling](../advanced/multi-url-crawling.md) docs.
\ No newline at end of file
diff --git a/docs/md_v2/api/async-webcrawler.md b/docs/md_v2/api/async-webcrawler.md
new file mode 100644
index 0000000..6bc8f36
--- /dev/null
+++ b/docs/md_v2/api/async-webcrawler.md
@@ -0,0 +1,311 @@
+# AsyncWebCrawler
+
+The **`AsyncWebCrawler`** is the core class for asynchronous web crawling in Crawl4AI. You typically create it **once**, optionally customize it with a **`BrowserConfig`** (e.g., headless, user agent), then **run** multiple **`arun()`** calls with different **`CrawlerRunConfig`** objects.
+
+**Recommended usage**:
+
+1. **Create** a `BrowserConfig` for global browser settings.  
+
+2. **Instantiate** `AsyncWebCrawler(config=browser_config)`.  
+
+3. **Use** the crawler in an async context manager (`async with`) or manage start/close manually.  
+
+4. **Call** `arun(url, config=crawler_run_config)` for each page you want.
+
+---
+
+## 1. Constructor Overview
+
+```python
+class AsyncWebCrawler:
+    def __init__(
+        self,
+        crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
+        config: Optional[BrowserConfig] = None,
+        always_bypass_cache: bool = False,           # deprecated
+        always_by_pass_cache: Optional[bool] = None, # also deprecated
+        base_directory: str = ...,
+        thread_safe: bool = False,
+        **kwargs,
+    ):
+        """
+        Create an AsyncWebCrawler instance.
+
+        Args:
+            crawler_strategy: 
+                (Advanced) Provide a custom crawler strategy if needed.
+            config: 
+                A BrowserConfig object specifying how the browser is set up.
+            always_bypass_cache: 
+                (Deprecated) Use CrawlerRunConfig.cache_mode instead.
+            base_directory:     
+                Folder for storing caches/logs (if relevant).
+            thread_safe: 
+                If True, attempts some concurrency safeguards. Usually False.
+            **kwargs: 
+                Additional legacy or debugging parameters.
+        """
+    )
+
+### Typical Initialization
+
+```python
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+browser_cfg = BrowserConfig(
+    browser_type="chromium",
+    headless=True,
+    verbose=True
+)
+
+crawler = AsyncWebCrawler(config=browser_cfg)
+```
+
+**Notes**:
+
+- **Legacy** parameters like `always_bypass_cache` remain for backward compatibility, but prefer to set **caching** in `CrawlerRunConfig`.
+
+---
+
+## 2. Lifecycle: Start/Close or Context Manager
+
+### 2.1 Context Manager (Recommended)
+
+```python
+async with AsyncWebCrawler(config=browser_cfg) as crawler:
+    result = await crawler.arun("https://example.com")
+    # The crawler automatically starts/closes resources
+```
+
+When the `async with` block ends, the crawler cleans up (closes the browser, etc.).
+
+### 2.2 Manual Start & Close
+
+```python
+crawler = AsyncWebCrawler(config=browser_cfg)
+await crawler.start()
+
+result1 = await crawler.arun("https://example.com")
+result2 = await crawler.arun("https://another.com")
+
+await crawler.close()
+```
+
+Use this style if you have a **long-running** application or need full control of the crawler’s lifecycle.
+
+---
+
+## 3. Primary Method: `arun()`
+
+```python
+async def arun(
+    self,
+    url: str,
+    config: Optional[CrawlerRunConfig] = None,
+    # Legacy parameters for backward compatibility...
+) -> RunManyReturn:
+    ...
+```
+
+### 3.1 New Approach
+
+You pass a `CrawlerRunConfig` object that sets up everything about a crawl—content filtering, caching, session reuse, JS code, screenshots, etc.
+
+```python
+import asyncio
+from crawl4ai import CrawlerRunConfig, CacheMode
+
+run_cfg = CrawlerRunConfig(
+    cache_mode=CacheMode.BYPASS,
+    css_selector="main.article",
+    word_count_threshold=10,
+    screenshot=True
+)
+
+async with AsyncWebCrawler(config=browser_cfg) as crawler:
+    result = await crawler.arun("https://example.com/news", config=run_cfg)
+    print("Crawled HTML length:", len(result.cleaned_html))
+    if result.screenshot:
+        print("Screenshot base64 length:", len(result.screenshot))
+```
+
+### 3.2 Legacy Parameters Still Accepted
+
+For **backward** compatibility, `arun()` can still accept direct arguments like `css_selector=...`, `word_count_threshold=...`, etc., but we strongly advise migrating them into a **`CrawlerRunConfig`**.
+
+---
+
+## 4. Batch Processing: `arun_many()`
+
+```python
+async def arun_many(
+    self,
+    urls: List[str],
+    config: Optional[CrawlerRunConfig] = None,
+    # Legacy parameters maintained for backwards compatibility...
+) -> RunManyReturn:
+    """
+    Process multiple URLs with intelligent rate limiting and resource monitoring.
+    """
+```
+
+### 4.1 Resource-Aware Crawling
+
+The `arun_many()` method now uses an intelligent dispatcher that:
+
+- Monitors system memory usage
+- Implements adaptive rate limiting
+- Provides detailed progress monitoring
+- Manages concurrent crawls efficiently
+
+### 4.2 Example Usage
+
+Check page [Multi-url Crawling](../advanced/multi-url-crawling.md) for a detailed example of how to use `arun_many()`.
+
+```python
+
+### 4.3 Key Features
+
+1. **Rate Limiting**
+   
+   - Automatic delay between requests
+   - Exponential backoff on rate limit detection
+   - Domain-specific rate limiting
+   - Configurable retry strategy
+
+2. **Resource Monitoring**
+
+   - Memory usage tracking
+   - Adaptive concurrency based on system load
+   - Automatic pausing when resources are constrained
+
+3. **Progress Monitoring**
+
+   - Detailed or aggregated progress display
+   - Real-time status updates
+   - Memory usage statistics
+
+4. **Error Handling**
+
+   - Graceful handling of rate limits
+   - Automatic retries with backoff
+   - Detailed error reporting
+
+---
+
+## 5. `CrawlResult` Output
+
+Each `arun()` returns a **`CrawlResult`** containing:
+
+- `url`: Final URL (if redirected).
+- `html`: Original HTML.
+- `cleaned_html`: Sanitized HTML.
+- `markdown_v2`: Removed in v0.5. Accessing it raises `AttributeError`; use `markdown`.
+- `extracted_content`: If an extraction strategy was used (JSON for CSS/LLM strategies).
+- `screenshot`, `pdf`: If screenshots/PDF requested.
+- `media`, `links`: Information about discovered images/links.
+- `success`, `error_message`: Status info.
+
+For details, see [CrawlResult doc](./crawl-result.md).
+
+---
+
+## 6. Quick Example
+
+Below is an example hooking it all together:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai import JsonCssExtractionStrategy
+import json
+
+async def main():
+    # 1. Browser config
+    browser_cfg = BrowserConfig(
+        browser_type="firefox",
+        headless=False,
+        verbose=True
+    )
+
+    # 2. Run config
+    schema = {
+        "name": "Articles",
+        "baseSelector": "article.post",
+        "fields": [
+            {
+                "name": "title", 
+                "selector": "h2", 
+                "type": "text"
+            },
+            {
+                "name": "url", 
+                "selector": "a", 
+                "type": "attribute", 
+                "attribute": "href"
+            }
+        ]
+    }
+
+    run_cfg = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=JsonCssExtractionStrategy(schema),
+        word_count_threshold=15,
+        remove_overlay_elements=True,
+        wait_for="css:.post"  # Wait for posts to appear
+    )
+
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        result = await crawler.arun(
+            url="https://example.com/blog",
+            config=run_cfg
+        )
+
+        if result.success:
+            print("Cleaned HTML length:", len(result.cleaned_html))
+            if result.extracted_content:
+                articles = json.loads(result.extracted_content)
+                print("Extracted articles:", articles[:2])
+        else:
+            print("Error:", result.error_message)
+
+asyncio.run(main())
+```
+
+**Explanation**:
+
+- We define a **`BrowserConfig`** with Firefox, no headless, and `verbose=True`.  
+- We define a **`CrawlerRunConfig`** that **bypasses cache**, uses a **CSS** extraction schema, has a `word_count_threshold=15`, etc.  
+- We pass them to `AsyncWebCrawler(config=...)` and `arun(url=..., config=...)`.
+
+---
+
+## 7. Best Practices & Migration Notes
+
+1. **Use** `BrowserConfig` for **global** settings about the browser’s environment.  
+2. **Use** `CrawlerRunConfig` for **per-crawl** logic (caching, content filtering, extraction strategies, wait conditions).  
+3. **Avoid** legacy parameters like `css_selector` or `word_count_threshold` directly in `arun()`. Instead:
+
+   ```python
+   run_cfg = CrawlerRunConfig(css_selector=".main-content", word_count_threshold=20)
+   result = await crawler.arun(url="...", config=run_cfg)
+   ```
+
+4. **Context Manager** usage is simplest unless you want a persistent crawler across many calls.
+
+---
+
+## 8. Summary
+
+**AsyncWebCrawler** is your entry point to asynchronous crawling:
+
+- **Constructor** accepts **`BrowserConfig`** (or defaults).  
+- **`arun(url, config=CrawlerRunConfig)`** is the main method for single-page crawls.  
+- **`arun_many(urls, config=CrawlerRunConfig)`** handles concurrency across multiple URLs.  
+- For advanced lifecycle control, use `start()` and `close()` explicitly.  
+
+**Migration**:  
+
+- If you used `AsyncWebCrawler(browser_type="chromium", css_selector="...")`, move browser settings to `BrowserConfig(...)` and content/crawl logic to `CrawlerRunConfig(...)`.
+
+This modular approach ensures your code is **clean**, **scalable**, and **easy to maintain**. For any advanced or rarely used parameters, see the [BrowserConfig docs](../api/parameters.md).
diff --git a/docs/md_v2/api/c4a-script-reference.md b/docs/md_v2/api/c4a-script-reference.md
new file mode 100644
index 0000000..57de39b
--- /dev/null
+++ b/docs/md_v2/api/c4a-script-reference.md
@@ -0,0 +1,992 @@
+# C4A-Script API Reference
+
+Complete reference for all C4A-Script commands, syntax, and advanced features.
+
+## Command Categories
+
+### 🧭 Navigation Commands
+
+Navigate between pages and manage browser history.
+
+#### `GO `
+Navigate to a specific URL.
+
+**Syntax:**
+```c4a
+GO 
+```
+
+**Parameters:**
+- `url` - Target URL (string)
+
+**Examples:**
+```c4a
+GO https://example.com
+GO https://api.example.com/login
+GO /relative/path
+```
+
+**Notes:**
+- Supports both absolute and relative URLs
+- Automatically handles protocol detection
+- Waits for page load to complete
+
+---
+
+#### `RELOAD`
+Refresh the current page.
+
+**Syntax:**
+```c4a
+RELOAD
+```
+
+**Examples:**
+```c4a
+RELOAD
+```
+
+**Notes:**
+- Equivalent to pressing F5 or clicking browser refresh
+- Waits for page reload to complete
+- Preserves current URL
+
+---
+
+#### `BACK`
+Navigate back in browser history.
+
+**Syntax:**
+```c4a
+BACK
+```
+
+**Examples:**
+```c4a
+BACK
+```
+
+**Notes:**
+- Equivalent to clicking browser back button
+- Does nothing if no previous page exists
+- Waits for navigation to complete
+
+---
+
+#### `FORWARD`
+Navigate forward in browser history.
+
+**Syntax:**
+```c4a
+FORWARD
+```
+
+**Examples:**
+```c4a
+FORWARD
+```
+
+**Notes:**
+- Equivalent to clicking browser forward button
+- Does nothing if no next page exists
+- Waits for navigation to complete
+
+### ⏱️ Wait Commands
+
+Control timing and synchronization with page elements.
+
+#### `WAIT 

+
+
+
+
\ No newline at end of file
diff --git a/docs/md_v2/core/local-files.md b/docs/md_v2/core/local-files.md
new file mode 100644
index 0000000..a1692bf
--- /dev/null
+++ b/docs/md_v2/core/local-files.md
@@ -0,0 +1,158 @@
+# Prefix-Based Input Handling in Crawl4AI
+
+This guide will walk you through using the Crawl4AI library to crawl web pages, local HTML files, and raw HTML strings. We'll demonstrate these capabilities using a Wikipedia page as an example.
+
+## Crawling a Web URL
+
+To crawl a live web page, provide the URL starting with `http://` or `https://`, using a `CrawlerRunConfig` object:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
+
+async def crawl_web():
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://en.wikipedia.org/wiki/apple", 
+            config=config
+        )
+        if result.success:
+            print("Markdown Content:")
+            print(result.markdown)
+        else:
+            print(f"Failed to crawl: {result.error_message}")
+
+asyncio.run(crawl_web())
+```
+
+## Crawling a Local HTML File
+
+To crawl a local HTML file, prefix the file path with `file://`.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
+
+async def crawl_local_file():
+    local_file_path = "/path/to/apple.html"  # Replace with your file path
+    file_url = f"file://{local_file_path}"
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url=file_url, config=config)
+        if result.success:
+            print("Markdown Content from Local File:")
+            print(result.markdown)
+        else:
+            print(f"Failed to crawl local file: {result.error_message}")
+
+asyncio.run(crawl_local_file())
+```
+
+## Crawling Raw HTML Content
+
+To crawl raw HTML content, prefix the HTML string with `raw:`.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def crawl_raw_html():
+    raw_html = "
Hello, World!
"
+    raw_html_url = f"raw:{raw_html}"
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url=raw_html_url, config=config)
+        if result.success:
+            print("Markdown Content from Raw HTML:")
+            print(result.markdown)
+        else:
+            print(f"Failed to crawl raw HTML: {result.error_message}")
+
+asyncio.run(crawl_raw_html())
+```
+
+---
+
+# Complete Example
+
+Below is a comprehensive script that:
+
+1. Crawls the Wikipedia page for "Apple."
+2. Saves the HTML content to a local file (`apple.html`).
+3. Crawls the local HTML file and verifies the markdown length matches the original crawl.
+4. Crawls the raw HTML content from the saved file and verifies consistency.
+
+```python
+import os
+import sys
+import asyncio
+from pathlib import Path
+from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
+
+async def main():
+    wikipedia_url = "https://en.wikipedia.org/wiki/apple"
+    script_dir = Path(__file__).parent
+    html_file_path = script_dir / "apple.html"
+
+    async with AsyncWebCrawler() as crawler:
+        # Step 1: Crawl the Web URL
+        print("\n=== Step 1: Crawling the Wikipedia URL ===")
+        web_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+        result = await crawler.arun(url=wikipedia_url, config=web_config)
+
+        if not result.success:
+            print(f"Failed to crawl {wikipedia_url}: {result.error_message}")
+            return
+
+        with open(html_file_path, 'w', encoding='utf-8') as f:
+            f.write(result.html)
+        web_crawl_length = len(result.markdown)
+        print(f"Length of markdown from web crawl: {web_crawl_length}\n")
+
+        # Step 2: Crawl from the Local HTML File
+        print("=== Step 2: Crawling from the Local HTML File ===")
+        file_url = f"file://{html_file_path.resolve()}"
+        file_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+        local_result = await crawler.arun(url=file_url, config=file_config)
+
+        if not local_result.success:
+            print(f"Failed to crawl local file {file_url}: {local_result.error_message}")
+            return
+
+        local_crawl_length = len(local_result.markdown)
+        assert web_crawl_length == local_crawl_length, "Markdown length mismatch"
+        print("✅ Markdown length matches between web and local file crawl.\n")
+
+        # Step 3: Crawl Using Raw HTML Content
+        print("=== Step 3: Crawling Using Raw HTML Content ===")
+        with open(html_file_path, 'r', encoding='utf-8') as f:
+            raw_html_content = f.read()
+        raw_html_url = f"raw:{raw_html_content}"
+        raw_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+        raw_result = await crawler.arun(url=raw_html_url, config=raw_config)
+
+        if not raw_result.success:
+            print(f"Failed to crawl raw HTML content: {raw_result.error_message}")
+            return
+
+        raw_crawl_length = len(raw_result.markdown)
+        assert web_crawl_length == raw_crawl_length, "Markdown length mismatch"
+        print("✅ Markdown length matches between web and raw HTML crawl.\n")
+
+        print("All tests passed successfully!")
+    if html_file_path.exists():
+        os.remove(html_file_path)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+# Conclusion
+
+With the unified `url` parameter and prefix-based handling in **Crawl4AI**, you can seamlessly handle web URLs, local HTML files, and raw HTML content. Use `CrawlerRunConfig` for flexible and consistent configuration in all scenarios.
\ No newline at end of file
diff --git a/docs/md_v2/core/markdown-generation.md b/docs/md_v2/core/markdown-generation.md
new file mode 100644
index 0000000..af9b35b
--- /dev/null
+++ b/docs/md_v2/core/markdown-generation.md
@@ -0,0 +1,504 @@
+# Markdown Generation Basics
+
+One of Crawl4AI’s core features is generating **clean, structured markdown** from web pages. Originally built to solve the problem of extracting only the “actual” content and discarding boilerplate or noise, Crawl4AI’s markdown system remains one of its biggest draws for AI workflows.
+
+In this tutorial, you’ll learn:
+
+1. How to configure the **Default Markdown Generator**  
+2. How **content filters** (BM25 or Pruning) help you refine markdown and discard junk  
+3. The difference between raw markdown (`result.markdown`) and filtered markdown (`fit_markdown`)  
+
+> **Prerequisites**  
+> - You’ve completed or read [AsyncWebCrawler Basics](../core/simple-crawling.md) to understand how to run a simple crawl.  
+> - You know how to configure `CrawlerRunConfig`.
+
+---
+
+## 1. Quick Example
+
+Here’s a minimal code snippet that uses the **DefaultMarkdownGenerator** with no additional filtering:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def main():
+    config = CrawlerRunConfig(
+        markdown_generator=DefaultMarkdownGenerator()
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com", config=config)
+        
+        if result.success:
+            print("Raw Markdown Output:\n")
+            print(result.markdown)  # The unfiltered markdown from the page
+        else:
+            print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**What’s happening?**  
+- `CrawlerRunConfig( markdown_generator = DefaultMarkdownGenerator() )` instructs Crawl4AI to convert the final HTML into markdown at the end of each crawl.  
+- The resulting markdown is accessible via `result.markdown`.
+
+---
+
+## 2. How Markdown Generation Works
+
+### 2.1 HTML-to-Text Conversion (Forked & Modified)
+
+Under the hood, **DefaultMarkdownGenerator** uses a specialized HTML-to-text approach that:
+
+- Preserves headings, code blocks, bullet points, etc.  
+- Removes extraneous tags (scripts, styles) that don’t add meaningful content.  
+- Can optionally generate references for links or skip them altogether.
+
+A set of **options** (passed as a dict) allows you to customize precisely how HTML converts to markdown. These map to standard html2text-like configuration plus your own enhancements (e.g., ignoring internal links, preserving certain tags verbatim, or adjusting line widths).
+
+### 2.2 Link Citations & References
+
+By default, the generator can convert `` elements into `[text][1]` citations, then place the actual links at the bottom of the document. This is handy for research workflows that demand references in a structured manner.
+
+### 2.3 Optional Content Filters
+
+Before or after the HTML-to-Markdown step, you can apply a **content filter** (like BM25 or Pruning) to reduce noise and produce a “fit_markdown”—a heavily pruned version focusing on the page’s main text. We’ll cover these filters shortly.
+
+---
+
+## 3. Configuring the Default Markdown Generator
+
+You can tweak the output by passing an `options` dict to `DefaultMarkdownGenerator`. For example:
+
+```python
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    # Example: ignore all links, don't escape HTML, and wrap text at 80 characters
+    md_generator = DefaultMarkdownGenerator(
+        options={
+            "ignore_links": True,
+            "escape_html": False,
+            "body_width": 80
+        }
+    )
+
+    config = CrawlerRunConfig(
+        markdown_generator=md_generator
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com/docs", config=config)
+        if result.success:
+            print("Markdown:\n", result.markdown[:500])  # Just a snippet
+        else:
+            print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
+```
+
+Some commonly used `options`:
+
+- **`ignore_links`** (bool): Whether to remove all hyperlinks in the final markdown.  
+- **`ignore_images`** (bool): Remove all `![image]()` references.  
+- **`escape_html`** (bool): Turn HTML entities into text (default is often `True`).  
+- **`body_width`** (int): Wrap text at N characters. `0` or `None` means no wrapping.  
+- **`skip_internal_links`** (bool): If `True`, omit `#localAnchors` or internal links referencing the same page.  
+- **`include_sup_sub`** (bool): Attempt to handle `` / `` in a more readable way.
+
+## 4. Selecting the HTML Source for Markdown Generation
+
+The `content_source` parameter allows you to control which HTML content is used as input for markdown generation. This gives you flexibility in how the HTML is processed before conversion to markdown.
+
+```python
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    # Option 1: Use the raw HTML directly from the webpage (before any processing)
+    raw_md_generator = DefaultMarkdownGenerator(
+        content_source="raw_html",
+        options={"ignore_links": True}
+    )
+    
+    # Option 2: Use the cleaned HTML (after scraping strategy processing - default)
+    cleaned_md_generator = DefaultMarkdownGenerator(
+        content_source="cleaned_html",  # This is the default
+        options={"ignore_links": True}
+    )
+    
+    # Option 3: Use preprocessed HTML optimized for schema extraction
+    fit_md_generator = DefaultMarkdownGenerator(
+        content_source="fit_html",
+        options={"ignore_links": True}
+    )
+    
+    # Use one of the generators in your crawler config
+    config = CrawlerRunConfig(
+        markdown_generator=raw_md_generator  # Try each of the generators
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com", config=config)
+        if result.success:
+            print("Markdown:\n", result.markdown.raw_markdown[:500])
+        else:
+            print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
+```
+
+### HTML Source Options
+
+- **`"cleaned_html"`** (default): Uses the HTML after it has been processed by the scraping strategy. This HTML is typically cleaner and more focused on content, with some boilerplate removed.
+
+- **`"raw_html"`**: Uses the original HTML directly from the webpage, before any cleaning or processing. This preserves more of the original content, but may include navigation bars, ads, footers, and other elements that might not be relevant to the main content.
+
+- **`"fit_html"`**: Uses HTML preprocessed for schema extraction. This HTML is optimized for structured data extraction and may have certain elements simplified or removed.
+
+### When to Use Each Option
+
+- Use **`"cleaned_html"`** (default) for most cases where you want a balance of content preservation and noise removal.
+- Use **`"raw_html"`** when you need to preserve all original content, or when the cleaning process is removing content you actually want to keep.
+- Use **`"fit_html"`** when working with structured data or when you need HTML that's optimized for schema extraction.
+
+---
+
+## 5. Content Filters
+
+**Content filters** selectively remove or rank sections of text before turning them into Markdown. This is especially helpful if your page has ads, nav bars, or other clutter you don’t want.
+
+### 5.1 BM25ContentFilter
+
+If you have a **search query**, BM25 is a good choice:
+
+```python
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+from crawl4ai import CrawlerRunConfig
+
+bm25_filter = BM25ContentFilter(
+    user_query="machine learning",
+    bm25_threshold=1.2,
+    language="english"
+)
+
+md_generator = DefaultMarkdownGenerator(
+    content_filter=bm25_filter,
+    options={"ignore_links": True}
+)
+
+config = CrawlerRunConfig(markdown_generator=md_generator)
+```
+
+- **`user_query`**: The term you want to focus on. BM25 tries to keep only content blocks relevant to that query.  
+- **`bm25_threshold`**: Raise it to keep fewer blocks; lower it to keep more.  
+- **`use_stemming`** *(default `True`)*: Whether to apply stemming to the query and content.
+- **`language (str)`**: Language for stemming (default: 'english').
+
+**No query provided?** BM25 tries to glean a context from page metadata, or you can simply treat it as a scorched-earth approach that discards text with low generic score. Realistically, you want to supply a query for best results.
+
+### 5.2 PruningContentFilter
+
+If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
+
+```python
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+prune_filter = PruningContentFilter(
+    threshold=0.5,
+    threshold_type="fixed",  # or "dynamic"
+    min_word_threshold=50
+)
+```
+
+- **`threshold`**: Score boundary. Blocks below this score get removed.  
+- **`threshold_type`**:  
+    - `"fixed"`: Straight comparison (`score >= threshold` keeps the block).  
+    - `"dynamic"`: The filter adjusts threshold in a data-driven manner.  
+- **`min_word_threshold`**: Discard blocks under N words as likely too short or unhelpful.
+
+**When to Use PruningContentFilter**  
+- You want a broad cleanup without a user query.  
+- The page has lots of repeated sidebars, footers, or disclaimers that hamper text extraction.
+
+### 5.3 LLMContentFilter
+
+For intelligent content filtering and high-quality markdown generation, you can use the **LLMContentFilter**. This filter leverages LLMs to generate relevant markdown while preserving the original content's meaning and structure:
+
+```python
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, LLMConfig, DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import LLMContentFilter
+
+async def main():
+    # Initialize LLM filter with specific instruction
+    filter = LLMContentFilter(
+        llm_config = LLMConfig(provider="openai/gpt-4o",api_token="your-api-token"), #or use environment variable
+        instruction="""
+        Focus on extracting the core educational content.
+        Include:
+        - Key concepts and explanations
+        - Important code examples
+        - Essential technical details
+        Exclude:
+        - Navigation elements
+        - Sidebars
+        - Footer content
+        Format the output as clean markdown with proper code blocks and headers.
+        """,
+        chunk_token_threshold=4096,  # Adjust based on your needs
+        verbose=True
+    )
+    md_generator = DefaultMarkdownGenerator(
+        content_filter=filter,
+        options={"ignore_links": True}
+    )
+    config = CrawlerRunConfig(
+        markdown_generator=md_generator,
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com", config=config)
+        print(result.markdown.fit_markdown)  # Filtered markdown content
+```
+
+**Key Features:**
+- **Intelligent Filtering**: Uses LLMs to understand and extract relevant content while maintaining context
+- **Customizable Instructions**: Tailor the filtering process with specific instructions
+- **Chunk Processing**: Handles large documents by processing them in chunks (controlled by `chunk_token_threshold`)
+- **Parallel Processing**: For better performance, use smaller `chunk_token_threshold` (e.g., 2048 or 4096) to enable parallel processing of content chunks
+
+**Two Common Use Cases:**
+
+1. **Exact Content Preservation**:
+```python
+filter = LLMContentFilter(
+    instruction="""
+    Extract the main educational content while preserving its original wording and substance completely.
+    1. Maintain the exact language and terminology
+    2. Keep all technical explanations and examples intact
+    3. Preserve the original flow and structure
+    4. Remove only clearly irrelevant elements like navigation menus and ads
+    """,
+    chunk_token_threshold=4096
+)
+```
+
+2. **Focused Content Extraction**:
+```python
+filter = LLMContentFilter(
+    instruction="""
+    Focus on extracting specific types of content:
+    - Technical documentation
+    - Code examples
+    - API references
+    Reformat the content into clear, well-structured markdown
+    """,
+    chunk_token_threshold=4096
+)
+```
+
+> **Performance Tip**: Set a smaller `chunk_token_threshold` (e.g., 2048 or 4096) to enable parallel processing of content chunks. The default value is infinity, which processes the entire content as a single chunk.
+
+---
+
+## 6. Using Fit Markdown
+
+When a content filter is active, the library produces two forms of markdown inside `result.markdown`:
+
+1. **`raw_markdown`**: The full unfiltered markdown.  
+2. **`fit_markdown`**: A “fit” version where the filter has removed or trimmed noisy segments.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+async def main():
+    config = CrawlerRunConfig(
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(threshold=0.6),
+            options={"ignore_links": True}
+        )
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://news.example.com/tech", config=config)
+        if result.success:
+            print("Raw markdown:\n", result.markdown)
+            
+            # If a filter is used, we also have .fit_markdown:
+            md_object = result.markdown  # or your equivalent
+            print("Filtered markdown:\n", md_object.fit_markdown)
+        else:
+            print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+## 7. The `MarkdownGenerationResult` Object
+
+If your library stores detailed markdown output in an object like `MarkdownGenerationResult`, you’ll see fields such as:
+
+- **`raw_markdown`**: The direct HTML-to-markdown transformation (no filtering).  
+- **`markdown_with_citations`**: A version that moves links to reference-style footnotes.  
+- **`references_markdown`**: A separate string or section containing the gathered references.  
+- **`fit_markdown`**: The filtered markdown if you used a content filter.  
+- **`fit_html`**: The corresponding HTML snippet used to generate `fit_markdown` (helpful for debugging or advanced usage).
+
+**Example**:
+
+```python
+md_obj = result.markdown  # your library’s naming may vary
+print("RAW:\n", md_obj.raw_markdown)
+print("CITED:\n", md_obj.markdown_with_citations)
+print("REFERENCES:\n", md_obj.references_markdown)
+print("FIT:\n", md_obj.fit_markdown)
+```
+
+**Why Does This Matter?**  
+- You can supply `raw_markdown` to an LLM if you want the entire text.  
+- Or feed `fit_markdown` into a vector database to reduce token usage.  
+- `references_markdown` can help you keep track of link provenance.
+
+---
+
+Below is a **revised section** under “Combining Filters (BM25 + Pruning)” that demonstrates how you can run **two** passes of content filtering without re-crawling, by taking the HTML (or text) from a first pass and feeding it into the second filter. It uses real code patterns from the snippet you provided for **BM25ContentFilter**, which directly accepts **HTML** strings (and can also handle plain text with minimal adaptation).
+
+---
+
+## 8. Combining Filters (BM25 + Pruning) in Two Passes
+
+You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
+
+1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).  
+2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query.
+
+### Two-Pass Example
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from bs4 import BeautifulSoup
+
+async def main():
+    # 1. Crawl with minimal or no markdown generator, just get raw HTML
+    config = CrawlerRunConfig(
+        # If you only want raw HTML, you can skip passing a markdown_generator
+        # or provide one but focus on .html in this example
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com/tech-article", config=config)
+
+        if not result.success or not result.html:
+            print("Crawl failed or no HTML content.")
+            return
+        
+        raw_html = result.html
+        
+        # 2. First pass: PruningContentFilter on raw HTML
+        pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50)
+        
+        # filter_content returns a list of "text chunks" or cleaned HTML sections
+        pruned_chunks = pruning_filter.filter_content(raw_html)
+        # This list is basically pruned content blocks, presumably in HTML or text form
+        
+        # For demonstration, let's combine these chunks back into a single HTML-like string
+        # or you could do further processing. It's up to your pipeline design.
+        pruned_html = "\n".join(pruned_chunks)
+        
+        # 3. Second pass: BM25ContentFilter with a user query
+        bm25_filter = BM25ContentFilter(
+            user_query="machine learning",
+            bm25_threshold=1.2,
+            language="english"
+        )
+        
+        # returns a list of text chunks
+        bm25_chunks = bm25_filter.filter_content(pruned_html)  
+        
+        if not bm25_chunks:
+            print("Nothing matched the BM25 query after pruning.")
+            return
+        
+        # 4. Combine or display final results
+        final_text = "\n---\n".join(bm25_chunks)
+        
+        print("==== PRUNED OUTPUT (first pass) ====")
+        print(pruned_html[:500], "... (truncated)")  # preview
+
+        print("\n==== BM25 OUTPUT (second pass) ====")
+        print(final_text[:500], "... (truncated)")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### What’s Happening?
+
+1. **Raw HTML**: We crawl once and store the raw HTML in `result.html`.  
+2. **PruningContentFilter**: Takes HTML + optional parameters. It extracts blocks of text or partial HTML, removing headings/sections deemed “noise.” It returns a **list of text chunks**.  
+3. **Combine or Transform**: We join these pruned chunks back into a single HTML-like string. (Alternatively, you could store them in a list for further logic—whatever suits your pipeline.)  
+4. **BM25ContentFilter**: We feed the pruned string into `BM25ContentFilter` with a user query. This second pass further narrows the content to chunks relevant to “machine learning.”
+
+**No Re-Crawling**: We used `raw_html` from the first pass, so there’s no need to run `arun()` again—**no second network request**.
+
+### Tips & Variations
+
+- **Plain Text vs. HTML**: If your pruned output is mostly text, BM25 can still handle it; just keep in mind it expects a valid string input. If you supply partial HTML (like `"
some text
"`), it will parse it as HTML.  
+- **Chaining in a Single Pipeline**: If your code supports it, you can chain multiple filters automatically. Otherwise, manual two-pass filtering (as shown) is straightforward.  
+- **Adjust Thresholds**: If you see too much or too little text in step one, tweak `threshold=0.5` or `min_word_threshold=50`. Similarly, `bm25_threshold=1.2` can be raised/lowered for more or fewer chunks in step two.
+
+### One-Pass Combination?
+
+If your codebase or pipeline design allows applying multiple filters in one pass, you could do so. But often it’s simpler—and more transparent—to run them sequentially, analyzing each step’s result.
+
+**Bottom Line**: By **manually chaining** your filtering logic in two passes, you get powerful incremental control over the final content. First, remove “global” clutter with Pruning, then refine further with BM25-based query relevance—without incurring a second network crawl.
+
+---
+
+## 9. Common Pitfalls & Tips
+
+1. **No Markdown Output?**  
+   - Make sure the crawler actually retrieved HTML. If the site is heavily JS-based, you may need to enable dynamic rendering or wait for elements.  
+   - Check if your content filter is too aggressive. Lower thresholds or disable the filter to see if content reappears.
+
+2. **Performance Considerations**  
+   - Very large pages with multiple filters can be slower. Consider `cache_mode` to avoid re-downloading.  
+   - If your final use case is LLM ingestion, consider summarizing further or chunking big texts.
+
+3. **Take Advantage of `fit_markdown`**  
+   - Great for RAG pipelines, semantic search, or any scenario where extraneous boilerplate is unwanted.  
+   - Still verify the textual quality—some sites have crucial data in footers or sidebars.
+
+4. **Adjusting `html2text` Options**  
+   - If you see lots of raw HTML slipping into the text, turn on `escape_html`.  
+   - If code blocks look messy, experiment with `mark_code` or `handle_code_in_pre`.
+
+---
+
+## 10. Summary & Next Steps
+
+In this **Markdown Generation Basics** tutorial, you learned to:
+
+- Configure the **DefaultMarkdownGenerator** with HTML-to-text options.  
+- Select different HTML sources using the `content_source` parameter.  
+- Use **BM25ContentFilter** for query-specific extraction or **PruningContentFilter** for general noise removal.  
+- Distinguish between raw and filtered markdown (`fit_markdown`).  
+- Leverage the `MarkdownGenerationResult` object to handle different forms of output (citations, references, etc.).
+
+Now you can produce high-quality Markdown from any website, focusing on exactly the content you need—an essential step for powering AI models, summarization pipelines, or knowledge-base queries.
+
+**Last Updated**: 2025-01-01
diff --git a/docs/md_v2/core/page-interaction.md b/docs/md_v2/core/page-interaction.md
new file mode 100644
index 0000000..4b28cac
--- /dev/null
+++ b/docs/md_v2/core/page-interaction.md
@@ -0,0 +1,432 @@
+# Page Interaction
+
+Crawl4AI provides powerful features for interacting with **dynamic** webpages, handling JavaScript execution, waiting for conditions, and managing multi-step flows. By combining **js_code**, **wait_for**, and certain **CrawlerRunConfig** parameters, you can:
+
+1. Click “Load More” buttons  
+2. Fill forms and submit them  
+3. Wait for elements or data to appear  
+4. Reuse sessions across multiple steps  
+
+Below is a quick overview of how to do it.
+
+---
+
+## 1. JavaScript Execution
+
+### Basic Execution
+
+**`js_code`** in **`CrawlerRunConfig`** accepts either a single JS string or a list of JS snippets. It runs **after** `wait_for` and `delay_before_return_html` — so the page is fully loaded when your code executes.
+
+**Example**: We'll scroll to the bottom of the page, then optionally click a "Load More" button.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    # Single JS command
+    config = CrawlerRunConfig(
+        js_code="window.scrollTo(0, document.body.scrollHeight);"
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://news.ycombinator.com",  # Example site
+            config=config
+        )
+        print("Crawled length:", len(result.cleaned_html))
+
+    # Multiple commands
+    js_commands = [
+        "window.scrollTo(0, document.body.scrollHeight);",
+        # 'More' link on Hacker News
+        "document.querySelector('a.morelink')?.click();",  
+    ]
+    config = CrawlerRunConfig(js_code=js_commands)
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://news.ycombinator.com",  # Another pass
+            config=config
+        )
+        print("After scroll+click, length:", len(result.cleaned_html))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Relevant `CrawlerRunConfig` params**:
+- **`js_code`**: JavaScript to run **after** `wait_for` and `delay_before_return_html` complete. Runs on the fully-loaded page.
+- **`js_code_before_wait`**: JavaScript to run **before** `wait_for`. Use when you need to trigger loading that `wait_for` then checks.
+- **`js_only`**: If set to `True` on subsequent calls, indicates we're continuing an existing session without a new full navigation.
+- **`session_id`**: If you want to keep the same page across multiple calls, specify an ID.
+
+### Execution Order
+
+Understanding when your JavaScript runs relative to other pipeline steps:
+
+```
+1. Page navigation (page.goto)
+2. js_code_before_wait     ← triggers loading / clicks tabs
+3. wait_for                ← waits for content to appear
+4. delay_before_return_html ← extra safety margin
+5. js_code                 ← runs on the fully-loaded page
+6. flatten_shadow_dom      ← if enabled
+7. page.content()          ← HTML capture
+```
+
+If you need JS to trigger something and then wait for the result, use `js_code_before_wait` + `wait_for`:
+
+```python
+config = CrawlerRunConfig(
+    # Click a tab first
+    js_code_before_wait="document.querySelector('#specs-tab')?.click();",
+    # Then wait for the tab content to appear
+    wait_for="css:#specs-panel .content",
+)
+```
+
+---
+
+## 2. Wait Conditions
+
+### 2.1 CSS-Based Waiting
+
+Sometimes, you just want to wait for a specific element to appear. For example:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    config = CrawlerRunConfig(
+        # Wait for at least 30 items on Hacker News
+        wait_for="css:.athing:nth-child(30)"  
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://news.ycombinator.com",
+            config=config
+        )
+        print("We have at least 30 items loaded!")
+        # Rough check
+        print("Total items in HTML:", result.cleaned_html.count("athing"))  
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key param**:
+- **`wait_for="css:..."`**: Tells the crawler to wait until that CSS selector is present.
+
+### 2.2 JavaScript-Based Waiting
+
+For more complex conditions (e.g., waiting for content length to exceed a threshold), prefix `js:`:
+
+```python
+wait_condition = """() => {
+    const items = document.querySelectorAll('.athing');
+    return items.length > 50;  // Wait for at least 51 items
+}"""
+
+config = CrawlerRunConfig(wait_for=f"js:{wait_condition}")
+```
+
+**Behind the Scenes**: Crawl4AI keeps polling the JS function until it returns `true` or a timeout occurs.
+
+---
+
+## 3. Handling Dynamic Content
+
+Many modern sites require **multiple steps**: scrolling, clicking “Load More,” or updating via JavaScript. Below are typical patterns.
+
+### 3.1 Load More Example (Hacker News “More” Link)
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    # Step 1: Load initial Hacker News page
+    config = CrawlerRunConfig(
+        wait_for="css:.athing:nth-child(30)"  # Wait for 30 items
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://news.ycombinator.com",
+            config=config
+        )
+        print("Initial items loaded.")
+
+        # Step 2: Let's scroll and click the "More" link
+        load_more_js = [
+            "window.scrollTo(0, document.body.scrollHeight);",
+            # The "More" link at page bottom
+            "document.querySelector('a.morelink')?.click();"  
+        ]
+        
+        next_page_conf = CrawlerRunConfig(
+            js_code=load_more_js,
+            wait_for="""js:() => {
+                return document.querySelectorAll('.athing').length > 30;
+            }""",
+            # Mark that we do not re-navigate, but run JS in the same session:
+            js_only=True,
+            session_id="hn_session"
+        )
+
+        # Re-use the same crawler session
+        result2 = await crawler.arun(
+            url="https://news.ycombinator.com",  # same URL but continuing session
+            config=next_page_conf
+        )
+        total_items = result2.cleaned_html.count("athing")
+        print("Items after load-more:", total_items)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key params**:
+- **`session_id="hn_session"`**: Keep the same page across multiple calls to `arun()`.
+- **`js_only=True`**: We’re not performing a full reload, just applying JS in the existing page.
+- **`wait_for`** with `js:`: Wait for item count to grow beyond 30.
+
+---
+
+### 3.2 Form Interaction
+
+If the site has a search or login form, you can fill fields and submit them with **`js_code`**. For instance, if GitHub had a local search form:
+
+```python
+js_form_interaction = """
+document.querySelector('#your-search').value = 'TypeScript commits';
+document.querySelector('form').submit();
+"""
+
+config = CrawlerRunConfig(
+    js_code=js_form_interaction,
+    wait_for="css:.commit"
+)
+result = await crawler.arun(url="https://github.com/search", config=config)
+```
+
+**In reality**: Replace IDs or classes with the real site’s form selectors.
+
+---
+
+## 4. Timing Control
+
+1. **`page_timeout`** (ms): Overall page load or script execution time limit.  
+2. **`delay_before_return_html`** (seconds): Wait an extra moment before capturing the final HTML.  
+3. **`mean_delay`** & **`max_range`**: If you call `arun_many()` with multiple URLs, these add a random pause between each request.
+
+**Example**:
+
+```python
+config = CrawlerRunConfig(
+    page_timeout=60000,  # 60s limit
+    delay_before_return_html=2.5
+)
+```
+
+---
+
+## 5. Multi-Step Interaction Example
+
+Below is a simplified script that does multiple “Load More” clicks on GitHub’s TypeScript commits page. It **re-uses** the same session to accumulate new commits each time. The code includes the relevant **`CrawlerRunConfig`** parameters you’d rely on.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def multi_page_commits():
+    browser_cfg = BrowserConfig(
+        headless=False,  # Visible for demonstration
+        verbose=True
+    )
+    session_id = "github_ts_commits"
+    
+    base_wait = """js:() => {
+        const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+        return commits.length > 0;
+    }"""
+
+    # Step 1: Load initial commits
+    config1 = CrawlerRunConfig(
+        wait_for=base_wait,
+        session_id=session_id,
+        cache_mode=CacheMode.BYPASS,
+        # Not using js_only yet since it's our first load
+    )
+
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        result = await crawler.arun(
+            url="https://github.com/microsoft/TypeScript/commits/main",
+            config=config1
+        )
+        print("Initial commits loaded. Count:", result.cleaned_html.count("commit"))
+
+        # Step 2: For subsequent pages, we run JS to click 'Next Page' if it exists
+        js_next_page = """
+        const selector = 'a[data-testid="pagination-next-button"]';
+        const button = document.querySelector(selector);
+        if (button) button.click();
+        """
+        
+        # Wait until new commits appear
+        wait_for_more = """js:() => {
+            const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+            if (!window.firstCommit && commits.length>0) {
+                window.firstCommit = commits[0].textContent;
+                return false;
+            }
+            // If top commit changes, we have new commits
+            const topNow = commits[0]?.textContent.trim();
+            return topNow && topNow !== window.firstCommit;
+        }"""
+
+        for page in range(2):  # let's do 2 more "Next" pages
+            config_next = CrawlerRunConfig(
+                session_id=session_id,
+                js_code=js_next_page,
+                wait_for=wait_for_more,
+                js_only=True,       # We're continuing from the open tab
+                cache_mode=CacheMode.BYPASS
+            )
+            result2 = await crawler.arun(
+                url="https://github.com/microsoft/TypeScript/commits/main",
+                config=config_next
+            )
+            print(f"Page {page+2} commits count:", result2.cleaned_html.count("commit"))
+
+        # Optionally kill session
+        await crawler.crawler_strategy.kill_session(session_id)
+
+async def main():
+    await multi_page_commits()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key Points**:
+
+- **`session_id`**: Keep the same page open.  
+- **`js_code`** + **`wait_for`** + **`js_only=True`**: We do partial refreshes, waiting for new commits to appear.  
+- **`cache_mode=CacheMode.BYPASS`** ensures we always see fresh data each step.
+
+---
+
+## 6. Combine Interaction with Extraction
+
+Once dynamic content is loaded, you can attach an **`extraction_strategy`** (like `JsonCssExtractionStrategy` or `LLMExtractionStrategy`). For example:
+
+```python
+from crawl4ai import JsonCssExtractionStrategy
+
+schema = {
+    "name": "Commits",
+    "baseSelector": "li.Box-sc-g0xbh4-0",
+    "fields": [
+        {"name": "title", "selector": "h4.markdown-title", "type": "text"}
+    ]
+}
+config = CrawlerRunConfig(
+    session_id="ts_commits_session",
+    js_code=js_next_page,
+    wait_for=wait_for_more,
+    extraction_strategy=JsonCssExtractionStrategy(schema)
+)
+```
+
+When done, check `result.extracted_content` for the JSON.
+
+---
+
+## 7. Shadow DOM Flattening
+
+Sites built with **Web Components** (Stencil, Lit, Shoelace, etc.) render content inside Shadow DOM — an encapsulated sub-tree that is invisible to normal page serialization. Set `flatten_shadow_dom=True` to extract it:
+
+```python
+config = CrawlerRunConfig(
+    flatten_shadow_dom=True,
+    wait_until="load",
+    delay_before_return_html=3.0,  # give components time to hydrate
+)
+```
+
+This walks all shadow trees, resolves `` projections, and produces flat HTML. It also force-opens closed shadow roots via an init script. For details and a full example, see [Flattening Shadow DOM](content-selection.md#31-flattening-shadow-dom) and [`shadow_dom_crawling.py`](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/shadow_dom_crawling.py).
+
+---
+
+## 8. Relevant `CrawlerRunConfig` Parameters
+
+Below are the key interaction-related parameters in `CrawlerRunConfig`. For a full list, see [Configuration Parameters](../api/parameters.md).
+
+- **`js_code`**: JavaScript to run after `wait_for` + `delay_before_return_html`, on the fully-loaded page.
+- **`js_code_before_wait`**: JavaScript to run before `wait_for`. For triggering loading that `wait_for` then checks.
+- **`js_only`**: If `True`, no new page navigation—only JS in the existing session.
+- **`wait_for`**: CSS (`"css:..."`) or JS (`"js:..."`) expression to wait for.
+- **`session_id`**: Reuse the same page across calls.
+- **`cache_mode`**: Whether to read/write from the cache or bypass.
+- **`flatten_shadow_dom`**: Flatten Shadow DOM content into the light DOM before capture.
+- **`process_iframes`**: Inline iframe content into the main document.
+- **`remove_overlay_elements`**: Remove certain popups automatically.
+- **`remove_consent_popups`**: Remove GDPR/cookie consent popups from known CMP providers (OneTrust, Cookiebot, Didomi, etc.).
+- **`simulate_user`, `override_navigator`, `magic`**: Anti-bot or "human-like" interactions.
+
+---
+
+## 9. Conclusion
+
+Crawl4AI's **page interaction** features let you:
+
+1. **Execute JavaScript** for scrolling, clicks, or form filling.  
+2. **Wait** for CSS or custom JS conditions before capturing data.  
+3. **Handle** multi-step flows (like “Load More”) with partial reloads or persistent sessions.  
+4. **Flatten Shadow DOM** on Web Component sites to extract hidden content.
+5. Combine with **structured extraction** for dynamic sites.
+
+With these tools, you can scrape modern, interactive webpages confidently. For advanced hooking, user simulation, or in-depth config, check the [API reference](../api/parameters.md) or related advanced docs. Happy scripting!
+
+---
+
+## 10. Virtual Scrolling
+
+For sites that use **virtual scrolling** (where content is replaced rather than appended as you scroll, like Twitter or Instagram), Crawl4AI provides a dedicated `VirtualScrollConfig`:
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, VirtualScrollConfig
+
+async def crawl_twitter_timeline():
+    # Configure virtual scroll for Twitter-like feeds
+    virtual_config = VirtualScrollConfig(
+        container_selector="[data-testid='primaryColumn']",  # Twitter's main column
+        scroll_count=30,                # Scroll 30 times
+        scroll_by="container_height",   # Scroll by container height each time
+        wait_after_scroll=1.0          # Wait 1 second after each scroll
+    )
+    
+    config = CrawlerRunConfig(
+        virtual_scroll_config=virtual_config
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://twitter.com/search?q=AI",
+            config=config
+        )
+        # result.html now contains ALL tweets from the virtual scroll
+```
+
+### Virtual Scroll vs JavaScript Scrolling
+
+| Feature | Virtual Scroll | JS Code Scrolling |
+|---------|---------------|-------------------|
+| **Use Case** | Content replaced during scroll | Content appended or simple scroll |
+| **Configuration** | `VirtualScrollConfig` object | `js_code` with scroll commands |
+| **Automatic Merging** | Yes - merges all unique content | No - captures final state only |
+| **Best For** | Twitter, Instagram, virtual tables | Traditional pages, load more buttons |
+
+For detailed examples and configuration options, see the [Virtual Scroll documentation](../advanced/virtual-scroll.md).
diff --git a/docs/md_v2/core/quickstart.md b/docs/md_v2/core/quickstart.md
new file mode 100644
index 0000000..78209ef
--- /dev/null
+++ b/docs/md_v2/core/quickstart.md
@@ -0,0 +1,465 @@
+# Getting Started with Crawl4AI
+
+Welcome to **Crawl4AI**, an open-source LLM-friendly Web Crawler & Scraper. In this tutorial, you’ll:
+
+1. Run your **first crawl** using minimal configuration.  
+2. Generate **Markdown** output (and learn how it’s influenced by content filters).  
+3. Experiment with a simple **CSS-based extraction** strategy.  
+4. See a glimpse of **LLM-based extraction** (including open-source and closed-source model options).  
+5. Crawl a **dynamic** page that loads content via JavaScript.
+
+---
+
+## 1. Introduction
+
+Crawl4AI provides:
+
+- An asynchronous crawler, **`AsyncWebCrawler`**.  
+- Configurable browser and run settings via **`BrowserConfig`** and **`CrawlerRunConfig`**.  
+- Automatic HTML-to-Markdown conversion via **`DefaultMarkdownGenerator`** (supports optional filters).  
+- Multiple extraction strategies (LLM-based or “traditional” CSS/XPath-based).
+
+By the end of this guide, you’ll have performed a basic crawl, generated Markdown, tried out two extraction strategies, and crawled a dynamic page that uses “Load More” buttons or JavaScript updates.
+
+---
+
+## 2. Your First Crawl
+
+Here’s a minimal Python script that creates an **`AsyncWebCrawler`**, fetches a webpage, and prints the first 300 characters of its Markdown output:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com")
+        print(result.markdown[:300])  # Print first 300 chars
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**What’s happening?**
+- **`AsyncWebCrawler`** launches a headless browser (Chromium by default).
+- It fetches `https://example.com`.
+- Crawl4AI automatically converts the HTML into Markdown.
+
+You now have a simple, working crawl!
+
+---
+
+## 3. Basic Configuration (Light Introduction)
+
+Crawl4AI’s crawler can be heavily customized using two main classes:
+
+1. **`BrowserConfig`**: Controls browser behavior (headless or full UI, user agent, JavaScript toggles, etc.).  
+2. **`CrawlerRunConfig`**: Controls how each crawl runs (caching, extraction, timeouts, hooking, etc.).
+
+Below is an example with minimal usage:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def main():
+    browser_conf = BrowserConfig(headless=True)  # or False to see the browser
+    run_conf = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler(config=browser_conf) as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=run_conf
+        )
+        print(result.markdown)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+> IMPORTANT: By default cache mode is set to `CacheMode.BYPASS` to have fresh content. Set `CacheMode.ENABLED` to enable caching.
+
+We’ll explore more advanced config in later tutorials (like enabling proxies, PDF output, multi-tab sessions, etc.). For now, just note how you pass these objects to manage crawling.
+
+---
+
+## 4. Generating Markdown Output
+
+By default, Crawl4AI automatically generates Markdown from each crawled page. However, the exact output depends on whether you specify a **markdown generator** or **content filter**.
+
+- **`result.markdown`**:  
+  The direct HTML-to-Markdown conversion.  
+- **`result.markdown.fit_markdown`**:  
+  The same content after applying any configured **content filter** (e.g., `PruningContentFilter`).
+
+### Example: Using a Filter with `DefaultMarkdownGenerator`
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+md_generator = DefaultMarkdownGenerator(
+    content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
+)
+
+config = CrawlerRunConfig(
+    cache_mode=CacheMode.BYPASS,
+    markdown_generator=md_generator
+)
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun("https://news.ycombinator.com", config=config)
+    print("Raw Markdown length:", len(result.markdown.raw_markdown))
+    print("Fit Markdown length:", len(result.markdown.fit_markdown))
+```
+
+**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilter` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
+
+---
+
+## 5. Simple Data Extraction (CSS-based)
+
+Crawl4AI can also extract structured data (JSON) using CSS or XPath selectors. Below is a minimal CSS-based example:
+
+> **New!** Crawl4AI now provides a powerful utility to automatically generate extraction schemas using LLM. This is a one-time cost that gives you a reusable schema for fast, LLM-free extractions:
+
+```python
+from crawl4ai import JsonCssExtractionStrategy
+from crawl4ai import LLMConfig
+
+# Generate a schema (one-time cost)
+html = "
Gaming Laptop
$999.99
"
+
+# Using OpenAI (requires API token)
+schema = JsonCssExtractionStrategy.generate_schema(
+    html,
+    llm_config = LLMConfig(provider="openai/gpt-4o",api_token="your-openai-token")  # Required for OpenAI
+)
+
+# Or using Ollama (open source, no token needed)
+schema = JsonCssExtractionStrategy.generate_schema(
+    html,
+    llm_config = LLMConfig(provider="ollama/llama3.3", api_token=None)  # Not needed for Ollama
+)
+
+# Use the schema for fast, repeated extractions
+strategy = JsonCssExtractionStrategy(schema)
+```
+
+For a complete guide on schema generation and advanced usage, see [No-LLM Extraction Strategies](../extraction/no-llm-strategies.md).
+
+Here's a basic extraction example:
+
+```python
+import asyncio
+import json
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai import JsonCssExtractionStrategy
+
+async def main():
+    schema = {
+        "name": "Example Items",
+        "baseSelector": "div.item",
+        "fields": [
+            {"name": "title", "selector": "h2", "type": "text"},
+            {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"}
+        ]
+    }
+
+    raw_html = "
Item 1
Link 1
"
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="raw://" + raw_html,
+            config=CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS,
+                extraction_strategy=JsonCssExtractionStrategy(schema)
+            )
+        )
+        # The JSON output is stored in 'extracted_content'
+        data = json.loads(result.extracted_content)
+        print(data)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Why is this helpful?**
+- Great for repetitive page structures (e.g., item listings, articles).
+- No AI usage or costs.
+- The crawler returns a JSON string you can parse or store.
+
+> Tips: You can pass raw HTML to the crawler instead of a URL. To do so, prefix the HTML with `raw://`.
+
+---
+
+## 6. Simple Data Extraction (LLM-based)
+
+For more complex or irregular pages, a language model can parse text intelligently into a structure you define. Crawl4AI supports **open-source** or **closed-source** providers:
+
+- **Open-Source Models** (e.g., `ollama/llama3.3`, `no_token`)  
+- **OpenAI Models** (e.g., `openai/gpt-4`, requires `api_token`)  
+- Or any provider supported by the underlying library
+
+Below is an example using **open-source** style (no token) and closed-source:
+
+```python
+import os
+import json
+import asyncio
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, LLMConfig
+from crawl4ai import LLMExtractionStrategy
+
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(
+        ..., description="Fee for output token for the OpenAI model."
+    )
+
+async def extract_structured_data_using_llm(
+    provider: str, api_token: str = None, extra_headers: Dict[str, str] = None
+):
+    print(f"\n--- Extracting Structured Data with {provider} ---")
+
+    if api_token is None and provider != "ollama":
+        print(f"API token is required for {provider}. Skipping this example.")
+        return
+
+    browser_config = BrowserConfig(headless=True)
+
+    extra_args = {"temperature": 0, "top_p": 0.9, "max_tokens": 2000}
+    if extra_headers:
+        extra_args["extra_headers"] = extra_headers
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        word_count_threshold=1,
+        page_timeout=80000,
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config = LLMConfig(provider=provider,api_token=api_token),
+            schema=OpenAIModelFee.model_json_schema(),
+            extraction_type="schema",
+            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+            Do not miss any models in the entire content.""",
+            extra_args=extra_args,
+        ),
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://openai.com/api/pricing/", config=crawler_config
+        )
+        print(result.extracted_content)
+
+if __name__ == "__main__":
+
+    asyncio.run(
+        extract_structured_data_using_llm(
+            provider="openai/gpt-4o", api_token=os.getenv("OPENAI_API_KEY")
+        )
+    )
+```
+
+**What’s happening?**
+- We define a Pydantic schema (`PricingInfo`) describing the fields we want.
+- The LLM extraction strategy uses that schema and your instructions to transform raw text into structured JSON.
+- Depending on the **provider** and **api_token**, you can use local models or a remote API.
+
+---
+
+## 7. Adaptive Crawling (New!)
+
+Crawl4AI now includes intelligent adaptive crawling that automatically determines when sufficient information has been gathered. Here's a quick example:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, AdaptiveCrawler
+
+async def adaptive_example():
+    async with AsyncWebCrawler() as crawler:
+        adaptive = AdaptiveCrawler(crawler)
+        
+        # Start adaptive crawling
+        result = await adaptive.digest(
+            start_url="https://docs.python.org/3/",
+            query="async context managers"
+        )
+        
+        # View results
+        adaptive.print_stats()
+        print(f"Crawled {len(result.crawled_urls)} pages")
+        print(f"Achieved {adaptive.confidence:.0%} confidence")
+
+if __name__ == "__main__":
+    asyncio.run(adaptive_example())
+```
+
+**What's special about adaptive crawling?**
+- **Automatic stopping**: Stops when sufficient information is gathered
+- **Intelligent link selection**: Follows only relevant links
+- **Confidence scoring**: Know how complete your information is
+
+[Learn more about Adaptive Crawling →](adaptive-crawling.md)
+
+---
+
+## 8. Multi-URL Concurrency (Preview)
+
+If you need to crawl multiple URLs in **parallel**, you can use `arun_many()`. By default, Crawl4AI employs a **MemoryAdaptiveDispatcher**, automatically adjusting concurrency based on system resources. Here’s a quick glimpse:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+
+async def quick_parallel_example():
+    urls = [
+        "https://example.com/page1",
+        "https://example.com/page2",
+        "https://example.com/page3"
+    ]
+    
+    run_conf = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        stream=True  # Enable streaming mode
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        # Stream results as they complete
+        async for result in await crawler.arun_many(urls, config=run_conf):
+            if result.success:
+                print(f"[OK] {result.url}, length: {len(result.markdown.raw_markdown)}")
+            else:
+                print(f"[ERROR] {result.url} => {result.error_message}")
+
+        # Or get all results at once (default behavior)
+        run_conf = run_conf.clone(stream=False)
+        results = await crawler.arun_many(urls, config=run_conf)
+        for res in results:
+            if res.success:
+                print(f"[OK] {res.url}, length: {len(res.markdown.raw_markdown)}")
+            else:
+                print(f"[ERROR] {res.url} => {res.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(quick_parallel_example())
+```
+
+The example above shows two ways to handle multiple URLs:
+1. **Streaming mode** (`stream=True`): Process results as they become available using `async for`
+2. **Batch mode** (`stream=False`): Wait for all results to complete
+
+For more advanced concurrency (e.g., a **semaphore-based** approach, **adaptive memory usage throttling**, or customized rate limiting), see [Advanced Multi-URL Crawling](../advanced/multi-url-crawling.md).
+
+---
+
+## 8. Dynamic Content Example
+
+Some sites require multiple “page clicks” or dynamic JavaScript updates. Below is an example showing how to **click** a “Next Page” button and wait for new commits to load on GitHub, using **`BrowserConfig`** and **`CrawlerRunConfig`**:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai import JsonCssExtractionStrategy
+
+async def extract_structured_data_using_css_extractor():
+    print("\n--- Using JsonCssExtractionStrategy for Fast Structured Output ---")
+    schema = {
+        "name": "KidoCode Courses",
+        "baseSelector": "section.charge-methodology .w-tab-content > div",
+        "fields": [
+            {
+                "name": "section_title",
+                "selector": "h3.heading-50",
+                "type": "text",
+            },
+            {
+                "name": "section_description",
+                "selector": ".charge-content",
+                "type": "text",
+            },
+            {
+                "name": "course_name",
+                "selector": ".text-block-93",
+                "type": "text",
+            },
+            {
+                "name": "course_description",
+                "selector": ".course-content-text",
+                "type": "text",
+            },
+            {
+                "name": "course_icon",
+                "selector": ".image-92",
+                "type": "attribute",
+                "attribute": "src",
+            },
+        ],
+    }
+
+    browser_config = BrowserConfig(headless=True, java_script_enabled=True)
+
+    js_click_tabs = """
+    (async () => {
+        const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");
+        for(let tab of tabs) {
+            tab.scrollIntoView();
+            tab.click();
+            await new Promise(r => setTimeout(r, 500));
+        }
+    })();
+    """
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=JsonCssExtractionStrategy(schema),
+        js_code=[js_click_tabs],
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology", config=crawler_config
+        )
+
+        companies = json.loads(result.extracted_content)
+        print(f"Successfully extracted {len(companies)} companies")
+        print(json.dumps(companies[0], indent=2))
+
+async def main():
+    await extract_structured_data_using_css_extractor()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key Points**:
+
+- **`BrowserConfig(headless=False)`**: We want to watch it click “Next Page.”  
+- **`CrawlerRunConfig(...)`**: We specify the extraction strategy, pass `session_id` to reuse the same page.  
+- **`js_code`** and **`wait_for`** are used for subsequent pages (`page > 0`) to click the “Next” button and wait for new commits to load.  
+- **`js_only=True`** indicates we’re not re-navigating but continuing the existing session.  
+- Finally, we call `kill_session()` to clean up the page and browser session.
+
+---
+
+## 9. Next Steps
+
+Congratulations! You have:
+
+1. Performed a basic crawl and printed Markdown.  
+2. Used **content filters** with a markdown generator.  
+3. Extracted JSON via **CSS** or **LLM** strategies.  
+4. Handled **dynamic** pages with JavaScript triggers.
+
+If you’re ready for more, check out:
+
+- **Installation**: A deeper dive into advanced installs, Docker usage (experimental), or optional dependencies.  
+- **Hooks & Auth**: Learn how to run custom JavaScript or handle logins with cookies, local storage, etc.  
+- **Deployment**: Explore ephemeral testing in Docker or plan for the upcoming stable Docker release.  
+- **Browser Management**: Delve into user simulation, stealth modes, and concurrency best practices.  
+
+Crawl4AI is a powerful, flexible tool. Enjoy building out your scrapers, data pipelines, or AI-driven extraction flows. Happy crawling!
\ No newline at end of file
diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md
new file mode 100644
index 0000000..1420774
--- /dev/null
+++ b/docs/md_v2/core/self-hosting.md
@@ -0,0 +1,2641 @@
+# Self-Hosting Crawl4AI 🚀
+
+> **🔐 0.9.0 is secure-by-default (breaking changes).** The self-hosted Docker
+> server now requires authentication by default, binds to loopback unless you
+> set a token, validates request bodies against a strict trust boundary, uses
+> declarative hooks instead of inline Python, and returns artifact ids for
+> screenshot/pdf. If you are upgrading from 0.8.x, read the
+> [migration guide](https://github.com/unclecode/crawl4ai/blob/main/deploy/docker/MIGRATION.md)
+> first. Some examples below are being updated for 0.9.0; the migration guide is
+> the authoritative reference for the new defaults.
+
+**Take Control of Your Web Crawling Infrastructure**
+
+Self-hosting Crawl4AI gives you complete control over your web crawling and data extraction pipeline. Unlike cloud-based solutions, you own your data, infrastructure, and destiny.
+
+## Why Self-Host?
+
+- **🔒 Data Privacy**: Your crawled data never leaves your infrastructure
+- **💰 Cost Control**: No per-request pricing - scale within your own resources
+- **🎯 Customization**: Full control over browser configurations, extraction strategies, and performance tuning
+- **📊 Transparency**: Real-time monitoring dashboard shows exactly what's happening
+- **⚡ Performance**: Direct access without API rate limits or geographic restrictions
+- **🛡️ Security**: Keep sensitive data extraction workflows behind your firewall
+- **🔧 Flexibility**: Customize, extend, and integrate with your existing infrastructure
+
+When you self-host, you can scale from a single container to a full browser infrastructure, all while maintaining complete control and visibility.
+
+## Table of Contents
+- [Prerequisites](#prerequisites)
+- [Installation](#installation)
+  - [Option 1: Using Pre-built Docker Hub Images (Recommended)](#option-1-using-pre-built-docker-hub-images-recommended)
+  - [Option 2: Using Docker Compose](#option-2-using-docker-compose)
+  - [Option 3: Manual Local Build & Run](#option-3-manual-local-build--run)
+- [MCP (Model Context Protocol) Support](#mcp-model-context-protocol-support)
+  - [What is MCP?](#what-is-mcp)
+  - [Connecting via MCP](#connecting-via-mcp)
+  - [Using with Claude Code](#using-with-claude-code)
+  - [Available MCP Tools](#available-mcp-tools)
+  - [Testing MCP Connections](#testing-mcp-connections)
+  - [MCP Schemas](#mcp-schemas)
+- [Real-time Monitoring & Operations](#real-time-monitoring--operations)
+  - [Monitoring Dashboard](#monitoring-dashboard)
+  - [Monitor API Endpoints](#monitor-api-endpoints)
+  - [WebSocket Streaming](#websocket-streaming)
+  - [Control Actions](#control-actions)
+  - [Production Integration](#production-integration)
+- [Deployment Scenarios](#deployment-scenarios)
+- [Complete Examples](#complete-examples)
+- [Server Configuration](#server-configuration)
+  - [Understanding config.yml](#understanding-configyml)
+  - [JWT Authentication](#jwt-authentication)
+  - [Configuration Tips and Best Practices](#configuration-tips-and-best-practices)
+  - [Customizing Your Configuration](#customizing-your-configuration)
+  - [Configuration Recommendations](#configuration-recommendations)
+- [Getting Help](#getting-help)
+- [Summary](#summary)
+
+## Prerequisites
+
+Before we dive in, make sure you have:
+- Docker installed and running (version 20.10.0 or higher), including `docker compose` (usually bundled with Docker Desktop).
+- `git` for cloning the repository.
+- At least 4GB of RAM available for the container (more recommended for heavy use).
+- Python 3.10+ (if using the Python SDK).
+- Node.js 16+ (if using the Node.js examples).
+
+> 💡 **Pro tip**: Run `docker info` to check your Docker installation and available resources.
+
+## Installation
+
+We offer several ways to get the Crawl4AI server running. The quickest way is to use our pre-built Docker Hub images.
+
+### Option 1: Using Pre-built Docker Hub Images (Recommended)
+
+Pull and run images directly from Docker Hub without building locally.
+
+#### 1. Pull the Image
+
+Our latest release is `0.8.0`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system.
+
+> 💡 **Note**: The `latest` tag points to the stable `0.8.0` version.
+
+```bash
+# Pull the latest version
+docker pull unclecode/crawl4ai:0.8.0
+
+# Or pull using the latest tag
+docker pull unclecode/crawl4ai:latest
+```
+
+#### 2. Setup Environment (API Keys)
+
+If you plan to use LLMs, create a `.llm.env` file in your working directory:
+
+```bash
+# Create a .llm.env file with your API keys
+cat > .llm.env << EOL
+# OpenAI
+OPENAI_API_KEY=sk-your-key
+
+# Anthropic
+ANTHROPIC_API_KEY=your-anthropic-key
+
+# Other providers as needed
+# DEEPSEEK_API_KEY=your-deepseek-key
+# GROQ_API_KEY=your-groq-key
+# TOGETHER_API_KEY=your-together-key
+# MISTRAL_API_KEY=your-mistral-key
+# GEMINI_API_TOKEN=your-gemini-token
+
+# Optional: Global LLM settings
+# LLM_PROVIDER=openai/gpt-4o-mini
+# LLM_TEMPERATURE=0.7
+# LLM_BASE_URL=https://api.custom.com/v1
+
+# Optional: Provider-specific overrides
+# OPENAI_TEMPERATURE=0.5
+# OPENAI_BASE_URL=https://custom-openai.com/v1
+# ANTHROPIC_TEMPERATURE=0.3
+EOL
+```
+> 🔑 **Note**: Keep your API keys secure! Never commit `.llm.env` to version control.
+
+#### 3. Run the Container
+
+*   **Basic run:**
+    ```bash
+    docker run -d \
+      -p 11235:11235 \
+      --name crawl4ai \
+      --shm-size=1g \
+      unclecode/crawl4ai:latest
+    ```
+
+*   **With LLM support:**
+    ```bash
+    # Make sure .llm.env is in the current directory
+    docker run -d \
+      -p 11235:11235 \
+      --name crawl4ai \
+      --env-file .llm.env \
+      --shm-size=1g \
+      unclecode/crawl4ai:latest
+    ```
+
+> The server will be available at `http://localhost:11235`. Visit `/playground` to access the interactive testing interface.
+
+#### 4. Stopping the Container
+
+```bash
+docker stop crawl4ai && docker rm crawl4ai
+```
+
+#### Docker Hub Versioning Explained
+
+*   **Image Name:** `unclecode/crawl4ai`
+*   **Tag Format:** `LIBRARY_VERSION[-SUFFIX]` (e.g., `0.8.0`)
+    *   `LIBRARY_VERSION`: The semantic version of the core `crawl4ai` Python library
+    *   `SUFFIX`: Optional tag for release candidates (``) and revisions (`r1`)
+*   **`latest` Tag:** Points to the most recent stable version
+*   **Multi-Architecture Support:** All images support both `linux/amd64` and `linux/arm64` architectures through a single tag
+
+### Option 2: Using Docker Compose
+
+Docker Compose simplifies building and running the service, especially for local development and testing.
+
+#### 1. Clone Repository
+
+```bash
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+```
+
+#### 2. Environment Setup (API Keys)
+
+If you plan to use LLMs, copy the example environment file and add your API keys. This file should be in the **project root directory**.
+
+```bash
+# Make sure you are in the 'crawl4ai' root directory
+cp deploy/docker/.llm.env.example .llm.env
+
+# Now edit .llm.env and add your API keys
+```
+
+**Flexible LLM Provider Configuration:**
+
+The Docker setup now supports flexible LLM provider configuration through a hierarchical system:
+
+1. **API Request Parameters** (Highest Priority): Specify per request
+   ```json
+   {
+     "url": "https://example.com",
+     "f": "llm",
+     "provider": "groq/mixtral-8x7b",
+     "temperature": 0.7,
+     "base_url": "https://api.custom.com/v1"
+   }
+   ```
+
+2. **Provider-Specific Environment Variables**: Override for specific providers
+   ```bash
+   # In your .llm.env file:
+   OPENAI_TEMPERATURE=0.5
+   OPENAI_BASE_URL=https://custom-openai.com/v1
+   ANTHROPIC_TEMPERATURE=0.3
+   ```
+
+3. **Global Environment Variables**: Set defaults for all providers
+   ```bash
+   # In your .llm.env file:
+   LLM_PROVIDER=anthropic/claude-3-opus
+   LLM_TEMPERATURE=0.7
+   LLM_BASE_URL=https://api.proxy.com/v1
+   ```
+
+4. **Config File Default**: Falls back to `config.yml` (default: `openai/gpt-4o-mini`)
+
+The system automatically selects the appropriate API key based on the provider. LiteLLM handles finding the correct environment variable for each provider (e.g., OPENAI_API_KEY for OpenAI, GEMINI_API_TOKEN for Google Gemini, etc.).
+
+**Supported LLM Parameters:**
+- `provider`: LLM provider and model (e.g., "openai/gpt-4", "anthropic/claude-3-opus")
+- `temperature`: Controls randomness (0.0-2.0, lower = more focused, higher = more creative)
+- `base_url`: Custom API endpoint for proxy servers or alternative endpoints
+
+#### 3. Build and Run with Compose
+
+The `docker-compose.yml` file in the project root provides a simplified approach that automatically handles architecture detection using buildx.
+
+*   **Run Pre-built Image from Docker Hub:**
+    ```bash
+    # Pulls and runs the release candidate from Docker Hub
+    # Automatically selects the correct architecture
+    IMAGE=unclecode/crawl4ai:latest docker compose up -d
+    ```
+
+*   **Build and Run Locally:**
+    ```bash
+    # Builds the image locally using Dockerfile and runs it
+    # Automatically uses the correct architecture for your machine
+    docker compose up --build -d
+    ```
+
+*   **Customize the Build:**
+    ```bash
+    # Build with all features (includes torch and transformers)
+    INSTALL_TYPE=all docker compose up --build -d
+    
+    # Build with GPU support (for AMD64 platforms)
+    ENABLE_GPU=true docker compose up --build -d
+    ```
+
+> The server will be available at `http://localhost:11235`.
+
+#### 4. Stopping the Service
+
+```bash
+# Stop the service
+docker compose down
+```
+
+### Option 3: Manual Local Build & Run
+
+If you prefer not to use Docker Compose for direct control over the build and run process.
+
+#### 1. Clone Repository & Setup Environment
+
+Follow steps 1 and 2 from the Docker Compose section above (clone repo, `cd crawl4ai`, create `.llm.env` in the root).
+
+#### 2. Build the Image (Multi-Arch)
+
+Use `docker buildx` to build the image. Crawl4AI now uses buildx to handle multi-architecture builds automatically.
+
+```bash
+# Make sure you are in the 'crawl4ai' root directory
+# Build for the current architecture and load it into Docker
+docker buildx build -t crawl4ai-local:latest --load .
+
+# Or build for multiple architectures (useful for publishing)
+docker buildx build --platform linux/amd64,linux/arm64 -t crawl4ai-local:latest --load .
+
+# Build with additional options
+docker buildx build \
+  --build-arg INSTALL_TYPE=all \
+  --build-arg ENABLE_GPU=false \
+  -t crawl4ai-local:latest --load .
+```
+
+#### 3. Run the Container
+
+*   **Basic run (no LLM support):**
+    ```bash
+    docker run -d \
+      -p 11235:11235 \
+      --name crawl4ai-standalone \
+      --shm-size=1g \
+      crawl4ai-local:latest
+    ```
+
+*   **With LLM support:**
+    ```bash
+    # Make sure .llm.env is in the current directory (project root)
+    docker run -d \
+      -p 11235:11235 \
+      --name crawl4ai-standalone \
+      --env-file .llm.env \
+      --shm-size=1g \
+      crawl4ai-local:latest
+    ```
+
+> The server will be available at `http://localhost:11235`.
+
+#### 4. Stopping the Manual Container
+
+```bash
+docker stop crawl4ai-standalone && docker rm crawl4ai-standalone
+```
+
+---
+
+## MCP (Model Context Protocol) Support
+
+Crawl4AI server includes support for the Model Context Protocol (MCP), allowing you to connect the server's capabilities directly to MCP-compatible clients like Claude Code.
+
+### What is MCP?
+
+MCP is an open protocol that standardizes how applications provide context to LLMs. It allows AI models to access external tools, data sources, and services through a standardized interface.
+
+### Connecting via MCP
+
+The Crawl4AI server exposes two MCP endpoints:
+
+- **Server-Sent Events (SSE)**: `http://localhost:11235/mcp/sse`
+- **WebSocket**: `ws://localhost:11235/mcp/ws`
+
+### Using with Claude Code
+
+You can add Crawl4AI as an MCP tool provider in Claude Code with a simple command:
+
+```bash
+# Add the Crawl4AI server as an MCP provider
+claude mcp add --transport sse c4ai-sse http://localhost:11235/mcp/sse
+
+# List all MCP providers to verify it was added
+claude mcp list
+```
+
+Once connected, Claude Code can directly use Crawl4AI's capabilities like screenshot capture, PDF generation, and HTML processing without having to make separate API calls.
+
+### Available MCP Tools
+
+When connected via MCP, the following tools are available:
+
+- `md` - Generate markdown from web content
+- `html` - Extract preprocessed HTML
+- `screenshot` - Capture webpage screenshots
+- `pdf` - Generate PDF documents
+- `execute_js` - Run JavaScript on web pages
+- `crawl` - Perform multi-URL crawling
+- `ask` - Query the Crawl4AI library context
+
+### Testing MCP Connections
+
+You can test the MCP WebSocket connection using the test file included in the repository:
+
+```bash
+# From the repository root
+python tests/mcp/test_mcp_socket.py
+```
+
+### MCP Schemas
+
+Access the MCP tool schemas at `http://localhost:11235/mcp/schema` for detailed information on each tool's parameters and capabilities.
+
+---
+
+## Additional API Endpoints
+
+In addition to the core `/crawl` and `/crawl/stream` endpoints, the server provides several specialized endpoints:
+
+### HTML Extraction Endpoint
+
+```
+POST /html
+```
+
+Crawls the URL and returns preprocessed HTML optimized for schema extraction.
+
+```json
+{
+  "url": "https://example.com"
+}
+```
+
+### Screenshot Endpoint
+
+```
+POST /screenshot
+```
+
+Captures a full-page PNG screenshot of the specified URL.
+
+```json
+{
+  "url": "https://example.com",
+  "screenshot_wait_for": 2,
+  "output_path": "/path/to/save/screenshot.png"
+}
+```
+
+- `screenshot_wait_for`: Optional delay in seconds before capture (default: 2)
+- `output_path`: Optional path to save the screenshot (recommended)
+
+### PDF Export Endpoint
+
+```
+POST /pdf
+```
+
+Generates a PDF document of the specified URL.
+
+```json
+{
+  "url": "https://example.com",
+  "output_path": "/path/to/save/document.pdf"
+}
+```
+
+- `output_path`: Optional path to save the PDF (recommended)
+
+### JavaScript Execution Endpoint
+
+```
+POST /execute_js
+```
+
+Executes JavaScript snippets on the specified URL and returns the full crawl result.
+
+```json
+{
+  "url": "https://example.com",
+  "scripts": [
+    "return document.title",
+    "return Array.from(document.querySelectorAll('a')).map(a => a.href)"
+  ]
+}
+```
+
+- `scripts`: List of JavaScript snippets to execute sequentially
+
+---
+
+## User-Provided Hooks API
+
+> **⚠️ Changed in 0.9.0.** The inline-Python hook API described below was removed.
+> Sending arbitrary Python code to the server is no longer accepted (it was an
+> unauthenticated code-execution surface). 0.9.0 replaces it with **declarative
+> hooks**: a fixed set of safe, server-validated actions (for example
+> `add_cookies`, `set_headers`, `block_resources`) supplied as JSON, with no code
+> execution. See the [migration guide](https://github.com/unclecode/crawl4ai/blob/main/deploy/docker/MIGRATION.md)
+> for the declarative hook format. The inline-code examples in this section apply
+> to 0.8.x only and are kept for reference until this page is fully rewritten.
+
+The Docker API supports user-provided hook functions, allowing you to customize the crawling behavior by injecting your own Python code at specific points in the crawling pipeline. This powerful feature enables authentication, performance optimization, and custom content extraction without modifying the server code.
+
+> ⚠️ **IMPORTANT SECURITY WARNING**: 
+> - **Never use hooks with untrusted code or on untrusted websites**
+> - **Be extremely careful when crawling sites that might be phishing or malicious**
+> - **Hook code has access to page context and can interact with the website**
+> - **Always validate and sanitize any data extracted through hooks**
+> - **Never expose credentials or sensitive data in hook code**
+> - **Consider running the Docker container in an isolated network when testing**
+
+### Hook Information Endpoint
+
+```
+GET /hooks/info
+```
+
+Returns information about available hook points and their signatures:
+
+```bash
+curl http://localhost:11235/hooks/info
+```
+
+### Available Hook Points
+
+The API supports 8 hook points that match the local SDK:
+
+| Hook Point | Parameters | Description | Best Use Cases |
+|------------|------------|-------------|----------------|
+| `on_browser_created` | `browser` | After browser instance creation | Light setup tasks |
+| `on_page_context_created` | `page, context` | After page/context creation | **Authentication, cookies, route blocking** |
+| `before_goto` | `page, context, url` | Before navigating to URL | Custom headers, logging |
+| `after_goto` | `page, context, url, response` | After navigation completes | Verification, waiting for elements |
+| `on_user_agent_updated` | `page, context, user_agent` | When user agent changes | UA-specific logic |
+| `on_execution_started` | `page, context` | When JS execution begins | JS-related setup |
+| `before_retrieve_html` | `page, context` | Before getting final HTML | **Scrolling, lazy loading** |
+| `before_return_html` | `page, context, html` | Before returning HTML | Final modifications, metrics |
+
+### Using Hooks in Requests
+
+Add hooks to any crawl request by including the `hooks` parameter:
+
+```json
+{
+  "urls": ["https://httpbin.org/html"],
+  "hooks": {
+    "code": {
+      "hook_point_name": "async def hook(...): ...",
+      "another_hook": "async def hook(...): ..."
+    },
+    "timeout": 30  // Optional, default 30 seconds (max 120)
+  }
+}
+```
+
+### Hook Examples with Real URLs
+
+#### 1. Authentication with Cookies (GitHub)
+
+```python
+import requests
+
+# Example: Setting GitHub session cookie (use your actual session)
+hooks_code = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    # Add authentication cookies for GitHub
+    # WARNING: Never hardcode real credentials!
+    await context.add_cookies([
+        {
+            'name': 'user_session',
+            'value': 'your_github_session_token',  # Replace with actual token
+            'domain': '.github.com',
+            'path': '/',
+            'httpOnly': True,
+            'secure': True,
+            'sameSite': 'Lax'
+        }
+    ])
+    return page
+"""
+}
+
+response = requests.post("http://localhost:11235/crawl", json={
+    "urls": ["https://github.com/settings/profile"],  # Protected page
+    "hooks": {"code": hooks_code, "timeout": 30}
+})
+```
+
+#### 2. Basic Authentication (httpbin.org for testing)
+
+```python
+# Safe testing with httpbin.org (a service designed for HTTP testing)
+hooks_code = {
+    "before_goto": """
+async def hook(page, context, url, **kwargs):
+    import base64
+    # httpbin.org/basic-auth expects username="user" and password="passwd"
+    credentials = base64.b64encode(b"user:passwd").decode('ascii')
+    
+    await page.set_extra_http_headers({
+        'Authorization': f'Basic {credentials}'
+    })
+    return page
+"""
+}
+
+response = requests.post("http://localhost:11235/crawl", json={
+    "urls": ["https://httpbin.org/basic-auth/user/passwd"],
+    "hooks": {"code": hooks_code, "timeout": 15}
+})
+```
+
+#### 3. Performance Optimization (News Sites)
+
+```python
+# Example: Optimizing crawling of news sites like CNN or BBC
+hooks_code = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    # Block images, fonts, and media to speed up crawling
+    await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
+    await context.route("**/*.{woff,woff2,ttf,otf,eot}", lambda route: route.abort())
+    await context.route("**/*.{mp4,webm,ogg,mp3,wav,flac}", lambda route: route.abort())
+    
+    # Block common tracking and ad domains
+    await context.route("**/googletagmanager.com/*", lambda route: route.abort())
+    await context.route("**/google-analytics.com/*", lambda route: route.abort())
+    await context.route("**/doubleclick.net/*", lambda route: route.abort())
+    await context.route("**/facebook.com/tr/*", lambda route: route.abort())
+    await context.route("**/amazon-adsystem.com/*", lambda route: route.abort())
+    
+    # Disable CSS animations for faster rendering
+    await page.add_style_tag(content='''
+        *, *::before, *::after {
+            animation-duration: 0s !important;
+            transition-duration: 0s !important;
+        }
+    ''')
+    
+    return page
+"""
+}
+
+response = requests.post("http://localhost:11235/crawl", json={
+    "urls": ["https://www.bbc.com/news"],  # Heavy news site
+    "hooks": {"code": hooks_code, "timeout": 30}
+})
+```
+
+#### 4. Handling Infinite Scroll (Twitter/X)
+
+```python
+# Example: Scrolling on Twitter/X (requires authentication)
+hooks_code = {
+    "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    # Scroll to load more tweets
+    previous_height = 0
+    for i in range(5):  # Limit scrolls to avoid infinite loop
+        current_height = await page.evaluate("document.body.scrollHeight")
+        if current_height == previous_height:
+            break  # No more content to load
+            
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        await page.wait_for_timeout(2000)  # Wait for content to load
+        previous_height = current_height
+    
+    return page
+"""
+}
+
+# Note: Twitter requires authentication for most content
+response = requests.post("http://localhost:11235/crawl", json={
+    "urls": ["https://twitter.com/nasa"],  # Public profile
+    "hooks": {"code": hooks_code, "timeout": 30}
+})
+```
+
+#### 5. E-commerce Login (Example Pattern)
+
+```python
+# SECURITY WARNING: This is a pattern example. 
+# Never use real credentials in code!
+# Always use environment variables or secure vaults.
+
+hooks_code = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    # Example pattern for e-commerce sites
+    # DO NOT use real credentials here!
+    
+    # Navigate to login page first
+    await page.goto("https://example-shop.com/login")
+    
+    # Wait for login form to load
+    await page.wait_for_selector("#email", timeout=5000)
+    
+    # Fill login form (use environment variables in production!)
+    await page.fill("#email", "test@example.com")  # Never use real email
+    await page.fill("#password", "test_password")   # Never use real password
+    
+    # Handle "Remember Me" checkbox if present
+    try:
+        await page.uncheck("#remember_me")  # Don't remember on shared systems
+    except:
+        pass
+    
+    # Submit form
+    await page.click("button[type='submit']")
+    
+    # Wait for redirect after login
+    await page.wait_for_url("**/account/**", timeout=10000)
+    
+    return page
+"""
+}
+```
+
+#### 6. Extracting Structured Data (Wikipedia)
+
+```python
+# Safe example using Wikipedia
+hooks_code = {
+    "after_goto": """
+async def hook(page, context, url, response, **kwargs):
+    # Wait for Wikipedia content to load
+    await page.wait_for_selector("#content", timeout=5000)
+    return page
+""",
+    
+    "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    # Extract structured data from Wikipedia infobox
+    metadata = await page.evaluate('''() => {
+        const infobox = document.querySelector('.infobox');
+        if (!infobox) return null;
+        
+        const data = {};
+        const rows = infobox.querySelectorAll('tr');
+        
+        rows.forEach(row => {
+            const header = row.querySelector('th');
+            const value = row.querySelector('td');
+            if (header && value) {
+                data[header.innerText.trim()] = value.innerText.trim();
+            }
+        });
+        
+        return data;
+    }''')
+    
+    if metadata:
+        print("Extracted metadata:", metadata)
+    
+    return page
+"""
+}
+
+response = requests.post("http://localhost:11235/crawl", json={
+    "urls": ["https://en.wikipedia.org/wiki/Python_(programming_language)"],
+    "hooks": {"code": hooks_code, "timeout": 20}
+})
+```
+
+### Security Best Practices
+
+> 🔒 **Critical Security Guidelines**:
+
+1. **Never Trust User Input**: If accepting hook code from users, always validate and sandbox it
+2. **Avoid Phishing Sites**: Never use hooks on suspicious or unverified websites
+3. **Protect Credentials**: 
+   - Never hardcode passwords, tokens, or API keys in hook code
+   - Use environment variables or secure secret management
+   - Rotate credentials regularly
+4. **Network Isolation**: Run the Docker container in an isolated network when testing
+5. **Audit Hook Code**: Always review hook code before execution
+6. **Limit Permissions**: Use the least privileged access needed
+7. **Monitor Execution**: Check hook execution logs for suspicious behavior
+8. **Timeout Protection**: Always set reasonable timeouts (default 30s)
+
+### Hook Response Information
+
+When hooks are used, the response includes detailed execution information:
+
+```json
+{
+  "success": true,
+  "results": [...],
+  "hooks": {
+    "status": {
+      "status": "success",  // or "partial" or "failed"
+      "attached_hooks": ["on_page_context_created", "before_retrieve_html"],
+      "validation_errors": [],
+      "successfully_attached": 2,
+      "failed_validation": 0
+    },
+    "execution_log": [
+      {
+        "hook_point": "on_page_context_created",
+        "status": "success",
+        "execution_time": 0.523,
+        "timestamp": 1234567890.123
+      }
+    ],
+    "errors": [],  // Any runtime errors
+    "summary": {
+      "total_executions": 2,
+      "successful": 2,
+      "failed": 0,
+      "timed_out": 0,
+      "success_rate": 100.0
+    }
+  }
+}
+```
+
+### Error Handling
+
+The hooks system is designed to be resilient:
+
+1. **Validation Errors**: Caught before execution (syntax errors, wrong parameters)
+2. **Runtime Errors**: Handled gracefully - crawl continues with original page object
+3. **Timeout Protection**: Hooks automatically terminated after timeout (configurable 1-120s)
+
+### Complete Example: Safe Multi-Hook Crawling
+
+```python
+import requests
+import json
+import os
+
+# Safe example using httpbin.org for testing
+hooks_code = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    # Set viewport and test cookies
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    await context.add_cookies([
+        {"name": "test_cookie", "value": "test_value", "domain": ".httpbin.org", "path": "/"}
+    ])
+    
+    # Block unnecessary resources for httpbin
+    await context.route("**/*.{png,jpg,jpeg}", lambda route: route.abort())
+    return page
+""",
+    
+    "before_goto": """
+async def hook(page, context, url, **kwargs):
+    # Add custom headers for testing
+    await page.set_extra_http_headers({
+        "X-Test-Header": "crawl4ai-test",
+        "Accept-Language": "en-US,en;q=0.9"
+    })
+    print(f"[HOOK] Navigating to: {url}")
+    return page
+""",
+    
+    "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    # Simple scroll for any lazy-loaded content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    return page
+"""
+}
+
+# Make the request to safe testing endpoints
+response = requests.post("http://localhost:11235/crawl", json={
+    "urls": [
+        "https://httpbin.org/html",
+        "https://httpbin.org/json"
+    ],
+    "hooks": {
+        "code": hooks_code,
+        "timeout": 30
+    },
+    "crawler_config": {
+        "cache_mode": "bypass"
+    }
+})
+
+# Check results
+if response.status_code == 200:
+    data = response.json()
+    
+    # Check hook execution
+    if data['hooks']['status']['status'] == 'success':
+        print(f"✅ All {len(data['hooks']['status']['attached_hooks'])} hooks executed successfully")
+        print(f"Execution stats: {data['hooks']['summary']}")
+    
+    # Process crawl results
+    for result in data['results']:
+        print(f"Crawled: {result['url']} - Success: {result['success']}")
+else:
+    print(f"Error: {response.status_code}")
+```
+
+> 💡 **Remember**: Always test your hooks on safe, known websites first before using them on production sites. Never crawl sites that you don't have permission to access or that might be malicious.
+
+### Hooks Utility: Function-Based Approach (Python)
+
+For Python developers, Crawl4AI provides a more convenient way to work with hooks using the `hooks_to_string()` utility function and Docker client integration.
+
+#### Why Use Function-Based Hooks?
+
+**String-Based Approach (shown above)**:
+```python
+hooks_code = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+"""
+}
+```
+
+**Function-Based Approach (recommended for Python)**:
+```python
+from crawl4ai import Crawl4aiDockerClient
+
+async def my_hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+
+async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client:
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks={"on_page_context_created": my_hook}
+    )
+```
+
+**Benefits**:
+- ✅ Write hooks as regular Python functions
+- ✅ Full IDE support (autocomplete, syntax highlighting, type checking)
+- ✅ Easy to test and debug
+- ✅ Reusable hook libraries
+- ✅ Automatic conversion to API format
+
+#### Using the Hooks Utility
+
+The `hooks_to_string()` utility converts Python function objects to the string format required by the API:
+
+```python
+from crawl4ai import hooks_to_string
+
+# Define your hooks as functions
+async def setup_hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    await context.add_cookies([{
+        "name": "session",
+        "value": "token",
+        "domain": ".example.com"
+    }])
+    return page
+
+async def scroll_hook(page, context, **kwargs):
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    return page
+
+# Convert to string format
+hooks_dict = {
+    "on_page_context_created": setup_hook,
+    "before_retrieve_html": scroll_hook
+}
+hooks_string = hooks_to_string(hooks_dict)
+
+# Now use with REST API or Docker client
+# hooks_string contains the string representations
+```
+
+#### Docker Client with Automatic Conversion
+
+The Docker client automatically detects and converts function objects:
+
+```python
+from crawl4ai import Crawl4aiDockerClient
+
+async def auth_hook(page, context, **kwargs):
+    """Add authentication cookies"""
+    await context.add_cookies([{
+        "name": "auth_token",
+        "value": "your_token",
+        "domain": ".example.com"
+    }])
+    return page
+
+async def performance_hook(page, context, **kwargs):
+    """Block unnecessary resources"""
+    await context.route("**/*.{png,jpg,gif}", lambda r: r.abort())
+    await context.route("**/analytics/*", lambda r: r.abort())
+    return page
+
+async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client:
+    # Pass functions directly - automatic conversion!
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks={
+            "on_page_context_created": performance_hook,
+            "before_goto": auth_hook
+        },
+        hooks_timeout=30  # Optional timeout in seconds (1-120)
+    )
+
+    print(f"Success: {result.success}")
+    print(f"HTML: {len(result.html)} chars")
+```
+
+#### Creating Reusable Hook Libraries
+
+Build collections of reusable hooks:
+
+```python
+# hooks_library.py
+class CrawlHooks:
+    """Reusable hook collection for common crawling tasks"""
+
+    @staticmethod
+    async def block_images(page, context, **kwargs):
+        """Block all images to speed up crawling"""
+        await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda r: r.abort())
+        return page
+
+    @staticmethod
+    async def block_analytics(page, context, **kwargs):
+        """Block analytics and tracking scripts"""
+        tracking_domains = [
+            "**/google-analytics.com/*",
+            "**/googletagmanager.com/*",
+            "**/facebook.com/tr/*",
+            "**/doubleclick.net/*"
+        ]
+        for domain in tracking_domains:
+            await context.route(domain, lambda r: r.abort())
+        return page
+
+    @staticmethod
+    async def scroll_infinite(page, context, **kwargs):
+        """Handle infinite scroll to load more content"""
+        previous_height = 0
+        for i in range(5):  # Max 5 scrolls
+            current_height = await page.evaluate("document.body.scrollHeight")
+            if current_height == previous_height:
+                break
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await page.wait_for_timeout(1000)
+            previous_height = current_height
+        return page
+
+    @staticmethod
+    async def wait_for_dynamic_content(page, context, url, response, **kwargs):
+        """Wait for dynamic content to load"""
+        await page.wait_for_timeout(2000)
+        try:
+            # Click "Load More" if present
+            load_more = await page.query_selector('[class*="load-more"]')
+            if load_more:
+                await load_more.click()
+                await page.wait_for_timeout(1000)
+        except:
+            pass
+        return page
+
+# Use in your application
+from hooks_library import CrawlHooks
+from crawl4ai import Crawl4aiDockerClient
+
+async def crawl_with_optimizations(url):
+    async with Crawl4aiDockerClient() as client:
+        result = await client.crawl(
+            [url],
+            hooks={
+                "on_page_context_created": CrawlHooks.block_images,
+                "before_retrieve_html": CrawlHooks.scroll_infinite
+            }
+        )
+        return result
+```
+
+#### Choosing the Right Approach
+
+| Approach | Best For | IDE Support | Language |
+|----------|----------|-------------|----------|
+| **String-based** | Non-Python clients, REST APIs, other languages | ❌ None | Any |
+| **Function-based** | Python applications, local development | ✅ Full | Python only |
+| **Docker Client** | Python apps with automatic conversion | ✅ Full | Python only |
+
+**Recommendation**:
+- **Python applications**: Use Docker client with function objects (easiest)
+- **Non-Python or REST API**: Use string-based hooks (most flexible)
+- **Manual control**: Use `hooks_to_string()` utility (middle ground)
+
+#### Complete Example with Function Hooks
+
+```python
+from crawl4ai import Crawl4aiDockerClient, BrowserConfig, CrawlerRunConfig, CacheMode
+
+# Define hooks as regular Python functions
+async def setup_environment(page, context, **kwargs):
+    """Setup crawling environment"""
+    # Set viewport
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+
+    # Block resources for speed
+    await context.route("**/*.{png,jpg,gif}", lambda r: r.abort())
+
+    # Add custom headers
+    await page.set_extra_http_headers({
+        "Accept-Language": "en-US",
+        "X-Custom-Header": "Crawl4AI"
+    })
+
+    print("[HOOK] Environment configured")
+    return page
+
+async def extract_content(page, context, **kwargs):
+    """Extract and prepare content"""
+    # Scroll to load lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+
+    # Extract metadata
+    metadata = await page.evaluate('''() => ({
+        title: document.title,
+        links: document.links.length,
+        images: document.images.length
+    })''')
+
+    print(f"[HOOK] Page metadata: {metadata}")
+    return page
+
+async def main():
+    async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client:
+        # Configure crawl
+        browser_config = BrowserConfig(headless=True)
+        crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+
+        # Crawl with hooks
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            browser_config=browser_config,
+            crawler_config=crawler_config,
+            hooks={
+                "on_page_context_created": setup_environment,
+                "before_retrieve_html": extract_content
+            },
+            hooks_timeout=30
+        )
+
+        if result.success:
+            print(f"✅ Crawl successful!")
+            print(f"   URL: {result.url}")
+            print(f"   HTML: {len(result.html)} chars")
+            print(f"   Markdown: {len(result.markdown)} chars")
+        else:
+            print(f"❌ Crawl failed: {result.error_message}")
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
+```
+
+#### Additional Resources
+
+- **Comprehensive Examples**: See `/docs/examples/hooks_docker_client_example.py` for Python function-based examples
+- **REST API Examples**: See `/docs/examples/hooks_rest_api_example.py` for string-based examples
+- **Comparison Guide**: See `/docs/examples/README_HOOKS.md` for detailed comparison
+- **Utility Documentation**: See `/docs/hooks-utility-guide.md` for complete guide
+
+---
+
+## Job Queue & Webhook API
+
+The Docker deployment includes a powerful asynchronous job queue system with webhook support for both crawling and LLM extraction tasks. Instead of waiting for long-running operations to complete, submit jobs and receive real-time notifications via webhooks when they finish.
+
+### Why Use the Job Queue API?
+
+**Traditional Synchronous API (`/crawl`):**
+- Client waits for entire crawl to complete
+- Timeout issues with long-running crawls
+- Resource blocking during execution
+- Constant polling required for status updates
+
+**Asynchronous Job Queue API (`/crawl/job`, `/llm/job`):**
+- ✅ Submit job and continue immediately
+- ✅ No timeout concerns for long operations
+- ✅ Real-time webhook notifications on completion
+- ✅ Better resource utilization
+- ✅ Perfect for batch processing
+- ✅ Ideal for microservice architectures
+
+### Available Endpoints
+
+#### 1. Crawl Job Endpoint
+
+```
+POST /crawl/job
+```
+
+Submit an asynchronous crawl job with optional webhook notification.
+
+**Request Body:**
+```json
+{
+  "urls": ["https://example.com"],
+  "cache_mode": "bypass",
+  "extraction_strategy": {
+    "type": "JsonCssExtractionStrategy",
+    "schema": {
+      "title": "h1",
+      "content": ".article-body"
+    }
+  },
+  "webhook_config": {
+    "webhook_url": "https://your-app.com/webhook/crawl-complete",
+    "webhook_data_in_payload": true,
+    "webhook_headers": {
+      "X-Webhook-Secret": "your-secret-token",
+      "X-Custom-Header": "value"
+    }
+  }
+}
+```
+
+**Response:**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "message": "Crawl job submitted"
+}
+```
+
+#### 2. LLM Extraction Job Endpoint
+
+```
+POST /llm/job
+```
+
+Submit an asynchronous LLM extraction job with optional webhook notification.
+
+**Request Body:**
+```json
+{
+  "url": "https://example.com/article",
+  "q": "Extract the article title, author, publication date, and main points",
+  "provider": "openai/gpt-4o-mini",
+  "schema": "{\"title\": \"string\", \"author\": \"string\", \"date\": \"string\", \"points\": [\"string\"]}",
+  "cache": false,
+  "webhook_config": {
+    "webhook_url": "https://your-app.com/webhook/llm-complete",
+    "webhook_data_in_payload": true,
+    "webhook_headers": {
+      "X-Webhook-Secret": "your-secret-token"
+    }
+  }
+}
+```
+
+**Response:**
+```json
+{
+  "task_id": "llm_1698765432",
+  "message": "LLM job submitted"
+}
+```
+
+#### 3. Job Status Endpoint
+
+```
+GET /job/{task_id}
+```
+
+Check the status and retrieve results of a submitted job.
+
+**Response (In Progress):**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "status": "processing",
+  "message": "Job is being processed"
+}
+```
+
+**Response (Completed):**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "status": "completed",
+  "result": {
+    "markdown": "# Page Title\n\nContent...",
+    "extracted_content": {...},
+    "links": {...}
+  }
+}
+```
+
+### Webhook Configuration
+
+Webhooks provide real-time notifications when your jobs complete, eliminating the need for constant polling.
+
+#### Webhook Config Parameters
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `webhook_url` | string | Yes | Your HTTP(S) endpoint to receive notifications |
+| `webhook_data_in_payload` | boolean | No | Include full result data in webhook payload (default: false) |
+| `webhook_headers` | object | No | Custom headers for authentication/identification |
+
+#### Webhook Payload Format
+
+**Success Notification (Crawl Job):**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "task_type": "crawl",
+  "status": "completed",
+  "timestamp": "2025-10-22T12:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "data": {
+    "markdown": "# Page content...",
+    "extracted_content": {...},
+    "links": {...}
+  }
+}
+```
+
+**Success Notification (LLM Job):**
+```json
+{
+  "task_id": "llm_1698765432",
+  "task_type": "llm_extraction",
+  "status": "completed",
+  "timestamp": "2025-10-22T12:30:00.000000+00:00",
+  "urls": ["https://example.com/article"],
+  "data": {
+    "extracted_content": {
+      "title": "Understanding Web Scraping",
+      "author": "John Doe",
+      "date": "2025-10-22",
+      "points": ["Point 1", "Point 2"]
+    }
+  }
+}
+```
+
+**Failure Notification:**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "task_type": "crawl",
+  "status": "failed",
+  "timestamp": "2025-10-22T12:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "error": "Connection timeout after 30 seconds"
+}
+```
+
+#### Webhook Delivery & Retry
+
+- **Delivery Method:** HTTP POST to your `webhook_url`
+- **Content-Type:** `application/json`
+- **Retry Policy:** Exponential backoff with 5 attempts
+  - Attempt 1: Immediate
+  - Attempt 2: 1 second delay
+  - Attempt 3: 2 seconds delay
+  - Attempt 4: 4 seconds delay
+  - Attempt 5: 8 seconds delay
+- **Success Status Codes:** 200-299
+- **Custom Headers:** Your `webhook_headers` are included in every request
+
+### Usage Examples
+
+#### Example 1: Python with Webhook Handler (Flask)
+
+```python
+from flask import Flask, request, jsonify
+import requests
+
+app = Flask(__name__)
+
+# Webhook handler
+@app.route('/webhook/crawl-complete', methods=['POST'])
+def handle_crawl_webhook():
+    payload = request.json
+
+    if payload['status'] == 'completed':
+        print(f"✅ Job {payload['task_id']} completed!")
+        print(f"Task type: {payload['task_type']}")
+
+        # Access the crawl results
+        if 'data' in payload:
+            markdown = payload['data'].get('markdown', '')
+            extracted = payload['data'].get('extracted_content', {})
+            print(f"Extracted {len(markdown)} characters")
+            print(f"Structured data: {extracted}")
+    else:
+        print(f"❌ Job {payload['task_id']} failed: {payload.get('error')}")
+
+    return jsonify({"status": "received"}), 200
+
+# Submit a crawl job with webhook
+def submit_crawl_job():
+    response = requests.post(
+        "http://localhost:11235/crawl/job",
+        json={
+            "urls": ["https://example.com"],
+            "extraction_strategy": {
+                "type": "JsonCssExtractionStrategy",
+                "schema": {
+                    "name": "Example Schema",
+                    "baseSelector": "body",
+                    "fields": [
+                        {"name": "title", "selector": "h1", "type": "text"},
+                        {"name": "description", "selector": "meta[name='description']", "type": "attribute", "attribute": "content"}
+                    ]
+                }
+            },
+            "webhook_config": {
+                "webhook_url": "https://your-app.com/webhook/crawl-complete",
+                "webhook_data_in_payload": True,
+                "webhook_headers": {
+                    "X-Webhook-Secret": "your-secret-token"
+                }
+            }
+        }
+    )
+
+    task_id = response.json()['task_id']
+    print(f"Job submitted: {task_id}")
+    return task_id
+
+if __name__ == '__main__':
+    app.run(port=5000)
+```
+
+#### Example 2: LLM Extraction with Webhooks
+
+```python
+import requests
+
+def submit_llm_job_with_webhook():
+    response = requests.post(
+        "http://localhost:11235/llm/job",
+        json={
+            "url": "https://example.com/article",
+            "q": "Extract the article title, author, and main points",
+            "provider": "openai/gpt-4o-mini",
+            "webhook_config": {
+                "webhook_url": "https://your-app.com/webhook/llm-complete",
+                "webhook_data_in_payload": True,
+                "webhook_headers": {
+                    "X-Webhook-Secret": "your-secret-token"
+                }
+            }
+        }
+    )
+
+    task_id = response.json()['task_id']
+    print(f"LLM job submitted: {task_id}")
+    return task_id
+
+# Webhook handler for LLM jobs
+@app.route('/webhook/llm-complete', methods=['POST'])
+def handle_llm_webhook():
+    payload = request.json
+
+    if payload['status'] == 'completed':
+        extracted = payload['data']['extracted_content']
+        print(f"✅ LLM extraction completed!")
+        print(f"Results: {extracted}")
+    else:
+        print(f"❌ LLM extraction failed: {payload.get('error')}")
+
+    return jsonify({"status": "received"}), 200
+```
+
+#### Example 3: Without Webhooks (Polling)
+
+If you don't use webhooks, you can poll for results:
+
+```python
+import requests
+import time
+
+# Submit job
+response = requests.post(
+    "http://localhost:11235/crawl/job",
+    json={"urls": ["https://example.com"]}
+)
+task_id = response.json()['task_id']
+
+# Poll for results
+while True:
+    result = requests.get(f"http://localhost:11235/job/{task_id}")
+    data = result.json()
+
+    if data['status'] == 'completed':
+        print("Job completed!")
+        print(data['result'])
+        break
+    elif data['status'] == 'failed':
+        print(f"Job failed: {data.get('error')}")
+        break
+
+    print("Still processing...")
+    time.sleep(2)
+```
+
+#### Example 4: Global Webhook Configuration
+
+Set a default webhook URL in your `config.yml` to avoid repeating it in every request:
+
+```yaml
+# config.yml
+api:
+  crawler:
+    # ... other settings ...
+    webhook:
+      default_url: "https://your-app.com/webhook/default"
+      default_headers:
+        X-Webhook-Secret: "your-secret-token"
+```
+
+Then submit jobs without webhook config:
+
+```python
+# Uses the global webhook configuration
+response = requests.post(
+    "http://localhost:11235/crawl/job",
+    json={"urls": ["https://example.com"]}
+)
+```
+
+### Webhook Best Practices
+
+1. **Authentication:** Always use custom headers for webhook authentication
+   ```json
+   "webhook_headers": {
+     "X-Webhook-Secret": "your-secret-token"
+   }
+   ```
+
+2. **Idempotency:** Design your webhook handler to be idempotent (safe to receive duplicate notifications)
+
+3. **Fast Response:** Return HTTP 200 quickly; process data asynchronously if needed
+   ```python
+   @app.route('/webhook', methods=['POST'])
+   def webhook():
+       payload = request.json
+       # Queue for background processing
+       queue.enqueue(process_webhook, payload)
+       return jsonify({"status": "received"}), 200
+   ```
+
+4. **Error Handling:** Handle both success and failure notifications
+   ```python
+   if payload['status'] == 'completed':
+       # Process success
+   elif payload['status'] == 'failed':
+       # Log error, retry, or alert
+   ```
+
+5. **Validation:** Verify webhook authenticity using custom headers
+   ```python
+   secret = request.headers.get('X-Webhook-Secret')
+   if secret != os.environ['EXPECTED_SECRET']:
+       return jsonify({"error": "Unauthorized"}), 401
+   ```
+
+6. **Logging:** Log webhook deliveries for debugging
+   ```python
+   logger.info(f"Webhook received: {payload['task_id']} - {payload['status']}")
+   ```
+
+### Use Cases
+
+**1. Batch Processing**
+Submit hundreds of URLs and get notified as each completes:
+```python
+urls = ["https://site1.com", "https://site2.com", ...]
+for url in urls:
+    submit_crawl_job(url, webhook_url="https://app.com/webhook")
+```
+
+**2. Microservice Integration**
+Integrate with event-driven architectures:
+```python
+# Service A submits job
+task_id = submit_crawl_job(url)
+
+# Service B receives webhook and triggers next step
+@app.route('/webhook')
+def webhook():
+    process_result(request.json)
+    trigger_next_service()
+    return "OK", 200
+```
+
+**3. Long-Running Extractions**
+Handle complex LLM extractions without timeouts:
+```python
+submit_llm_job(
+    url="https://long-article.com",
+    q="Comprehensive summary with key points and analysis",
+    webhook_url="https://app.com/webhook/llm"
+)
+```
+
+### Troubleshooting
+
+**Webhook not receiving notifications?**
+- Check your webhook URL is publicly accessible
+- Verify firewall/security group settings
+- Use webhook testing tools like webhook.site for debugging
+- Check server logs for delivery attempts
+- Ensure your handler returns 200-299 status code
+
+**Job stuck in processing?**
+- Check Redis connection: `docker logs  | grep redis`
+- Verify worker processes: `docker exec  ps aux | grep worker`
+- Check server logs: `docker logs `
+
+**Need to cancel a job?**
+Jobs are processed asynchronously. If you need to cancel:
+- Delete the task from Redis (requires Redis CLI access)
+- Or implement a cancellation endpoint in your webhook handler
+
+---
+
+## Dockerfile Parameters
+
+You can customize the image build process using build arguments (`--build-arg`). These are typically used via `docker buildx build` or within the `docker-compose.yml` file.
+
+```bash
+# Example: Build with 'all' features using buildx
+docker buildx build \
+  --platform linux/amd64,linux/arm64 \
+  --build-arg INSTALL_TYPE=all \
+  -t yourname/crawl4ai-all:latest \
+  --load \
+  . # Build from root context
+```
+
+### Build Arguments Explained
+
+| Argument     | Description                              | Default   | Options                            |
+| :----------- | :--------------------------------------- | :-------- | :--------------------------------- |
+| INSTALL_TYPE | Feature set                              | `default` | `default`, `all`, `torch`, `transformer` |
+| ENABLE_GPU   | GPU support (CUDA for AMD64)           | `false`   | `true`, `false`                    |
+| APP_HOME     | Install path inside container (advanced) | `/app`    | any valid path                   |
+| USE_LOCAL    | Install library from local source        | `true`    | `true`, `false`                    |
+| GITHUB_REPO  | Git repo to clone if USE_LOCAL=false   | *(see Dockerfile)* | any git URL                  |
+| GITHUB_BRANCH| Git branch to clone if USE_LOCAL=false   | `main`    | any branch name                  |
+
+*(Note: PYTHON_VERSION is fixed by the `FROM` instruction in the Dockerfile)*
+
+### Build Best Practices
+
+1.  **Choose the Right Install Type**
+    *   `default`: Basic installation, smallest image size. Suitable for most standard web scraping and markdown generation.
+    *   `all`: Full features including `torch` and `transformers` for advanced extraction strategies (e.g., CosineStrategy, certain LLM filters). Significantly larger image. Ensure you need these extras.
+2.  **Platform Considerations**
+    *   Use `buildx` for building multi-architecture images, especially for pushing to registries.
+    *   Use `docker compose` profiles (`local-amd64`, `local-arm64`) for easy platform-specific local builds.
+3.  **Performance Optimization**
+    *   The image automatically includes platform-specific optimizations (OpenMP for AMD64, OpenBLAS for ARM64).
+
+---
+
+## Using the API
+
+Communicate with the running Docker server via its REST API (defaulting to `http://localhost:11235`). You can use the Python SDK or make direct HTTP requests.
+
+### Playground Interface
+
+A built-in web playground is available at `http://localhost:11235/playground` for testing and generating API requests. The playground allows you to:
+
+1. Configure `CrawlerRunConfig` and `BrowserConfig` using the main library's Python syntax
+2. Test crawling operations directly from the interface
+3. Generate corresponding JSON for REST API requests based on your configuration
+
+This is the easiest way to translate Python configuration to JSON requests when building integrations.
+
+### Python SDK
+
+Install the SDK: `pip install crawl4ai`
+
+The Python SDK provides a convenient way to interact with the Docker API, including **automatic hook conversion** when using function objects.
+
+```python
+import asyncio
+from crawl4ai.docker_client import Crawl4aiDockerClient
+from crawl4ai import BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def main():
+    # Point to the correct server port
+    async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client:
+        # If JWT is enabled on the server, authenticate first:
+        # await client.authenticate("user@example.com") # See Server Configuration section
+
+        # Example Non-streaming crawl
+        print("--- Running Non-Streaming Crawl ---")
+        results = await client.crawl(
+            ["https://httpbin.org/html"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+        )
+        if results:
+            print(f"Non-streaming results success: {results.success}")
+            if results.success:
+                for result in results:
+                    print(f"URL: {result.url}, Success: {result.success}")
+        else:
+            print("Non-streaming crawl failed.")
+
+        # Example Streaming crawl
+        print("\n--- Running Streaming Crawl ---")
+        stream_config = CrawlerRunConfig(stream=True, cache_mode=CacheMode.BYPASS)
+        try:
+            async for result in await client.crawl(
+                ["https://httpbin.org/html", "https://httpbin.org/links/5/0"],
+                browser_config=BrowserConfig(headless=True),
+                crawler_config=stream_config
+            ):
+                print(f"Streamed result: URL: {result.url}, Success: {result.success}")
+        except Exception as e:
+            print(f"Streaming crawl failed: {e}")
+
+        # Example with hooks (Python function objects)
+        print("\n--- Crawl with Hooks ---")
+
+        async def my_hook(page, context, **kwargs):
+            """Custom hook to optimize performance"""
+            await page.set_viewport_size({"width": 1920, "height": 1080})
+            await context.route("**/*.{png,jpg}", lambda r: r.abort())
+            print("[HOOK] Page optimized")
+            return page
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+            hooks={"on_page_context_created": my_hook},  # Pass function directly!
+            hooks_timeout=30
+        )
+        print(f"Crawl with hooks success: {result.success}")
+
+        # Example Get schema
+        print("\n--- Getting Schema ---")
+        schema = await client.get_schema()
+        print(f"Schema received: {bool(schema)}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+#### SDK Parameters
+
+The Docker client supports the following parameters:
+
+**Client Initialization**:
+- `base_url` (str): URL of the Docker server (default: `http://localhost:8000`)
+- `timeout` (float): Request timeout in seconds (default: 30.0)
+- `verify_ssl` (bool): Verify SSL certificates (default: True)
+- `verbose` (bool): Enable verbose logging (default: True)
+- `log_file` (Optional[str]): Path to log file (default: None)
+
+**crawl() Method**:
+- `urls` (List[str]): List of URLs to crawl
+- `browser_config` (Optional[BrowserConfig]): Browser configuration
+- `crawler_config` (Optional[CrawlerRunConfig]): Crawler configuration
+- `hooks` (Optional[Dict]): Hook functions or strings - **automatically converts function objects!**
+- `hooks_timeout` (int): Timeout for each hook execution in seconds (default: 30)
+
+**Returns**:
+- Single URL: `CrawlResult` object
+- Multiple URLs: `List[CrawlResult]`
+- Streaming: `AsyncGenerator[CrawlResult]`
+
+### Second Approach: Direct API Calls
+
+Crucially, when sending configurations directly via JSON, they **must** follow the `{"type": "ClassName", "params": {...}}` structure for any non-primitive value (like config objects or strategies). Dictionaries must be wrapped as `{"type": "dict", "value": {...}}`.
+
+*(Keep the detailed explanation of Configuration Structure, Basic Pattern, Simple vs Complex, Strategy Pattern, Complex Nested Example, Quick Grammar Overview, Important Rules, Pro Tip)*
+
+#### More Examples *(Ensure Schema example uses type/value wrapper)*
+
+**Advanced Crawler Configuration**
+*(Keep example, ensure cache_mode uses valid enum value like "bypass")*
+
+**Extraction Strategy**
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "extraction_strategy": {
+                "type": "JsonCssExtractionStrategy",
+                "params": {
+                    "schema": {
+                        "type": "dict",
+                        "value": {
+                           "baseSelector": "article.post",
+                           "fields": [
+                               {"name": "title", "selector": "h1", "type": "text"},
+                               {"name": "content", "selector": ".content", "type": "html"}
+                           ]
+                         }
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+**LLM Extraction Strategy** *(Keep example, ensure schema uses type/value wrapper)*
+*(Keep Deep Crawler Example)*
+
+### LLM Configuration Examples
+
+The Docker API supports dynamic LLM configuration through multiple levels:
+
+#### Temperature Control
+
+Temperature affects the randomness of LLM responses (0.0 = deterministic, 2.0 = very creative):
+
+```python
+import requests
+
+# Low temperature for factual extraction
+response = requests.post(
+    "http://localhost:11235/md",
+    json={
+        "url": "https://example.com",
+        "f": "llm",
+        "q": "Extract all dates and numbers from this page",
+        "temperature": 0.2  # Very focused, deterministic
+    }
+)
+
+# High temperature for creative tasks
+response = requests.post(
+    "http://localhost:11235/md",
+    json={
+        "url": "https://example.com", 
+        "f": "llm",
+        "q": "Write a creative summary of this content",
+        "temperature": 1.2  # More creative, varied responses
+    }
+)
+```
+
+#### Custom API Endpoints
+
+Use custom base URLs for proxy servers or alternative API endpoints:
+
+```python
+
+# Using a local LLM server
+response = requests.post(
+    "http://localhost:11235/md",
+    json={
+        "url": "https://example.com",
+        "f": "llm",
+        "q": "Extract key information",
+        "provider": "ollama/llama2",
+        "base_url": "http://localhost:11434/v1"
+    }
+)
+```
+
+#### Dynamic Provider Selection
+
+Switch between providers based on task requirements:
+
+```python
+async def smart_extraction(url: str, content_type: str):
+    """Select provider and temperature based on content type"""
+    
+    configs = {
+        "technical": {
+            "provider": "openai/gpt-4",
+            "temperature": 0.3,
+            "query": "Extract technical specifications and code examples"
+        },
+        "creative": {
+            "provider": "anthropic/claude-3-opus",
+            "temperature": 0.9,
+            "query": "Create an engaging narrative summary"
+        },
+        "quick": {
+            "provider": "groq/mixtral-8x7b",
+            "temperature": 0.5,
+            "query": "Quick summary in bullet points"
+        }
+    }
+    
+    config = configs.get(content_type, configs["quick"])
+    
+    response = await httpx.post(
+        "http://localhost:11235/md",
+        json={
+            "url": url,
+            "f": "llm",
+            "q": config["query"],
+            "provider": config["provider"],
+            "temperature": config["temperature"]
+        }
+    )
+    
+    return response.json()
+```
+
+### REST API Examples
+
+Update URLs to use port `11235`.
+
+#### Simple Crawl
+
+```python
+import requests
+
+# Configuration objects converted to the required JSON structure
+browser_config_payload = {
+    "type": "BrowserConfig",
+    "params": {"headless": True}
+}
+crawler_config_payload = {
+    "type": "CrawlerRunConfig",
+    "params": {"stream": False, "cache_mode": "bypass"} # Use string value of enum
+}
+
+crawl_payload = {
+    "urls": ["https://httpbin.org/html"],
+    "browser_config": browser_config_payload,
+    "crawler_config": crawler_config_payload
+}
+response = requests.post(
+    "http://localhost:11235/crawl", # Updated port
+    # headers={"Authorization": f"Bearer {token}"},  # If JWT is enabled
+    json=crawl_payload
+)
+print(f"Status Code: {response.status_code}")
+if response.ok:
+    print(response.json())
+else:
+    print(f"Error: {response.text}")
+
+```
+
+#### Streaming Results
+
+```python
+import json
+import httpx # Use httpx for async streaming example
+
+async def test_stream_crawl(token: str = None): # Made token optional
+    """Test the /crawl/stream endpoint with multiple URLs."""
+    url = "http://localhost:11235/crawl/stream" # Updated port
+    payload = {
+        "urls": [
+            "https://httpbin.org/html",
+            "https://httpbin.org/links/5/0",
+        ],
+        "browser_config": {
+            "type": "BrowserConfig",
+            "params": {"headless": True, "viewport": {"type": "dict", "value": {"width": 1200, "height": 800}}} # Viewport needs type:dict
+        },
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {"stream": True, "cache_mode": "bypass"}
+        }
+    }
+
+    headers = {}
+    # if token:
+    #    headers = {"Authorization": f"Bearer {token}"} # If JWT is enabled
+
+    try:
+        async with httpx.AsyncClient() as client:
+            async with client.stream("POST", url, json=payload, headers=headers, timeout=120.0) as response:
+                print(f"Status: {response.status_code} (Expected: 200)")
+                response.raise_for_status() # Raise exception for bad status codes
+
+                # Read streaming response line-by-line (NDJSON)
+                async for line in response.aiter_lines():
+                    if line:
+                        try:
+                            data = json.loads(line)
+                            # Check for completion marker
+                            if data.get("status") == "completed":
+                                print("Stream completed.")
+                                break
+                            print(f"Streamed Result: {json.dumps(data, indent=2)}")
+                        except json.JSONDecodeError:
+                            print(f"Warning: Could not decode JSON line: {line}")
+
+    except httpx.HTTPStatusError as e:
+         print(f"HTTP error occurred: {e.response.status_code} - {e.response.text}")
+    except Exception as e:
+        print(f"Error in streaming crawl test: {str(e)}")
+
+# To run this example:
+# import asyncio
+# asyncio.run(test_stream_crawl())
+```
+
+---
+
+## Real-time Monitoring & Operations
+
+One of the key advantages of self-hosting is complete visibility into your infrastructure. Crawl4AI includes a comprehensive real-time monitoring system that gives you full transparency and control.
+
+### Monitoring Dashboard
+
+Access the **built-in real-time monitoring dashboard** for complete operational visibility:
+
+```
+http://localhost:11235/monitor
+```
+
+![Monitoring Dashboard](https://via.placeholder.com/800x400?text=Crawl4AI+Monitoring+Dashboard)
+
+**Dashboard Features:**
+
+#### 1. System Health Overview
+- **CPU & Memory**: Live usage with progress bars and percentage indicators
+- **Network I/O**: Total bytes sent/received since startup
+- **Server Uptime**: How long your server has been running
+- **Browser Pool Status**:
+  - 🔥 Permanent browser (always-on default config, ~270MB)
+  - ♨️ Hot pool (frequently used configs, ~180MB each)
+  - ❄️ Cold pool (idle browsers awaiting cleanup, ~180MB each)
+- **Memory Pressure**: LOW/MEDIUM/HIGH indicator for janitor behavior
+
+#### 2. Live Request Tracking
+- **Active Requests**: Currently running crawls with:
+  - Request ID for tracking
+  - Target URL (truncated for display)
+  - Endpoint being used
+  - Elapsed time (updates in real-time)
+  - Memory usage from start
+- **Completed Requests**: Last 10 finished requests showing:
+  - Success/failure status (color-coded)
+  - Total execution time
+  - Memory delta (how much memory changed)
+  - Pool hit (was browser reused?)
+  - HTTP status code
+- **Filtering**: View all, success only, or errors only
+
+#### 3. Browser Pool Management
+Interactive table showing all active browsers:
+
+| Type | Signature | Age | Last Used | Hits | Actions |
+|------|-----------|-----|-----------|------|---------|
+| permanent | abc12345 | 2h | 5s ago | 1,247 | Restart |
+| hot | def67890 | 45m | 2m ago | 89 | Kill / Restart |
+| cold | ghi11213 | 30m | 15m ago | 3 | Kill / Restart |
+
+- **Reuse Rate**: Percentage of requests that reused existing browsers
+- **Memory Estimates**: Total memory used by browser pool
+- **Manual Control**: Kill or restart individual browsers
+
+#### 4. Janitor Events Log
+Real-time log of browser pool cleanup events:
+- When cold browsers are closed due to memory pressure
+- When browsers are promoted from cold to hot pool
+- Forced cleanups triggered manually
+- Detailed cleanup reasons and browser signatures
+
+#### 5. Error Monitoring
+Recent errors with full context:
+- Timestamp
+- Endpoint where error occurred
+- Target URL
+- Error message
+- Request ID for correlation
+
+**Live Updates:**
+The dashboard connects via WebSocket and refreshes every **2 seconds** with the latest data. Connection status indicator shows when you're connected/disconnected.
+
+---
+
+### Monitor API Endpoints
+
+For programmatic monitoring, automation, and integration with your existing infrastructure:
+
+#### Health & Statistics
+
+**Get System Health**
+```bash
+GET /monitor/health
+```
+
+Returns current system snapshot:
+```json
+{
+  "container": {
+    "memory_percent": 45.2,
+    "cpu_percent": 23.1,
+    "network_sent_mb": 1250.45,
+    "network_recv_mb": 3421.12,
+    "uptime_seconds": 7234
+  },
+  "pool": {
+    "permanent": {"active": true, "memory_mb": 270},
+    "hot": {"count": 3, "memory_mb": 540},
+    "cold": {"count": 1, "memory_mb": 180},
+    "total_memory_mb": 990
+  },
+  "janitor": {
+    "next_cleanup_estimate": "adaptive",
+    "memory_pressure": "MEDIUM"
+  }
+}
+```
+
+**Get Request Statistics**
+```bash
+GET /monitor/requests?status=all&limit=50
+```
+
+Query parameters:
+- `status`: Filter by `all`, `active`, `completed`, `success`, or `error`
+- `limit`: Number of completed requests to return (1-1000)
+
+**Get Browser Pool Details**
+```bash
+GET /monitor/browsers
+```
+
+Returns detailed information about all active browsers:
+```json
+{
+  "browsers": [
+    {
+      "type": "permanent",
+      "sig": "abc12345",
+      "age_seconds": 7234,
+      "last_used_seconds": 5,
+      "memory_mb": 270,
+      "hits": 1247,
+      "killable": false
+    },
+    {
+      "type": "hot",
+      "sig": "def67890",
+      "age_seconds": 2701,
+      "last_used_seconds": 120,
+      "memory_mb": 180,
+      "hits": 89,
+      "killable": true
+    }
+  ],
+  "summary": {
+    "total_count": 5,
+    "total_memory_mb": 990,
+    "reuse_rate_percent": 87.3
+  }
+}
+```
+
+**Get Endpoint Performance Statistics**
+```bash
+GET /monitor/endpoints/stats
+```
+
+Returns aggregated metrics per endpoint:
+```json
+{
+  "/crawl": {
+    "count": 1523,
+    "avg_latency_ms": 2341.5,
+    "success_rate_percent": 98.2,
+    "pool_hit_rate_percent": 89.1,
+    "errors": 27
+  },
+  "/md": {
+    "count": 891,
+    "avg_latency_ms": 1823.7,
+    "success_rate_percent": 99.4,
+    "pool_hit_rate_percent": 92.3,
+    "errors": 5
+  }
+}
+```
+
+**Get Timeline Data**
+```bash
+GET /monitor/timeline?metric=memory&window=5m
+```
+
+Parameters:
+- `metric`: `memory`, `requests`, or `browsers`
+- `window`: Currently only `5m` (5-minute window, 5-second resolution)
+
+Returns time-series data for charts:
+```json
+{
+  "timestamps": [1699564800, 1699564805, 1699564810, ...],
+  "values": [42.1, 43.5, 41.8, ...]
+}
+```
+
+#### Logs
+
+**Get Janitor Events**
+```bash
+GET /monitor/logs/janitor?limit=100
+```
+
+**Get Error Log**
+```bash
+GET /monitor/logs/errors?limit=100
+```
+
+---
+
+### WebSocket Streaming
+
+For real-time monitoring in your own dashboards or applications:
+
+```bash
+WS /monitor/ws
+```
+
+**Connection Example (Python):**
+```python
+import asyncio
+import websockets
+import json
+
+async def monitor_server():
+    uri = "ws://localhost:11235/monitor/ws"
+
+    async with websockets.connect(uri) as websocket:
+        print("Connected to Crawl4AI monitor")
+
+        while True:
+            # Receive update every 2 seconds
+            data = await websocket.recv()
+            update = json.loads(data)
+
+            # Extract key metrics
+            health = update['health']
+            active_requests = len(update['requests']['active'])
+            browsers = len(update['browsers'])
+
+            print(f"Memory: {health['container']['memory_percent']:.1f}% | "
+                  f"Active: {active_requests} | "
+                  f"Browsers: {browsers}")
+
+            # Check for high memory pressure
+            if health['janitor']['memory_pressure'] == 'HIGH':
+                print("⚠️  HIGH MEMORY PRESSURE - Consider cleanup")
+
+asyncio.run(monitor_server())
+```
+
+**Update Payload Structure:**
+```json
+{
+  "timestamp": 1699564823.456,
+  "health": { /* System health snapshot */ },
+  "requests": {
+    "active": [ /* Currently running */ ],
+    "completed": [ /* Last 10 completed */ ]
+  },
+  "browsers": [ /* All active browsers */ ],
+  "timeline": {
+    "memory": { /* Last 5 minutes */ },
+    "requests": { /* Request rate */ },
+    "browsers": { /* Pool composition */ }
+  },
+  "janitor": [ /* Last 10 cleanup events */ ],
+  "errors": [ /* Last 10 errors */ ]
+}
+```
+
+---
+
+### Control Actions
+
+Take manual control when needed:
+
+**Force Immediate Cleanup**
+```bash
+POST /monitor/actions/cleanup
+```
+
+Kills all cold pool browsers immediately (useful when memory is tight):
+```json
+{
+  "success": true,
+  "killed_browsers": 3
+}
+```
+
+**Kill Specific Browser**
+```bash
+POST /monitor/actions/kill_browser
+Content-Type: application/json
+
+{
+  "sig": "abc12345"  // First 8 chars of browser signature
+}
+```
+
+Response:
+```json
+{
+  "success": true,
+  "killed_sig": "abc12345",
+  "pool_type": "hot"
+}
+```
+
+**Restart Browser**
+```bash
+POST /monitor/actions/restart_browser
+Content-Type: application/json
+
+{
+  "sig": "permanent"  // Or first 8 chars of signature
+}
+```
+
+For permanent browser, this will close and reinitialize it. For hot/cold browsers, it kills them and lets new requests create fresh ones.
+
+**Reset Statistics**
+```bash
+POST /monitor/stats/reset
+```
+
+Clears endpoint counters (useful for starting fresh after testing).
+
+---
+
+### Production Integration
+
+#### Integration with Existing Monitoring Systems
+
+**Prometheus Integration:**
+```bash
+# Scrape metrics endpoint
+curl http://localhost:11235/metrics
+```
+
+**Custom Dashboard Integration:**
+```python
+# Example: Push metrics to your monitoring system
+import asyncio
+import websockets
+import json
+from your_monitoring import push_metric
+
+async def integrate_monitoring():
+    async with websockets.connect("ws://localhost:11235/monitor/ws") as ws:
+        while True:
+            data = json.loads(await ws.recv())
+
+            # Push to your monitoring system
+            push_metric("crawl4ai.memory.percent",
+                       data['health']['container']['memory_percent'])
+            push_metric("crawl4ai.active_requests",
+                       len(data['requests']['active']))
+            push_metric("crawl4ai.browser_count",
+                       len(data['browsers']))
+```
+
+**Alerting Example:**
+```python
+import requests
+import time
+
+def check_health():
+    """Poll health endpoint and alert on issues"""
+    response = requests.get("http://localhost:11235/monitor/health")
+    health = response.json()
+
+    # Alert on high memory
+    if health['container']['memory_percent'] > 85:
+        send_alert(f"High memory: {health['container']['memory_percent']}%")
+
+    # Alert on high error rate
+    stats = requests.get("http://localhost:11235/monitor/endpoints/stats").json()
+    for endpoint, metrics in stats.items():
+        if metrics['success_rate_percent'] < 95:
+            send_alert(f"{endpoint} success rate: {metrics['success_rate_percent']}%")
+
+# Run every minute
+while True:
+    check_health()
+    time.sleep(60)
+```
+
+**Log Aggregation:**
+```python
+import requests
+from datetime import datetime
+
+def aggregate_errors():
+    """Fetch and aggregate errors for logging system"""
+    response = requests.get("http://localhost:11235/monitor/logs/errors?limit=100")
+    errors = response.json()['errors']
+
+    for error in errors:
+        log_to_system({
+            'timestamp': datetime.fromtimestamp(error['timestamp']),
+            'service': 'crawl4ai',
+            'endpoint': error['endpoint'],
+            'url': error['url'],
+            'message': error['error'],
+            'request_id': error['request_id']
+        })
+```
+
+#### Key Metrics to Track
+
+For production self-hosted deployments, monitor these metrics:
+
+1. **Memory Usage Trends**
+   - Track `container.memory_percent` over time
+   - Alert when consistently above 80%
+   - Prevents OOM kills
+
+2. **Request Success Rates**
+   - Monitor per-endpoint success rates
+   - Alert when below 95%
+   - Indicates crawling issues
+
+3. **Average Latency**
+   - Track `avg_latency_ms` per endpoint
+   - Detect performance degradation
+   - Optimize slow endpoints
+
+4. **Browser Pool Efficiency**
+   - Monitor `reuse_rate_percent`
+   - Should be >80% for good efficiency
+   - Low rates indicate pool churn
+
+5. **Error Frequency**
+   - Count errors per time window
+   - Alert on sudden spikes
+   - Track error patterns
+
+6. **Janitor Activity**
+   - Monitor cleanup frequency
+   - Excessive cleanup indicates memory pressure
+   - Adjust pool settings if needed
+
+---
+
+### Quick Health Check
+
+For simple uptime monitoring:
+
+```bash
+curl http://localhost:11235/health
+```
+
+Returns:
+```json
+{
+  "status": "healthy",
+  "version": "0.7.4"
+}
+```
+
+Other useful endpoints:
+- `/metrics` - Prometheus metrics
+- `/schema` - Full API schema
+
+---
+
+## Server Configuration
+
+The server's behavior can be customized through the `config.yml` file.
+
+### Understanding config.yml
+
+The configuration file is loaded from `/app/config.yml` inside the container. By default, the file from `deploy/docker/config.yml` in the repository is copied there during the build.
+
+Here's a detailed breakdown of the configuration options (using defaults from `deploy/docker/config.yml`):
+
+```yaml
+# Application Configuration
+app:
+  title: "Crawl4AI API"
+  version: "1.0.0" # Consider setting this to match library version, e.g., "0.5.1"
+  host: "0.0.0.0"
+  port: 8020 # NOTE: This port is used ONLY when running server.py directly. Gunicorn overrides this (see supervisord.conf).
+  reload: False # Default set to False - suitable for production
+  timeout_keep_alive: 300
+
+# Default LLM Configuration
+llm:
+  provider: "openai/gpt-4o-mini"  # Can be overridden by LLM_PROVIDER env var
+  # api_key: sk-...  # If you pass the API key directly (not recommended)
+  # temperature and base_url are controlled via environment variables or request parameters
+
+# Redis Configuration (Used by internal Redis server managed by supervisord)
+redis:
+  host: "localhost"
+  port: 6379
+  db: 0
+  password: ""
+  # ... other redis options ...
+
+# Rate Limiting Configuration
+rate_limiting:
+  enabled: True
+  default_limit: "1000/minute"
+  trusted_proxies: []
+  storage_uri: "memory://"  # Use "redis://localhost:6379" if you need persistent/shared limits
+
+# Security Configuration
+# NOTE (0.9.0): the defaults below reflect 0.8.x. In 0.9.0 the Docker server is
+# secure-by-default - authentication is required, the server binds loopback
+# unless a token is set, and request bodies are validated against a trust
+# boundary. See the migration guide for the 0.9.0 defaults and config keys:
+# https://github.com/unclecode/crawl4ai/blob/main/deploy/docker/MIGRATION.md
+security:
+  enabled: false # Master toggle for security features
+  jwt_enabled: false # Enable JWT authentication (requires security.enabled=true)
+  https_redirect: false # Force HTTPS (requires security.enabled=true)
+  trusted_hosts: ["*"] # Allowed hosts (use specific domains in production)
+  headers: # Security headers (applied if security.enabled=true)
+    x_content_type_options: "nosniff"
+    x_frame_options: "DENY"
+    content_security_policy: "default-src 'self'"
+    strict_transport_security: "max-age=63072000; includeSubDomains"
+
+# Crawler Configuration
+crawler:
+  memory_threshold_percent: 95.0
+  rate_limiter:
+    base_delay: [1.0, 2.0] # Min/max delay between requests in seconds for dispatcher
+  timeouts:
+    stream_init: 30.0  # Timeout for stream initialization
+    batch_process: 300.0 # Timeout for non-streaming /crawl processing
+
+# Logging Configuration
+logging:
+  level: "INFO"
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Observability Configuration
+observability:
+  prometheus:
+    enabled: True
+    endpoint: "/metrics"
+  health_check:
+    endpoint: "/health"
+```
+
+*(JWT Authentication section remains the same, just note the default port is now 11235 for requests)*
+
+*(Configuration Tips and Best Practices remain the same)*
+
+### Customizing Your Configuration
+
+You can override the default `config.yml`.
+
+#### Method 1: Modify Before Build
+
+1.  Edit the `deploy/docker/config.yml` file in your local repository clone.
+2.  Build the image using `docker buildx` or `docker compose --profile local-... up --build`. The modified file will be copied into the image.
+
+#### Method 2: Runtime Mount (Recommended for Custom Deploys)
+
+1.  Create your custom configuration file, e.g., `my-custom-config.yml` locally. Ensure it contains all necessary sections.
+2.  Mount it when running the container:
+
+    *   **Using `docker run`:**
+        ```bash
+        # Assumes my-custom-config.yml is in the current directory
+        docker run -d -p 11235:11235 \
+          --name crawl4ai-custom-config \
+          --env-file .llm.env \
+          --shm-size=1g \
+          -v $(pwd)/my-custom-config.yml:/app/config.yml \
+          unclecode/crawl4ai:latest # Or your specific tag
+        ```
+
+    *   **Using `docker-compose.yml`:** Add a `volumes` section to the service definition:
+        ```yaml
+        services:
+          crawl4ai-hub-amd64: # Or your chosen service
+            image: unclecode/crawl4ai:latest
+            profiles: ["hub-amd64"]
+            <<: *base-config
+            volumes:
+              # Mount local custom config over the default one in the container
+              - ./my-custom-config.yml:/app/config.yml
+              # Keep the shared memory volume from base-config
+              - /dev/shm:/dev/shm
+        ```
+        *(Note: Ensure `my-custom-config.yml` is in the same directory as `docker-compose.yml`)*
+
+> 💡 When mounting, your custom file *completely replaces* the default one. Ensure it's a valid and complete configuration.
+
+### Configuration Recommendations
+
+1. **Security First** 🔒
+   - Always enable security in production
+   - Use specific trusted_hosts instead of wildcards
+   - Set up proper rate limiting to protect your server
+   - Consider your environment before enabling HTTPS redirect
+
+2. **Resource Management** 💻
+   - Adjust memory_threshold_percent based on available RAM
+   - Set timeouts according to your content size and network conditions
+   - Use Redis for rate limiting in multi-container setups
+
+3. **Monitoring** 📊
+   - Enable Prometheus if you need metrics
+   - Set DEBUG logging in development, INFO in production
+   - Regular health check monitoring is crucial
+
+4. **Performance Tuning** ⚡
+   - Start with conservative rate limiter delays
+   - Increase batch_process timeout for large content
+   - Adjust stream_init timeout based on initial response times
+
+## Getting Help
+
+We're here to help you succeed with Crawl4AI! Here's how to get support:
+
+- 📖 Check our [full documentation](https://docs.crawl4ai.com)
+- 🐛 Found a bug? [Open an issue](https://github.com/unclecode/crawl4ai/issues)
+- 💬 Join our [Discord community](https://discord.gg/crawl4ai)
+- ⭐ Star us on GitHub to show support!
+
+## Summary
+
+Congratulations! You now have everything you need to self-host your own Crawl4AI infrastructure with complete control and visibility.
+
+**What You've Learned:**
+- ✅ Multiple deployment options (Docker Hub, Docker Compose, manual builds)
+- ✅ Environment configuration and LLM integration
+- ✅ Using the interactive playground for testing
+- ✅ Making API requests with proper typing (SDK and REST)
+- ✅ Specialized endpoints (screenshots, PDFs, JavaScript execution)
+- ✅ MCP integration for AI-assisted development
+- ✅ **Real-time monitoring dashboard** for operational transparency
+- ✅ **Monitor API** for programmatic control and integration
+- ✅ Production deployment best practices
+
+**Why This Matters:**
+
+By self-hosting Crawl4AI, you:
+- 🔒 **Own Your Data**: Everything stays in your infrastructure
+- 📊 **See Everything**: Real-time dashboard shows exactly what's happening
+- 💰 **Control Costs**: Scale within your resources, no per-request fees
+- ⚡ **Maximize Performance**: Direct access with smart browser pooling (10x memory efficiency)
+- 🛡️ **Stay Secure**: Keep sensitive workflows behind your firewall
+- 🔧 **Customize Freely**: Full control over configs, strategies, and optimizations
+
+**Next Steps:**
+
+1. **Start Simple**: Deploy with Docker Hub image and test with the playground
+2. **Monitor Everything**: Open `http://localhost:11235/monitor` to watch your server
+3. **Integrate**: Connect your applications using the Python SDK or REST API
+4. **Scale Smart**: Use the monitoring data to optimize your deployment
+5. **Go Production**: Set up alerting, log aggregation, and automated cleanup
+
+**Key Resources:**
+- 🎮 **Playground**: `http://localhost:11235/playground` - Interactive testing
+- 📊 **Monitor Dashboard**: `http://localhost:11235/monitor` - Real-time visibility
+- 📖 **Architecture Docs**: `deploy/docker/ARCHITECTURE.md` - Deep technical dive
+- 💬 **Discord Community**: Get help and share experiences
+- ⭐ **GitHub**: Report issues, contribute, show support
+
+Remember: The monitoring dashboard is your window into your infrastructure. Use it to understand performance, troubleshoot issues, and optimize your deployment. The examples in the `examples` folder show real-world usage patterns you can adapt.
+
+**You're now in control of your web crawling destiny!** 🚀
+
+Happy crawling! 🕷️
diff --git a/docs/md_v2/core/simple-crawling.md b/docs/md_v2/core/simple-crawling.md
new file mode 100644
index 0000000..dd31bc9
--- /dev/null
+++ b/docs/md_v2/core/simple-crawling.md
@@ -0,0 +1,152 @@
+# Simple Crawling
+
+This guide covers the basics of web crawling with Crawl4AI. You'll learn how to set up a crawler, make your first request, and understand the response.
+
+## Basic Usage
+
+Set up a simple crawl using `BrowserConfig` and `CrawlerRunConfig`:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+async def main():
+    browser_config = BrowserConfig()  # Default browser configuration
+    run_config = CrawlerRunConfig()   # Default crawl run configuration
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=run_config
+        )
+        print(result.markdown)  # Print clean markdown content
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Understanding the Response
+
+The `arun()` method returns a `CrawlResult` object with several useful properties. Here's a quick overview (see [CrawlResult](../api/crawl-result.md) for complete details):
+
+```python
+config = CrawlerRunConfig(
+    markdown_generator=DefaultMarkdownGenerator(
+        content_filter=PruningContentFilter(threshold=0.6),
+        options={"ignore_links": True}
+    )
+)
+
+result = await crawler.arun(
+    url="https://example.com",
+    config=config
+)
+
+# Different content formats
+print(result.html)         # Raw HTML
+print(result.cleaned_html) # Cleaned HTML
+print(result.markdown.raw_markdown) # Raw markdown from cleaned html
+print(result.markdown.fit_markdown) # Most relevant content in markdown
+
+# Check success status
+print(result.success)      # True if crawl succeeded
+print(result.status_code)  # HTTP status code (e.g., 200, 404)
+
+# Access extracted media and links
+print(result.media)        # Dictionary of found media (images, videos, audio)
+print(result.links)        # Dictionary of internal and external links
+```
+
+## Adding Basic Options
+
+Customize your crawl using `CrawlerRunConfig`:
+
+```python
+run_config = CrawlerRunConfig(
+    word_count_threshold=10,        # Minimum words per content block
+    exclude_external_links=True,    # Remove external links
+    remove_overlay_elements=True,   # Remove popups/modals
+    process_iframes=True           # Process iframe content
+)
+
+result = await crawler.arun(
+    url="https://example.com",
+    config=run_config
+)
+```
+
+## Handling Errors
+
+Always check if the crawl was successful:
+
+```python
+run_config = CrawlerRunConfig()
+result = await crawler.arun(url="https://example.com", config=run_config)
+
+if not result.success:
+    print(f"Crawl failed: {result.error_message}")
+    print(f"Status code: {result.status_code}")
+```
+
+## Logging and Debugging
+
+Enable verbose logging in `BrowserConfig`:
+
+```python
+browser_config = BrowserConfig(verbose=True)
+
+async with AsyncWebCrawler(config=browser_config) as crawler:
+    run_config = CrawlerRunConfig()
+    result = await crawler.arun(url="https://example.com", config=run_config)
+```
+
+## Complete Example
+
+Here's a more comprehensive example demonstrating common usage patterns:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def main():
+    browser_config = BrowserConfig(verbose=True)
+    run_config = CrawlerRunConfig(
+        # Content filtering
+        word_count_threshold=10,
+        excluded_tags=['form', 'header'],
+        exclude_external_links=True,
+        
+        # Content processing
+        process_iframes=True,
+        remove_overlay_elements=True,
+        
+        # Cache control
+        cache_mode=CacheMode.ENABLED  # Use cache if available
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=run_config
+        )
+        
+        if result.success:
+            # Print clean content
+            print("Content:", result.markdown[:500])  # First 500 chars
+            
+            # Process images
+            for image in result.media["images"]:
+                print(f"Found image: {image['src']}")
+            
+            # Process links
+            for link in result.links["internal"]:
+                print(f"Internal link: {link['href']}")
+                
+        else:
+            print(f"Crawl failed: {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
diff --git a/docs/md_v2/core/table_extraction.md b/docs/md_v2/core/table_extraction.md
new file mode 100644
index 0000000..cdf9a71
--- /dev/null
+++ b/docs/md_v2/core/table_extraction.md
@@ -0,0 +1,807 @@
+# Table Extraction Strategies
+
+## Overview
+
+**New in v0.7.3+**: Table extraction now follows the **Strategy Design Pattern**, providing unprecedented flexibility and power for handling different table structures. Don't worry - **your existing code still works!** We maintain full backward compatibility while offering new capabilities.
+
+### What's Changed?
+- **Architecture**: Table extraction now uses pluggable strategies
+- **Backward Compatible**: Your existing code with `table_score_threshold` continues to work
+- **More Power**: Choose from multiple strategies or create your own
+- **Same Default Behavior**: By default, uses `DefaultTableExtraction` (same as before)
+
+### Key Points
+✅ **Old code still works** - No breaking changes  
+✅ **Same default behavior** - Uses the proven extraction algorithm  
+✅ **New capabilities** - Add LLM extraction or custom strategies when needed  
+✅ **Strategy pattern** - Clean, extensible architecture
+
+## Quick Start
+
+### The Simplest Way (Works Like Before)
+
+If you're already using Crawl4AI, nothing changes:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def extract_tables():
+    async with AsyncWebCrawler() as crawler:
+        # This works exactly like before - uses DefaultTableExtraction internally
+        result = await crawler.arun("https://example.com/data")
+        
+        # Tables are automatically extracted and available in result.tables
+        for table in result.tables:
+            print(f"Table with {len(table['rows'])} rows and {len(table['headers'])} columns")
+            print(f"Headers: {table['headers']}")
+            print(f"First row: {table['rows'][0] if table['rows'] else 'No data'}")
+
+asyncio.run(extract_tables())
+```
+
+### Using the Old Configuration (Still Supported)
+
+Your existing code with `table_score_threshold` continues to work:
+
+```python
+# This old approach STILL WORKS - we maintain backward compatibility
+config = CrawlerRunConfig(
+    table_score_threshold=7  # Internally creates DefaultTableExtraction(table_score_threshold=7)
+)
+result = await crawler.arun(url, config)
+```
+
+## Table Extraction Strategies
+
+### Understanding the Strategy Pattern
+
+The strategy pattern allows you to choose different table extraction algorithms at runtime. Think of it as having different tools in a toolbox - you pick the right one for the job:
+
+- **No explicit strategy?** → Uses `DefaultTableExtraction` automatically (same as v0.7.2 and earlier)
+- **Need complex table handling?** → Choose `LLMTableExtraction` (costs money, use sparingly)
+- **Want to disable tables?** → Use `NoTableExtraction`
+- **Have special requirements?** → Create a custom strategy
+
+### Available Strategies
+
+| Strategy | Description | Use Case | Cost | When to Use |
+|----------|-------------|----------|------|-------------|
+| `DefaultTableExtraction` | **RECOMMENDED**: Same algorithm as before v0.7.3 | General purpose (default) | Free | **Use this first - handles 95% of cases** |
+| `LLMTableExtraction` | AI-powered extraction for complex tables | Tables with complex rowspan/colspan | **$$$ Per API call** | Only when DefaultTableExtraction fails |
+| `NoTableExtraction` | Disables table extraction | When tables aren't needed | Free | For text-only extraction |
+| Custom strategies | User-defined extraction logic | Specialized requirements | Free | Domain-specific needs |
+
+> **⚠️ CRITICAL COST WARNING for LLMTableExtraction**: 
+> 
+> **DO NOT USE `LLMTableExtraction` UNLESS ABSOLUTELY NECESSARY!**
+> 
+> - **Always try `DefaultTableExtraction` first** - It's free and handles most tables perfectly
+> - LLM extraction **costs money** with every API call
+> - For large tables (100+ rows), LLM extraction can be **very slow**
+> - **For large tables**: If you must use LLM, choose fast providers:
+>   - ✅ **Groq** (fastest inference)
+>   - ✅ **Cerebras** (optimized for speed)
+>   - ⚠️ Avoid: OpenAI, Anthropic for large tables (slower)
+> 
+> **🚧 WORK IN PROGRESS**: 
+> We are actively developing an **advanced non-LLM algorithm** that will handle complex table structures (rowspan, colspan, nested tables) for **FREE**. This will replace the need for costly LLM extraction in most cases. Coming soon!
+
+### DefaultTableExtraction
+
+The default strategy uses a sophisticated scoring system to identify data tables:
+
+```python
+from crawl4ai import DefaultTableExtraction, CrawlerRunConfig
+
+# Customize the default extraction
+table_strategy = DefaultTableExtraction(
+    table_score_threshold=7,  # Scoring threshold (default: 7)
+    min_rows=2,               # Minimum rows required
+    min_cols=2,               # Minimum columns required
+    verbose=True              # Enable detailed logging
+)
+
+config = CrawlerRunConfig(
+    table_extraction=table_strategy
+)
+```
+
+#### Scoring System
+
+The scoring system evaluates multiple factors:
+
+| Factor | Score Impact | Description |
+|--------|--------------|-------------|
+| Has `` | +2 | Semantic table structure |
+| Has `` | +1 | Organized table body |
+| Has `` elements | +2 | Header cells present |
+| Headers in correct position | +1 | Proper semantic structure |
+| Consistent column count | +2 | Regular data structure |
+| Has caption | +2 | Descriptive caption |
+| Has summary | +1 | Summary attribute |
+| High text density | +2 to +3 | Content-rich cells |
+| Data attributes | +0.5 each | Data-* attributes |
+| Nested tables | -3 | Often indicates layout |
+| Role="presentation" | -3 | Explicitly non-data |
+| Too few rows | -2 | Insufficient data |
+
+### LLMTableExtraction (Use Sparingly!)
+
+**⚠️ WARNING**: Only use this when `DefaultTableExtraction` fails with complex tables!
+
+LLMTableExtraction uses AI to understand complex table structures that traditional parsers struggle with. It automatically handles large tables through intelligent chunking and parallel processing:
+
+```python
+from crawl4ai import LLMTableExtraction, LLMConfig, CrawlerRunConfig
+
+# Configure LLM (costs money per call!)
+llm_config = LLMConfig(
+    provider="groq/llama-3.3-70b-versatile",  # Fast provider for large tables
+    api_token="your_api_key",
+    temperature=0.1
+)
+
+# Create LLM extraction strategy with smart chunking
+table_strategy = LLMTableExtraction(
+    llm_config=llm_config,
+    max_tries=3,                      # Retry up to 3 times if extraction fails
+    css_selector="table",             # Optional: focus on specific tables
+    enable_chunking=True,             # Automatically chunk large tables (default: True)
+    chunk_token_threshold=3000,       # Split tables larger than this (default: 3000 tokens)
+    min_rows_per_chunk=10,            # Minimum rows per chunk (default: 10)
+    max_parallel_chunks=5,            # Process up to 5 chunks in parallel (default: 5)
+    verbose=True
+)
+
+config = CrawlerRunConfig(
+    table_extraction=table_strategy
+)
+
+result = await crawler.arun(url, config)
+```
+
+#### When to Use LLMTableExtraction
+
+✅ **Use ONLY when**:
+- Tables have complex merged cells (rowspan/colspan) that break DefaultTableExtraction
+- Nested tables that need semantic understanding
+- Tables with irregular structures
+- You've tried DefaultTableExtraction and it failed
+
+❌ **Never use when**:
+- DefaultTableExtraction works (99% of cases)
+- Tables are simple or well-structured
+- You're processing many pages (costs add up!)
+- Tables have 100+ rows (very slow)
+
+#### How Smart Chunking Works
+
+LLMTableExtraction automatically handles large tables through intelligent chunking:
+
+1. **Automatic Detection**: Tables exceeding the token threshold are automatically split
+2. **Smart Splitting**: Chunks are created at row boundaries, preserving table structure
+3. **Header Preservation**: Each chunk includes the original headers for context
+4. **Parallel Processing**: Multiple chunks are processed simultaneously for speed
+5. **Intelligent Merging**: Results are merged back into a single, complete table
+
+**Chunking Parameters**:
+- `enable_chunking` (default: `True`): Automatically handle large tables
+- `chunk_token_threshold` (default: `3000`): When to split tables
+- `min_rows_per_chunk` (default: `10`): Ensures meaningful chunk sizes
+- `max_parallel_chunks` (default: `5`): Concurrent processing for speed
+
+The chunking is completely transparent - you get the same output format whether the table was processed in one piece or multiple chunks.
+
+#### Performance Optimization for LLMTableExtraction
+
+**Provider Recommendations by Table Size**:
+
+| Table Size | Recommended Providers | Why |
+|------------|----------------------|-----|
+| Small (<50 rows) | Any provider | Fast enough |
+| Medium (50-200 rows) | Groq, Cerebras | Optimized inference |
+| Large (200+ rows) | **Groq** (best), Cerebras | Fastest inference + automatic chunking |
+| Very Large (500+ rows) | Groq with chunking | Parallel processing keeps it fast |
+
+### NoTableExtraction
+
+Disable table extraction for better performance when tables aren't needed:
+
+```python
+from crawl4ai import NoTableExtraction, CrawlerRunConfig
+
+config = CrawlerRunConfig(
+    table_extraction=NoTableExtraction()
+)
+
+# Tables won't be extracted, improving performance
+result = await crawler.arun(url, config)
+assert len(result.tables) == 0
+```
+
+## Extracted Table Structure
+
+Each extracted table contains:
+
+```python
+{
+    "headers": ["Column 1", "Column 2", ...],  # Column headers
+    "rows": [                                   # Data rows
+        ["Row 1 Col 1", "Row 1 Col 2", ...],
+        ["Row 2 Col 1", "Row 2 Col 2", ...],
+    ],
+    "caption": "Table Caption",                # If present
+    "summary": "Table Summary",                # If present
+    "metadata": {
+        "row_count": 10,                       # Number of rows
+        "column_count": 3,                      # Number of columns
+        "has_headers": True,                    # Headers detected
+        "has_caption": True,                    # Caption exists
+        "has_summary": False,                   # Summary exists
+        "id": "data-table-1",                   # Table ID if present
+        "class": "financial-data"               # Table class if present
+    }
+}
+```
+
+## Configuration Options
+
+### Basic Configuration
+
+```python
+config = CrawlerRunConfig(
+    # Table extraction settings
+    table_score_threshold=7,      # Default threshold (backward compatible)
+    table_extraction=strategy,     # Optional: custom strategy
+    
+    # Filter what to process
+    css_selector="main",          # Focus on specific area
+    excluded_tags=["nav", "aside"] # Exclude page sections
+)
+```
+
+### Advanced Configuration
+
+```python
+from crawl4ai import DefaultTableExtraction, CrawlerRunConfig
+
+# Fine-tuned extraction
+strategy = DefaultTableExtraction(
+    table_score_threshold=5,      # Lower = more permissive
+    min_rows=3,                   # Require at least 3 rows
+    min_cols=2,                   # Require at least 2 columns
+    verbose=True                  # Detailed logging
+)
+
+config = CrawlerRunConfig(
+    table_extraction=strategy,
+    css_selector="article.content", # Target specific content
+    exclude_domains=["ads.com"],   # Exclude ad domains
+    cache_mode=CacheMode.BYPASS    # Fresh extraction
+)
+```
+
+## Working with Extracted Tables
+
+### Convert to Pandas DataFrame
+
+```python
+import pandas as pd
+
+async def tables_to_dataframes(url):
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url)
+        
+        dataframes = []
+        for table_data in result.tables:
+            # Create DataFrame
+            if table_data['headers']:
+                df = pd.DataFrame(
+                    table_data['rows'],
+                    columns=table_data['headers']
+                )
+            else:
+                df = pd.DataFrame(table_data['rows'])
+            
+            # Add metadata as DataFrame attributes
+            df.attrs['caption'] = table_data.get('caption', '')
+            df.attrs['metadata'] = table_data.get('metadata', {})
+            
+            dataframes.append(df)
+        
+        return dataframes
+```
+
+### Filter Tables by Criteria
+
+```python
+async def extract_large_tables(url):
+    async with AsyncWebCrawler() as crawler:
+        # Configure minimum size requirements
+        strategy = DefaultTableExtraction(
+            min_rows=10,
+            min_cols=3,
+            table_score_threshold=6
+        )
+        
+        config = CrawlerRunConfig(
+            table_extraction=strategy
+        )
+        
+        result = await crawler.arun(url, config)
+        
+        # Further filter results
+        large_tables = [
+            table for table in result.tables
+            if table['metadata']['row_count'] > 10
+            and table['metadata']['column_count'] > 3
+        ]
+        
+        return large_tables
+```
+
+### Export Tables to Different Formats
+
+```python
+import json
+import csv
+
+async def export_tables(url):
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url)
+        
+        for i, table in enumerate(result.tables):
+            # Export as JSON
+            with open(f'table_{i}.json', 'w') as f:
+                json.dump(table, f, indent=2)
+            
+            # Export as CSV
+            with open(f'table_{i}.csv', 'w', newline='') as f:
+                writer = csv.writer(f)
+                if table['headers']:
+                    writer.writerow(table['headers'])
+                writer.writerows(table['rows'])
+            
+            # Export as Markdown
+            with open(f'table_{i}.md', 'w') as f:
+                # Write headers
+                if table['headers']:
+                    f.write('| ' + ' | '.join(table['headers']) + ' |\n')
+                    f.write('|' + '---|' * len(table['headers']) + '\n')
+                
+                # Write rows
+                for row in table['rows']:
+                    f.write('| ' + ' | '.join(str(cell) for cell in row) + ' |\n')
+```
+
+## Creating Custom Strategies
+
+Extend `TableExtractionStrategy` to create custom extraction logic:
+
+### Example: Financial Table Extractor
+
+```python
+from crawl4ai import TableExtractionStrategy
+from typing import List, Dict, Any
+import re
+
+class FinancialTableExtractor(TableExtractionStrategy):
+    """Extract tables containing financial data."""
+    
+    def __init__(self, currency_symbols=None, require_numbers=True, **kwargs):
+        super().__init__(**kwargs)
+        self.currency_symbols = currency_symbols or ['$', '€', '£', '¥']
+        self.require_numbers = require_numbers
+        self.number_pattern = re.compile(r'\d+[,.]?\d*')
+    
+    def extract_tables(self, element, **kwargs):
+        tables_data = []
+        
+        for table in element.xpath(".//table"):
+            # Check if table contains financial indicators
+            table_text = ''.join(table.itertext())
+            
+            # Must contain currency symbols
+            has_currency = any(sym in table_text for sym in self.currency_symbols)
+            if not has_currency:
+                continue
+            
+            # Must contain numbers if required
+            if self.require_numbers:
+                numbers = self.number_pattern.findall(table_text)
+                if len(numbers) < 3:  # Arbitrary minimum
+                    continue
+            
+            # Extract the table data
+            table_data = self._extract_financial_data(table)
+            if table_data:
+                tables_data.append(table_data)
+        
+        return tables_data
+    
+    def _extract_financial_data(self, table):
+        """Extract and clean financial data from table."""
+        headers = []
+        rows = []
+        
+        # Extract headers
+        for th in table.xpath(".//thead//th | .//tr[1]//th"):
+            headers.append(th.text_content().strip())
+        
+        # Extract and clean rows
+        for tr in table.xpath(".//tbody//tr | .//tr[position()>1]"):
+            row = []
+            for td in tr.xpath(".//td"):
+                text = td.text_content().strip()
+                # Clean currency formatting
+                text = re.sub(r'[$€£¥,]', '', text)
+                row.append(text)
+            if row:
+                rows.append(row)
+        
+        return {
+            "headers": headers,
+            "rows": rows,
+            "caption": self._get_caption(table),
+            "summary": table.get("summary", ""),
+            "metadata": {
+                "type": "financial",
+                "row_count": len(rows),
+                "column_count": len(headers) or len(rows[0]) if rows else 0
+            }
+        }
+    
+    def _get_caption(self, table):
+        caption = table.xpath(".//caption/text()")
+        return caption[0].strip() if caption else ""
+
+# Usage
+strategy = FinancialTableExtractor(
+    currency_symbols=['$', 'EUR'],
+    require_numbers=True
+)
+
+config = CrawlerRunConfig(
+    table_extraction=strategy
+)
+```
+
+### Example: Specific Table Extractor
+
+```python
+class SpecificTableExtractor(TableExtractionStrategy):
+    """Extract only tables matching specific criteria."""
+    
+    def __init__(self, 
+                 required_headers=None, 
+                 id_pattern=None,
+                 class_pattern=None,
+                 **kwargs):
+        super().__init__(**kwargs)
+        self.required_headers = required_headers or []
+        self.id_pattern = id_pattern
+        self.class_pattern = class_pattern
+    
+    def extract_tables(self, element, **kwargs):
+        tables_data = []
+        
+        for table in element.xpath(".//table"):
+            # Check ID pattern
+            if self.id_pattern:
+                table_id = table.get('id', '')
+                if not re.match(self.id_pattern, table_id):
+                    continue
+            
+            # Check class pattern
+            if self.class_pattern:
+                table_class = table.get('class', '')
+                if not re.match(self.class_pattern, table_class):
+                    continue
+            
+            # Extract headers to check requirements
+            headers = self._extract_headers(table)
+            
+            # Check if required headers are present
+            if self.required_headers:
+                if not all(req in headers for req in self.required_headers):
+                    continue
+            
+            # Extract full table data
+            table_data = self._extract_table_data(table, headers)
+            tables_data.append(table_data)
+        
+        return tables_data
+```
+
+## Combining with Other Strategies
+
+Table extraction works seamlessly with other Crawl4AI strategies:
+
+```python
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    DefaultTableExtraction,
+    LLMExtractionStrategy,
+    JsonCssExtractionStrategy
+)
+
+async def combined_extraction(url):
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            # Table extraction
+            table_extraction=DefaultTableExtraction(
+                table_score_threshold=6,
+                min_rows=2
+            ),
+            
+            # CSS-based extraction for specific elements
+            extraction_strategy=JsonCssExtractionStrategy({
+                "title": "h1",
+                "summary": "p.summary",
+                "date": "time"
+            }),
+            
+            # Focus on main content
+            css_selector="main.content"
+        )
+        
+        result = await crawler.arun(url, config)
+        
+        # Access different extraction results
+        tables = result.tables  # Table data
+        structured = json.loads(result.extracted_content)  # CSS extraction
+        
+        return {
+            "tables": tables,
+            "structured_data": structured,
+            "markdown": result.markdown
+        }
+```
+
+## Performance Considerations
+
+### Optimization Tips
+
+1. **Disable when not needed**: Use `NoTableExtraction` if tables aren't required
+2. **Target specific areas**: Use `css_selector` to limit processing scope
+3. **Set minimum thresholds**: Filter out small/irrelevant tables early
+4. **Cache results**: Use appropriate cache modes for repeated extractions
+
+```python
+# Optimized configuration for large pages
+config = CrawlerRunConfig(
+    # Only process main content area
+    css_selector="article.main-content",
+    
+    # Exclude navigation and sidebars
+    excluded_tags=["nav", "aside", "footer"],
+    
+    # Higher threshold for stricter filtering
+    table_extraction=DefaultTableExtraction(
+        table_score_threshold=8,
+        min_rows=5,
+        min_cols=3
+    ),
+    
+    # Enable caching for repeated access
+    cache_mode=CacheMode.ENABLED
+)
+```
+
+## Migration Guide
+
+### Important: Your Code Still Works!
+
+**No changes required!** The transition to the strategy pattern is **fully backward compatible**.
+
+### How It Works Internally
+
+#### v0.7.2 and Earlier
+```python
+# Old way - directly passing table_score_threshold
+config = CrawlerRunConfig(
+    table_score_threshold=7
+)
+# Internally: No strategy pattern, direct implementation
+```
+
+#### v0.7.3+ (Current)
+```python
+# Old way STILL WORKS - we handle it internally
+config = CrawlerRunConfig(
+    table_score_threshold=7
+)
+# Internally: Automatically creates DefaultTableExtraction(table_score_threshold=7)
+```
+
+### Taking Advantage of New Features
+
+While your old code works, you can now use the strategy pattern for more control:
+
+```python
+# Option 1: Keep using the old way (perfectly fine!)
+config = CrawlerRunConfig(
+    table_score_threshold=7  # Still supported
+)
+
+# Option 2: Use the new strategy pattern (more flexibility)
+from crawl4ai import DefaultTableExtraction
+
+strategy = DefaultTableExtraction(
+    table_score_threshold=7,
+    min_rows=2,  # New capability!
+    min_cols=2   # New capability!
+)
+
+config = CrawlerRunConfig(
+    table_extraction=strategy
+)
+
+# Option 3: Use advanced strategies when needed
+from crawl4ai import LLMTableExtraction, LLMConfig
+
+# Only for complex tables that DefaultTableExtraction can't handle
+# Automatically handles large tables with smart chunking
+llm_strategy = LLMTableExtraction(
+    llm_config=LLMConfig(
+        provider="groq/llama-3.3-70b-versatile",
+        api_token="your_key"
+    ),
+    max_tries=3,
+    enable_chunking=True,  # Automatically chunk large tables
+    chunk_token_threshold=3000,  # Chunk when exceeding 3000 tokens
+    max_parallel_chunks=5  # Process up to 5 chunks in parallel
+)
+
+config = CrawlerRunConfig(
+    table_extraction=llm_strategy  # Advanced extraction with automatic chunking
+)
+```
+
+### Summary
+
+- ✅ **No breaking changes** - Old code works as-is
+- ✅ **Same defaults** - DefaultTableExtraction is automatically used
+- ✅ **Gradual adoption** - Use new features when you need them
+- ✅ **Full compatibility** - result.tables structure unchanged
+
+## Best Practices
+
+### 1. Choose the Right Strategy (Cost-Conscious Approach)
+
+**Decision Flow**:
+```
+1. Do you need tables? 
+   → No: Use NoTableExtraction
+   → Yes: Continue to #2
+
+2. Try DefaultTableExtraction first (FREE)
+   → Works? Done! ✅
+   → Fails? Continue to #3
+
+3. Is the table critical and complex?
+   → No: Accept DefaultTableExtraction results
+   → Yes: Continue to #4
+
+4. Use LLMTableExtraction (COSTS MONEY)
+   → Small table (<50 rows): Any LLM provider
+   → Large table (50+ rows): Use Groq or Cerebras
+   → Very large (500+ rows): Reconsider - maybe chunk the page
+```
+
+**Strategy Selection Guide**:
+- **DefaultTableExtraction**: Use for 99% of cases - it's free and effective
+- **LLMTableExtraction**: Only for complex tables with merged cells that break DefaultTableExtraction
+- **NoTableExtraction**: When you only need text/markdown content
+- **Custom Strategy**: For specialized requirements (financial, scientific, etc.)
+
+### 2. Validate Extracted Data
+
+```python
+def validate_table(table):
+    """Validate table data quality."""
+    # Check structure
+    if not table.get('rows'):
+        return False
+    
+    # Check consistency
+    if table.get('headers'):
+        expected_cols = len(table['headers'])
+        for row in table['rows']:
+            if len(row) != expected_cols:
+                return False
+    
+    # Check minimum content
+    total_cells = sum(len(row) for row in table['rows'])
+    non_empty = sum(1 for row in table['rows'] 
+                    for cell in row if cell.strip())
+    
+    if non_empty / total_cells < 0.5:  # Less than 50% non-empty
+        return False
+    
+    return True
+
+# Filter valid tables
+valid_tables = [t for t in result.tables if validate_table(t)]
+```
+
+### 3. Handle Edge Cases
+
+```python
+async def robust_table_extraction(url):
+    """Extract tables with error handling."""
+    async with AsyncWebCrawler() as crawler:
+        try:
+            config = CrawlerRunConfig(
+                table_extraction=DefaultTableExtraction(
+                    table_score_threshold=6,
+                    verbose=True
+                )
+            )
+            
+            result = await crawler.arun(url, config)
+            
+            if not result.success:
+                print(f"Crawl failed: {result.error}")
+                return []
+            
+            # Process tables safely
+            processed_tables = []
+            for table in result.tables:
+                try:
+                    # Validate and process
+                    if validate_table(table):
+                        processed_tables.append(table)
+                except Exception as e:
+                    print(f"Error processing table: {e}")
+                    continue
+            
+            return processed_tables
+            
+        except Exception as e:
+            print(f"Extraction error: {e}")
+            return []
+```
+
+## Troubleshooting
+
+### Common Issues and Solutions
+
+| Issue | Cause | Solution |
+|-------|-------|----------|
+| No tables extracted | Score too high | Lower `table_score_threshold` |
+| Layout tables included | Score too low | Increase `table_score_threshold` |
+| Missing tables | CSS selector too specific | Broaden or remove `css_selector` |
+| Incomplete data | Complex table structure | Create custom strategy |
+| Performance issues | Processing entire page | Use `css_selector` to limit scope |
+
+### Debug Logging
+
+Enable verbose logging to understand extraction decisions:
+
+```python
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Enable verbose mode in strategy
+strategy = DefaultTableExtraction(
+    table_score_threshold=7,
+    verbose=True  # Detailed extraction logs
+)
+
+config = CrawlerRunConfig(
+    table_extraction=strategy,
+    verbose=True  # General crawler logs
+)
+```
+
+## See Also
+
+- [Extraction Strategies](extraction-strategies.md) - Overview of all extraction strategies
+- [Content Selection](content-selection.md) - Using CSS selectors and filters
+- [Performance Optimization](../optimization/performance-tuning.md) - Speed up extraction
+- [Examples](../examples/table_extraction_example.py) - Complete working examples
\ No newline at end of file
diff --git a/docs/md_v2/core/url-seeding.md b/docs/md_v2/core/url-seeding.md
new file mode 100644
index 0000000..7d12823
--- /dev/null
+++ b/docs/md_v2/core/url-seeding.md
@@ -0,0 +1,1173 @@
+# URL Seeding: The Smart Way to Crawl at Scale
+
+## Why URL Seeding?
+
+Web crawling comes in different flavors, each with its own strengths. Let's understand when to use URL seeding versus deep crawling.
+
+### Deep Crawling: Real-Time Discovery
+
+Deep crawling is perfect when you need:
+- **Fresh, real-time data** - discovering pages as they're created
+- **Dynamic exploration** - following links based on content
+- **Selective extraction** - stopping when you find what you need
+
+```python
+# Deep crawling example: Explore a website dynamically
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+async def deep_crawl_example():
+    # Configure a 2-level deep crawl
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=BFSDeepCrawlStrategy(
+            max_depth=2,           # Crawl 2 levels deep
+            include_external=False, # Stay within domain
+            max_pages=50           # Limit for efficiency
+        ),
+        verbose=True
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        # Start crawling and follow links dynamically
+        results = await crawler.arun("https://example.com", config=config)
+        
+        print(f"Discovered and crawled {len(results)} pages")
+        for result in results[:3]:
+            print(f"Found: {result.url} at depth {result.metadata.get('depth', 0)}")
+
+asyncio.run(deep_crawl_example())
+```
+
+### URL Seeding: Bulk Discovery
+
+URL seeding shines when you want:
+- **Comprehensive coverage** - get thousands of URLs in seconds
+- **Bulk processing** - filter before crawling
+- **Resource efficiency** - know exactly what you'll crawl
+
+```python
+# URL seeding example: Analyze all documentation
+from crawl4ai import AsyncUrlSeeder, SeedingConfig
+
+seeder = AsyncUrlSeeder()
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=True,
+    pattern="*/docs/*"
+)
+
+# Get ALL documentation URLs instantly
+urls = await seeder.urls("example.com", config)
+# 1000+ URLs discovered in seconds!
+```
+
+### The Trade-offs
+
+| Aspect | Deep Crawling | URL Seeding |
+|--------|---------------|-------------|
+| **Coverage** | Discovers pages dynamically | Gets most existing URLs instantly |
+| **Freshness** | Finds brand new pages | May miss very recent pages |
+| **Speed** | Slower, page by page | Extremely fast bulk discovery |
+| **Resource Usage** | Higher - crawls to discover | Lower - discovers then crawls |
+| **Control** | Can stop mid-process | Pre-filters before crawling |
+
+### When to Use Each
+
+**Choose Deep Crawling when:**
+- You need the absolute latest content
+- You're searching for specific information
+- The site structure is unknown or dynamic
+- You want to stop as soon as you find what you need
+
+**Choose URL Seeding when:**
+- You need to analyze large portions of a site
+- You want to filter URLs before crawling
+- You're doing comparative analysis
+- You need to optimize resource usage
+
+The magic happens when you understand both approaches and choose the right tool for your task. Sometimes, you might even combine them - use URL seeding for bulk discovery, then deep crawl specific sections for the latest updates.
+
+## Your First URL Seeding Adventure
+
+Let's see the magic in action. We'll discover blog posts about Python, filter for tutorials, and crawl only those pages.
+
+```python
+import asyncio
+from crawl4ai import AsyncUrlSeeder, AsyncWebCrawler, SeedingConfig, CrawlerRunConfig
+
+async def smart_blog_crawler():
+    # Step 1: Create our URL discoverer
+    seeder = AsyncUrlSeeder()
+    
+    # Step 2: Configure discovery - let's find all blog posts
+    config = SeedingConfig(
+        source="sitemap+cc",      # Use the website's sitemap+cc
+        pattern="*/courses/*",    # Only courses related posts
+        extract_head=True,          # Get page metadata
+        max_urls=100               # Limit for this example
+    )
+    
+    # Step 3: Discover URLs from the Python blog
+    print("🔍 Discovering course posts...")
+    urls = await seeder.urls("realpython.com", config)
+    print(f"✅ Found {len(urls)} course posts")
+    
+    # Step 4: Filter for Python tutorials (using metadata!)
+    tutorials = [
+        url for url in urls 
+        if url["status"] == "valid" and 
+        any(keyword in str(url["head_data"]).lower() 
+            for keyword in ["tutorial", "guide", "how to"])
+    ]
+    print(f"📚 Filtered to {len(tutorials)} tutorials")
+    
+    # Step 5: Show what we found
+    print("\n🎯 Found these tutorials:")
+    for tutorial in tutorials[:5]:  # First 5
+        title = tutorial["head_data"].get("title", "No title")
+        print(f"  - {title}")
+        print(f"    {tutorial['url']}")
+    
+    # Step 6: Now crawl ONLY these relevant pages
+    print("\n🚀 Crawling tutorials...")
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            only_text=True,
+            word_count_threshold=300,  # Only substantial articles
+            stream=True
+        )
+        
+        # Extract URLs and crawl them
+        tutorial_urls = [t["url"] for t in tutorials[:10]]
+        results = await crawler.arun_many(tutorial_urls, config=config)
+        
+        successful = 0
+        async for result in results:
+            if result.success:
+                successful += 1
+                print(f"✓ Crawled: {result.url[:60]}...")
+        
+        print(f"\n✨ Successfully crawled {successful} tutorials!")
+
+# Run it!
+asyncio.run(smart_blog_crawler())
+```
+
+**What just happened?**
+
+1. We discovered all blog URLs from the sitemap+cc
+2. We filtered using metadata (no crawling needed!)
+3. We crawled only the relevant tutorials
+4. We saved tons of time and bandwidth
+
+This is the power of URL seeding - you see everything before you crawl anything.
+
+## Understanding the URL Seeder
+
+Now that you've seen the magic, let's understand how it works.
+
+### Basic Usage
+
+Creating a URL seeder is simple:
+
+```python
+from crawl4ai import AsyncUrlSeeder
+
+# Method 1: Manual cleanup
+seeder = AsyncUrlSeeder()
+try:
+    config = SeedingConfig(source="sitemap")
+    urls = await seeder.urls("example.com", config)
+finally:
+    await seeder.close()
+
+# Method 2: Context manager (recommended)
+async with AsyncUrlSeeder() as seeder:
+    config = SeedingConfig(source="sitemap")
+    urls = await seeder.urls("example.com", config)
+    # Automatically cleaned up on exit
+```
+
+The seeder can discover URLs from two powerful sources:
+
+#### 1. Sitemaps (Fastest)
+
+```python
+# Discover from sitemap
+config = SeedingConfig(source="sitemap")
+urls = await seeder.urls("example.com", config)
+```
+
+Sitemaps are XML files that websites create specifically to list all their URLs. It's like getting a menu at a restaurant - everything is listed upfront.
+
+**Sitemap Index Support**: For large websites like TechCrunch that use sitemap indexes (a sitemap of sitemaps), the seeder automatically detects and processes all sub-sitemaps in parallel:
+
+```xml
+
+
+  
+    https://techcrunch.com/sitemap-1.xml
+  
+  
+    https://techcrunch.com/sitemap-2.xml
+  
+  
+
+```
+
+The seeder handles this transparently - you'll get all URLs from all sub-sitemaps automatically!
+
+#### 2. Common Crawl (Most Comprehensive)
+
+```python
+# Discover from Common Crawl
+config = SeedingConfig(source="cc")
+urls = await seeder.urls("example.com", config)
+```
+
+Common Crawl is a massive public dataset that regularly crawls the entire web. It's like having access to a pre-built index of the internet.
+
+#### 3. Both Sources (Maximum Coverage)
+
+```python
+# Use both sources
+config = SeedingConfig(source="sitemap+cc")
+urls = await seeder.urls("example.com", config)
+```
+
+### Configuration Magic: SeedingConfig
+
+The `SeedingConfig` object is your control panel. Here's everything you can configure:
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `source` | str | "sitemap+cc" | URL source: "cc" (Common Crawl), "sitemap", or "sitemap+cc" |
+| `pattern` | str | "*" | URL pattern filter (e.g., "*/blog/*", "*.html") |
+| `extract_head` | bool | False | Extract metadata from page `` |
+| `live_check` | bool | False | Verify URLs are accessible |
+| `max_urls` | int | -1 | Maximum URLs to return (-1 = unlimited) |
+| `concurrency` | int | 10 | Parallel workers for fetching |
+| `hits_per_sec` | int | 5 | Rate limit for requests |
+| `force` | bool | False | Bypass cache, fetch fresh data |
+| `verbose` | bool | False | Show detailed progress |
+| `query` | str | None | Search query for BM25 scoring |
+| `scoring_method` | str | None | Scoring method (currently "bm25") |
+| `score_threshold` | float | None | Minimum score to include URL |
+| `filter_nonsense_urls` | bool | True | Filter out utility URLs (robots.txt, etc.) |
+| `cache_ttl_hours` | int | 24 | Hours before sitemap cache expires (0 = no TTL) |
+| `validate_sitemap_lastmod` | bool | True | Check sitemap's lastmod and refetch if newer |
+
+#### Pattern Matching Examples
+
+```python
+# Match all blog posts
+config = SeedingConfig(pattern="*/blog/*")
+
+# Match only HTML files
+config = SeedingConfig(pattern="*.html")
+
+# Match product pages
+config = SeedingConfig(pattern="*/product/*")
+
+# Match everything except admin pages
+config = SeedingConfig(pattern="*")
+# Then filter: urls = [u for u in urls if "/admin/" not in u["url"]]
+```
+
+### URL Validation: Live Checking
+
+Sometimes you need to know if URLs are actually accessible. That's where live checking comes in:
+
+```python
+config = SeedingConfig(
+    source="sitemap",
+    live_check=True,  # Verify each URL is accessible
+    concurrency=20    # Check 20 URLs in parallel
+)
+async with AsyncUrlSeeder() as seeder:
+    urls = await seeder.urls("example.com", config)
+
+# Now you can filter by status
+live_urls = [u for u in urls if u["status"] == "valid"]
+dead_urls = [u for u in urls if u["status"] == "not_valid"]
+
+print(f"Live URLs: {len(live_urls)}")
+print(f"Dead URLs: {len(dead_urls)}")
+```
+
+**When to use live checking:**
+- Before a large crawling operation
+- When working with older sitemaps
+- When data freshness is critical
+
+**When to skip it:**
+- Quick explorations
+- When you trust the source
+- When speed is more important than accuracy
+
+### The Power of Metadata: Head Extraction
+
+This is where URL seeding gets really powerful. Instead of crawling entire pages, you can extract just the metadata:
+
+```python
+config = SeedingConfig(
+    extract_head=True  # Extract metadata from  section
+)
+async with AsyncUrlSeeder() as seeder:
+    urls = await seeder.urls("example.com", config)
+
+# Now each URL has rich metadata
+for url in urls[:3]:
+    print(f"\nURL: {url['url']}")
+    print(f"Title: {url['head_data'].get('title')}")
+    
+    meta = url['head_data'].get('meta', {})
+    print(f"Description: {meta.get('description')}")
+    print(f"Keywords: {meta.get('keywords')}")
+    
+    # Even Open Graph data!
+    print(f"OG Image: {meta.get('og:image')}")
+```
+
+#### What Can We Extract?
+
+The head extraction gives you a treasure trove of information:
+
+```python
+# Example of extracted head_data
+{
+    "title": "10 Python Tips for Beginners",
+    "charset": "utf-8",
+    "lang": "en",
+    "meta": {
+        "description": "Learn essential Python tips...",
+        "keywords": "python, programming, tutorial",
+        "author": "Jane Developer",
+        "viewport": "width=device-width, initial-scale=1",
+        
+        # Open Graph tags
+        "og:title": "10 Python Tips for Beginners",
+        "og:description": "Essential Python tips for new programmers",
+        "og:image": "https://example.com/python-tips.jpg",
+        "og:type": "article",
+        
+        # Twitter Card tags
+        "twitter:card": "summary_large_image",
+        "twitter:title": "10 Python Tips",
+        
+        # Dublin Core metadata
+        "dc.creator": "Jane Developer",
+        "dc.date": "2024-01-15"
+    },
+    "link": {
+        "canonical": [{"href": "https://example.com/blog/python-tips"}],
+        "alternate": [{"href": "/feed.xml", "type": "application/rss+xml"}]
+    },
+    "jsonld": [
+        {
+            "@type": "Article",
+            "headline": "10 Python Tips for Beginners",
+            "datePublished": "2024-01-15",
+            "author": {"@type": "Person", "name": "Jane Developer"}
+        }
+    ]
+}
+```
+
+This metadata is gold for filtering! You can find exactly what you need without crawling a single page.
+
+### Smart URL-Based Filtering (No Head Extraction)
+
+When `extract_head=False` but you still provide a query, the seeder uses intelligent URL-based scoring:
+
+```python
+# Fast filtering based on URL structure alone
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=False,  # Don't fetch page metadata
+    query="python tutorial async",
+    scoring_method="bm25",
+    score_threshold=0.3
+)
+async with AsyncUrlSeeder() as seeder:
+    urls = await seeder.urls("example.com", config)
+
+# URLs are scored based on:
+# 1. Domain parts matching (e.g., 'python' in python.example.com)
+# 2. Path segments (e.g., '/tutorials/python-async/')
+# 3. Query parameters (e.g., '?topic=python')
+# 4. Fuzzy matching using character n-grams
+
+# Example URL scoring:
+# https://example.com/tutorials/python/async-guide.html - High score
+# https://example.com/blog/javascript-tips.html - Low score
+```
+
+This approach is much faster than head extraction while still providing intelligent filtering!
+
+### Understanding Results
+
+Each URL in the results has this structure:
+
+```python
+{
+    "url": "https://example.com/blog/python-tips.html",
+    "status": "valid",        # "valid", "not_valid", or "unknown"
+    "head_data": {            # Only if extract_head=True
+        "title": "Page Title",
+        "meta": {...},
+        "link": {...},
+        "jsonld": [...]
+    },
+    "relevance_score": 0.85   # Only if using BM25 scoring
+}
+```
+
+Let's see a real example:
+
+```python
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=True,
+    live_check=True
+)
+async with AsyncUrlSeeder() as seeder:
+    urls = await seeder.urls("blog.example.com", config)
+
+# Analyze the results
+for url in urls[:5]:
+    print(f"\n{'='*60}")
+    print(f"URL: {url['url']}")
+    print(f"Status: {url['status']}")
+    
+    if url['head_data']:
+        data = url['head_data']
+        print(f"Title: {data.get('title', 'No title')}")
+        
+        # Check content type
+        meta = data.get('meta', {})
+        content_type = meta.get('og:type', 'unknown')
+        print(f"Content Type: {content_type}")
+        
+        # Publication date
+        pub_date = None
+        for jsonld in data.get('jsonld', []):
+            if isinstance(jsonld, dict):
+                pub_date = jsonld.get('datePublished')
+                if pub_date:
+                    break
+        
+        if pub_date:
+            print(f"Published: {pub_date}")
+        
+        # Word count (if available)
+        word_count = meta.get('word_count')
+        if word_count:
+            print(f"Word Count: {word_count}")
+```
+
+## Smart Filtering with BM25 Scoring
+
+Now for the really cool part - intelligent filtering based on relevance!
+
+### Introduction to Relevance Scoring
+
+BM25 is a ranking algorithm that scores how relevant a document is to a search query. With URL seeding, we can score URLs based on their metadata *before* crawling them.
+
+Think of it like this:
+- Traditional way: Read every book in the library to find ones about Python
+- Smart way: Check the titles and descriptions, score them, read only the most relevant
+
+### Query-Based Discovery
+
+Here's how to use BM25 scoring:
+
+```python
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=True,           # Required for scoring
+    query="python async tutorial",  # What we're looking for
+    scoring_method="bm25",       # Use BM25 algorithm
+    score_threshold=0.3          # Minimum relevance score
+)
+async with AsyncUrlSeeder() as seeder:
+    urls = await seeder.urls("realpython.com", config)
+
+# Results are automatically sorted by relevance!
+for url in urls[:5]:
+    print(f"Score: {url['relevance_score']:.2f} - {url['url']}")
+    print(f"  Title: {url['head_data']['title']}")
+```
+
+### Real Examples
+
+#### Finding Documentation Pages
+
+```python
+# Find API documentation
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=True,
+    query="API reference documentation endpoints",
+    scoring_method="bm25",
+    score_threshold=0.5,
+    max_urls=20
+)
+async with AsyncUrlSeeder() as seeder:
+    urls = await seeder.urls("docs.example.com", config)
+
+# The highest scoring URLs will be API docs!
+```
+
+#### Discovering Product Pages
+
+```python
+# Find specific products
+config = SeedingConfig(
+    source="sitemap+cc",  # Use both sources
+    extract_head=True,
+    query="wireless headphones noise canceling",
+    scoring_method="bm25",
+    score_threshold=0.4,
+    pattern="*/product/*"  # Combine with pattern matching
+)
+async with AsyncUrlSeeder() as seeder:
+    urls = await seeder.urls("shop.example.com", config)
+
+# Filter further by price (from metadata)
+affordable = [
+    u for u in urls 
+    if float(u['head_data'].get('meta', {}).get('product:price', '0')) < 200
+]
+```
+
+#### Filtering News Articles
+
+```python
+# Find recent news about AI
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=True,
+    query="artificial intelligence machine learning breakthrough",
+    scoring_method="bm25",
+    score_threshold=0.35
+)
+async with AsyncUrlSeeder() as seeder:
+    urls = await seeder.urls("technews.com", config)
+
+# Filter by date
+from datetime import datetime, timedelta
+
+recent = []
+cutoff = datetime.now() - timedelta(days=7)
+
+for url in urls:
+    # Check JSON-LD for publication date
+    for jsonld in url['head_data'].get('jsonld', []):
+        if 'datePublished' in jsonld:
+            pub_date = datetime.fromisoformat(jsonld['datePublished'].replace('Z', '+00:00'))
+            if pub_date > cutoff:
+                recent.append(url)
+                break
+```
+
+#### Complex Query Patterns
+
+```python
+# Multi-concept queries
+queries = [
+    "python async await concurrency tutorial",
+    "data science pandas numpy visualization",
+    "web scraping beautifulsoup selenium automation",
+    "machine learning tensorflow keras deep learning"
+]
+
+all_tutorials = []
+
+for query in queries:
+    config = SeedingConfig(
+        source="sitemap",
+        extract_head=True,
+        query=query,
+        scoring_method="bm25",
+        score_threshold=0.4,
+        max_urls=10  # Top 10 per topic
+    )
+    async with AsyncUrlSeeder() as seeder:
+        urls = await seeder.urls("learning-platform.com", config)
+    all_tutorials.extend(urls)
+
+# Remove duplicates while preserving order
+seen = set()
+unique_tutorials = []
+for url in all_tutorials:
+    if url['url'] not in seen:
+        seen.add(url['url'])
+        unique_tutorials.append(url)
+
+print(f"Found {len(unique_tutorials)} unique tutorials across all topics")
+```
+
+## Scaling Up: Multiple Domains
+
+When you need to discover URLs across multiple websites, URL seeding really shines.
+
+### The `many_urls` Method
+
+```python
+# Discover URLs from multiple domains in parallel
+domains = ["site1.com", "site2.com", "site3.com"]
+
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=True,
+    query="python tutorial",
+    scoring_method="bm25",
+    score_threshold=0.3
+)
+
+# Returns a dictionary: {domain: [urls]}
+async with AsyncUrlSeeder() as seeder:
+    results = await seeder.many_urls(domains, config)
+
+# Process results
+for domain, urls in results.items():
+    print(f"\n{domain}: Found {len(urls)} relevant URLs")
+    if urls:
+        top = urls[0]  # Highest scoring
+        print(f"  Top result: {top['url']}")
+        print(f"  Score: {top['relevance_score']:.2f}")
+```
+
+### Cross-Domain Examples
+
+#### Competitor Analysis
+
+```python
+# Analyze content strategies across competitors
+competitors = [
+    "competitor1.com",
+    "competitor2.com", 
+    "competitor3.com"
+]
+
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=True,
+    pattern="*/blog/*",
+    max_urls=100
+)
+async with AsyncUrlSeeder() as seeder:
+    results = await seeder.many_urls(competitors, config)
+
+# Analyze content types
+for domain, urls in results.items():
+    content_types = {}
+    
+    for url in urls:
+        # Extract content type from metadata
+        og_type = url['head_data'].get('meta', {}).get('og:type', 'unknown')
+        content_types[og_type] = content_types.get(og_type, 0) + 1
+    
+    print(f"\n{domain} content distribution:")
+    for ctype, count in sorted(content_types.items(), key=lambda x: x[1], reverse=True):
+        print(f"  {ctype}: {count}")
+```
+
+#### Industry Research
+
+```python
+# Research Python tutorials across educational sites
+educational_sites = [
+    "realpython.com",
+    "pythontutorial.net",
+    "learnpython.org",
+    "python.org"
+]
+
+config = SeedingConfig(
+    source="sitemap",
+    extract_head=True,
+    query="beginner python tutorial basics",
+    scoring_method="bm25",
+    score_threshold=0.3,
+    max_urls=20  # Per site
+)
+async with AsyncUrlSeeder() as seeder:
+    results = await seeder.many_urls(educational_sites, config)
+
+# Find the best beginner tutorials
+all_tutorials = []
+for domain, urls in results.items():
+    for url in urls:
+        url['domain'] = domain  # Add domain info
+        all_tutorials.append(url)
+
+# Sort by relevance across all domains
+all_tutorials.sort(key=lambda x: x['relevance_score'], reverse=True)
+
+print("Top 10 Python tutorials for beginners across all sites:")
+for i, tutorial in enumerate(all_tutorials[:10], 1):
+    print(f"{i}. [{tutorial['relevance_score']:.2f}] {tutorial['head_data']['title']}")
+    print(f"   {tutorial['url']}")
+    print(f"   From: {tutorial['domain']}")
+```
+
+#### Multi-Site Monitoring
+
+```python
+# Monitor news about your company across multiple sources
+news_sites = [
+    "techcrunch.com",
+    "theverge.com",
+    "wired.com",
+    "arstechnica.com"
+]
+
+company_name = "YourCompany"
+
+config = SeedingConfig(
+    source="cc",  # Common Crawl for recent content
+    extract_head=True,
+    query=f"{company_name} announcement news",
+    scoring_method="bm25",
+    score_threshold=0.5,  # High threshold for relevance
+    max_urls=10
+)
+async with AsyncUrlSeeder() as seeder:
+    results = await seeder.many_urls(news_sites, config)
+
+# Collect all mentions
+mentions = []
+for domain, urls in results.items():
+    mentions.extend(urls)
+
+if mentions:
+    print(f"Found {len(mentions)} mentions of {company_name}:")
+    for mention in mentions:
+        print(f"\n- {mention['head_data']['title']}")
+        print(f"  {mention['url']}")
+        print(f"  Score: {mention['relevance_score']:.2f}")
+else:
+    print(f"No recent mentions of {company_name} found")
+```
+
+## Advanced Integration Patterns
+
+Let's put everything together in a real-world example.
+
+### Building a Research Assistant
+
+Here's a complete example that discovers, scores, filters, and crawls intelligently:
+
+```python
+import asyncio
+from datetime import datetime
+from crawl4ai import AsyncUrlSeeder, AsyncWebCrawler, SeedingConfig, CrawlerRunConfig
+
+class ResearchAssistant:
+    def __init__(self):
+        self.seeder = None
+    
+    async def __aenter__(self):
+        self.seeder = AsyncUrlSeeder()
+        await self.seeder.__aenter__()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.seeder:
+            await self.seeder.__aexit__(exc_type, exc_val, exc_tb)
+        
+    async def research_topic(self, topic, domains, max_articles=20):
+        """Research a topic across multiple domains."""
+        
+        print(f"🔬 Researching '{topic}' across {len(domains)} domains...")
+        
+        # Step 1: Discover relevant URLs
+        config = SeedingConfig(
+            source="sitemap+cc",     # Maximum coverage
+            extract_head=True,       # Get metadata
+            query=topic,             # Research topic
+            scoring_method="bm25",   # Smart scoring
+            score_threshold=0.4,     # Quality threshold
+            max_urls=10,             # Per domain
+            concurrency=20,          # Fast discovery
+            verbose=True
+        )
+        
+        # Discover across all domains
+        discoveries = await self.seeder.many_urls(domains, config)
+        
+        # Step 2: Collect and rank all articles
+        all_articles = []
+        for domain, urls in discoveries.items():
+            for url in urls:
+                url['domain'] = domain
+                all_articles.append(url)
+        
+        # Sort by relevance
+        all_articles.sort(key=lambda x: x['relevance_score'], reverse=True)
+        
+        # Take top articles
+        top_articles = all_articles[:max_articles]
+        
+        print(f"\n📊 Found {len(all_articles)} relevant articles")
+        print(f"📌 Selected top {len(top_articles)} for deep analysis")
+        
+        # Step 3: Show what we're about to crawl
+        print("\n🎯 Articles to analyze:")
+        for i, article in enumerate(top_articles[:5], 1):
+            print(f"\n{i}. {article['head_data']['title']}")
+            print(f"   Score: {article['relevance_score']:.2f}")
+            print(f"   Source: {article['domain']}")
+            print(f"   URL: {article['url'][:60]}...")
+        
+        # Step 4: Crawl the selected articles
+        print(f"\n🚀 Deep crawling {len(top_articles)} articles...")
+        
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(
+                only_text=True,
+                word_count_threshold=200,  # Substantial content only
+                stream=True
+            )
+            
+            # Extract URLs and crawl all articles
+            article_urls = [article['url'] for article in top_articles]
+            results = []
+            crawl_results = await crawler.arun_many(article_urls, config=config)
+            async for result in crawl_results:
+                if result.success:
+                    results.append({
+                        'url': result.url,
+                        'title': result.metadata.get('title', 'No title'),
+                        'content': result.markdown.raw_markdown,
+                        'domain': next(a['domain'] for a in top_articles if a['url'] == result.url),
+                        'score': next(a['relevance_score'] for a in top_articles if a['url'] == result.url)
+                    })
+                    print(f"✓ Crawled: {result.url[:60]}...")
+            
+        # Step 5: Analyze and summarize
+        print(f"\n📝 Analysis complete! Crawled {len(results)} articles")
+        
+        return self.create_research_summary(topic, results)
+    
+    def create_research_summary(self, topic, articles):
+        """Create a research summary from crawled articles."""
+        
+        summary = {
+            'topic': topic,
+            'timestamp': datetime.now().isoformat(),
+            'total_articles': len(articles),
+            'sources': {}
+        }
+        
+        # Group by domain
+        for article in articles:
+            domain = article['domain']
+            if domain not in summary['sources']:
+                summary['sources'][domain] = []
+            
+            summary['sources'][domain].append({
+                'title': article['title'],
+                'url': article['url'],
+                'score': article['score'],
+                'excerpt': article['content'][:500] + '...' if len(article['content']) > 500 else article['content']
+            })
+        
+        return summary
+
+# Use the research assistant
+async def main():
+    async with ResearchAssistant() as assistant:
+        # Research Python async programming across multiple sources
+        topic = "python asyncio best practices performance optimization"
+        domains = [
+            "realpython.com",
+            "python.org",
+            "stackoverflow.com",
+            "medium.com"
+        ]
+        
+        summary = await assistant.research_topic(topic, domains, max_articles=15)
+    
+    # Display results
+    print("\n" + "="*60)
+    print("RESEARCH SUMMARY")
+    print("="*60)
+    print(f"Topic: {summary['topic']}")
+    print(f"Date: {summary['timestamp']}")
+    print(f"Total Articles Analyzed: {summary['total_articles']}")
+    
+    print("\nKey Findings by Source:")
+    for domain, articles in summary['sources'].items():
+        print(f"\n📚 {domain} ({len(articles)} articles)")
+        for article in articles[:2]:  # Top 2 per domain
+            print(f"\n  Title: {article['title']}")
+            print(f"  Relevance: {article['score']:.2f}")
+            print(f"  Preview: {article['excerpt'][:200]}...")
+
+asyncio.run(main())
+```
+
+### Performance Optimization Tips
+
+1. **Use caching wisely**
+```python
+# First run - populate cache
+config = SeedingConfig(source="sitemap", extract_head=True, force=True)
+urls = await seeder.urls("example.com", config)
+
+# Subsequent runs - use cache (much faster)
+config = SeedingConfig(source="sitemap", extract_head=True, force=False)
+urls = await seeder.urls("example.com", config)
+```
+
+2. **Optimize concurrency**
+```python
+# For many small requests (like HEAD checks)
+config = SeedingConfig(concurrency=50, hits_per_sec=20)
+
+# For fewer large requests (like full head extraction)
+config = SeedingConfig(concurrency=10, hits_per_sec=5)
+```
+
+3. **Stream large result sets**
+```python
+# When crawling many URLs
+async with AsyncWebCrawler() as crawler:
+    # Assuming urls is a list of URL strings
+    crawl_results = await crawler.arun_many(urls, config=config)
+    
+    # Process as they arrive
+    async for result in crawl_results:
+        process_immediately(result)  # Don't wait for all
+```
+
+4. **Memory protection for large domains**
+
+The seeder uses bounded queues to prevent memory issues when processing domains with millions of URLs:
+
+```python
+# Safe for domains with 1M+ URLs
+config = SeedingConfig(
+    source="cc+sitemap",
+    concurrency=50,  # Queue size adapts to concurrency
+    max_urls=100000  # Process in batches if needed
+)
+
+# The seeder automatically manages memory by:
+# - Using bounded queues (prevents RAM spikes)
+# - Applying backpressure when queue is full
+# - Processing URLs as they're discovered
+```
+
+## Best Practices & Tips
+
+### Cache Management
+
+The seeder automatically caches results to speed up repeated operations:
+
+- **Common Crawl cache**: `~/.crawl4ai/seeder_cache/[index]_[domain]_[hash].jsonl`
+- **Sitemap cache**: `~/.crawl4ai/seeder_cache/sitemap_[domain]_[hash].json`
+- **HEAD data cache**: `~/.cache/url_seeder/head/[hash].json`
+
+#### Smart TTL Cache for Sitemaps
+
+Sitemap caches now include intelligent validation:
+
+```python
+# Default: 24-hour TTL with lastmod validation
+config = SeedingConfig(
+    source="sitemap",
+    cache_ttl_hours=24,              # Cache expires after 24 hours
+    validate_sitemap_lastmod=True    # Also check if sitemap was updated
+)
+
+# Aggressive caching (1 week, no lastmod check)
+config = SeedingConfig(
+    source="sitemap",
+    cache_ttl_hours=168,             # 7 days
+    validate_sitemap_lastmod=False   # Trust TTL only
+)
+
+# Always validate (no TTL, only lastmod)
+config = SeedingConfig(
+    source="sitemap",
+    cache_ttl_hours=0,               # Disable TTL
+    validate_sitemap_lastmod=True    # Refetch if sitemap has newer lastmod
+)
+
+# Always fresh (bypass cache completely)
+config = SeedingConfig(
+    source="sitemap",
+    force=True                       # Ignore all caching
+)
+```
+
+**Cache validation priority:**
+1. `force=True` → Always refetch
+2. Cache doesn't exist → Fetch fresh
+3. `validate_sitemap_lastmod=True` and sitemap has newer `` → Refetch
+4. `cache_ttl_hours > 0` and cache is older than TTL → Refetch
+5. Cache corrupted → Refetch (automatic recovery)
+6. Otherwise → Use cache
+
+### Pattern Matching Strategies
+
+```python
+# Be specific when possible
+good_pattern = "*/blog/2024/*.html"  # Specific
+bad_pattern = "*"                     # Too broad
+
+# Combine patterns with metadata filtering
+config = SeedingConfig(
+    pattern="*/articles/*",
+    extract_head=True
+)
+urls = await seeder.urls("news.com", config)
+
+# Further filter by publish date, author, category, etc.
+recent = [u for u in urls if is_recent(u['head_data'])]
+```
+
+### Rate Limiting Considerations
+
+```python
+# Be respectful of servers
+config = SeedingConfig(
+    hits_per_sec=10,      # Max 10 requests per second
+    concurrency=20        # But use 20 workers
+)
+
+# For your own servers
+config = SeedingConfig(
+    hits_per_sec=None,    # No limit
+    concurrency=100       # Go fast
+)
+```
+
+## Quick Reference
+
+### Common Patterns
+
+```python
+# Blog post discovery
+config = SeedingConfig(
+    source="sitemap",
+    pattern="*/blog/*",
+    extract_head=True,
+    query="your topic",
+    scoring_method="bm25"
+)
+
+# E-commerce product discovery
+config = SeedingConfig(
+    source="sitemap+cc",
+    pattern="*/product/*",
+    extract_head=True,
+    live_check=True
+)
+
+# Documentation search
+config = SeedingConfig(
+    source="sitemap",
+    pattern="*/docs/*",
+    extract_head=True,
+    query="API reference",
+    scoring_method="bm25",
+    score_threshold=0.5
+)
+
+# News monitoring
+config = SeedingConfig(
+    source="cc",
+    extract_head=True,
+    query="company name",
+    scoring_method="bm25",
+    max_urls=50
+)
+```
+
+### Troubleshooting Guide
+
+| Issue | Solution |
+|-------|----------|
+| No URLs found | Try `source="cc+sitemap"`, check domain spelling |
+| Slow discovery | Reduce `concurrency`, add `hits_per_sec` limit |
+| Missing metadata | Ensure `extract_head=True` |
+| Low relevance scores | Refine query, lower `score_threshold` |
+| Rate limit errors | Reduce `hits_per_sec` and `concurrency` |
+| Memory issues with large sites | Use `max_urls` to limit results, reduce `concurrency` |
+| Connection not closed | Use context manager or call `await seeder.close()` |
+| Stale/outdated URLs | Set `cache_ttl_hours=0` or use `force=True` |
+| Cache not updating | Check `validate_sitemap_lastmod=True`, or use `force=True` |
+| Incomplete URL list | Delete cache file and refetch, or use `force=True` |
+
+### Performance Benchmarks
+
+Typical performance on a standard connection:
+
+- **Sitemap discovery**: 100-1,000 URLs/second
+- **Common Crawl discovery**: 50-500 URLs/second  
+- **HEAD checking**: 10-50 URLs/second
+- **Head extraction**: 5-20 URLs/second
+- **BM25 scoring**: 10,000+ URLs/second
+
+## Conclusion
+
+URL seeding transforms web crawling from a blind expedition into a surgical strike. By discovering and analyzing URLs before crawling, you can:
+
+- Save hours of crawling time
+- Reduce bandwidth usage by 90%+
+- Find exactly what you need
+- Scale across multiple domains effortlessly
+
+Whether you're building a research tool, monitoring competitors, or creating a content aggregator, URL seeding gives you the intelligence to crawl smarter, not harder.
+
+### Smart URL Filtering
+
+The seeder automatically filters out nonsense URLs that aren't useful for content crawling:
+
+```python
+# Enabled by default
+config = SeedingConfig(
+    source="sitemap",
+    filter_nonsense_urls=True  # Default: True
+)
+
+# URLs that get filtered:
+# - robots.txt, sitemap.xml, ads.txt
+# - API endpoints (/api/, /v1/, .json)
+# - Media files (.jpg, .mp4, .pdf)
+# - Archives (.zip, .tar.gz)
+# - Source code (.js, .css)
+# - Admin/login pages
+# - And many more...
+```
+
+To disable filtering (not recommended):
+
+```python
+config = SeedingConfig(
+    source="sitemap",
+    filter_nonsense_urls=False  # Include ALL URLs
+)
+```
+
+### Key Features Summary
+
+1. **Parallel Sitemap Index Processing**: Automatically detects and processes sitemap indexes in parallel
+2. **Memory Protection**: Bounded queues prevent RAM issues with large domains (1M+ URLs)
+3. **Context Manager Support**: Automatic cleanup with `async with` statement
+4. **URL-Based Scoring**: Smart filtering even without head extraction
+5. **Smart URL Filtering**: Automatically excludes utility/nonsense URLs
+6. **Smart TTL Cache**: Sitemap caches with TTL expiry and lastmod validation
+7. **Automatic Cache Recovery**: Corrupted or incomplete caches are automatically refreshed
+
+Now go forth and seed intelligently!
+
+## Need More Coverage?
+
+If you need to discover URLs across an entire domain — including subdomains, hidden services, and pages not listed in any sitemap — check out [Domain Mapping](domain-mapping.md). It combines 8 discovery sources (including Certificate Transparency, Wayback Machine, path probing, and soft-404 detection) to find everything under a domain.
\ No newline at end of file
diff --git a/docs/md_v2/extraction/chunking.md b/docs/md_v2/extraction/chunking.md
new file mode 100644
index 0000000..2a04a60
--- /dev/null
+++ b/docs/md_v2/extraction/chunking.md
@@ -0,0 +1,144 @@
+# Chunking Strategies
+Chunking strategies are critical for dividing large texts into manageable parts, enabling effective content processing and extraction. These strategies are foundational in cosine similarity-based extraction techniques, which allow users to retrieve only the most relevant chunks of content for a given query. Additionally, they facilitate direct integration into RAG (Retrieval-Augmented Generation) systems for structured and scalable workflows.
+
+### Why Use Chunking?
+1. **Cosine Similarity and Query Relevance**: Prepares chunks for semantic similarity analysis.
+2. **RAG System Integration**: Seamlessly processes and stores chunks for retrieval.
+3. **Structured Processing**: Allows for diverse segmentation methods, such as sentence-based, topic-based, or windowed approaches.
+
+### Methods of Chunking
+
+#### 1. Regex-Based Chunking
+Splits text based on regular expression patterns, useful for coarse segmentation.
+
+**Code Example**:
+```python
+class RegexChunking:
+    def __init__(self, patterns=None):
+        self.patterns = patterns or [r'\n\n']  # Default pattern for paragraphs
+
+    def chunk(self, text):
+        paragraphs = [text]
+        for pattern in self.patterns:
+            paragraphs = [seg for p in paragraphs for seg in re.split(pattern, p)]
+        return paragraphs
+
+# Example Usage
+text = """This is the first paragraph.
+
+This is the second paragraph."""
+chunker = RegexChunking()
+print(chunker.chunk(text))
+```
+
+#### 2. Sentence-Based Chunking
+Divides text into sentences using NLP tools, ideal for extracting meaningful statements.
+
+**Code Example**:
+```python
+from nltk.tokenize import sent_tokenize
+
+class NlpSentenceChunking:
+    def chunk(self, text):
+        sentences = sent_tokenize(text)
+        return [sentence.strip() for sentence in sentences]
+
+# Example Usage
+text = "This is sentence one. This is sentence two."
+chunker = NlpSentenceChunking()
+print(chunker.chunk(text))
+```
+
+#### 3. Topic-Based Segmentation
+Uses algorithms like TextTiling to create topic-coherent chunks.
+
+**Code Example**:
+```python
+from nltk.tokenize import TextTilingTokenizer
+
+class TopicSegmentationChunking:
+    def __init__(self):
+        self.tokenizer = TextTilingTokenizer()
+
+    def chunk(self, text):
+        return self.tokenizer.tokenize(text)
+
+# Example Usage
+text = """This is an introduction.
+This is a detailed discussion on the topic."""
+chunker = TopicSegmentationChunking()
+print(chunker.chunk(text))
+```
+
+#### 4. Fixed-Length Word Chunking
+Segments text into chunks of a fixed word count.
+
+**Code Example**:
+```python
+class FixedLengthWordChunking:
+    def __init__(self, chunk_size=100):
+        self.chunk_size = chunk_size
+
+    def chunk(self, text):
+        words = text.split()
+        return [' '.join(words[i:i + self.chunk_size]) for i in range(0, len(words), self.chunk_size)]
+
+# Example Usage
+text = "This is a long text with many words to be chunked into fixed sizes."
+chunker = FixedLengthWordChunking(chunk_size=5)
+print(chunker.chunk(text))
+```
+
+#### 5. Sliding Window Chunking
+Generates overlapping chunks for better contextual coherence.
+
+**Code Example**:
+```python
+class SlidingWindowChunking:
+    def __init__(self, window_size=100, step=50):
+        self.window_size = window_size
+        self.step = step
+
+    def chunk(self, text):
+        words = text.split()
+        chunks = []
+        for i in range(0, len(words) - self.window_size + 1, self.step):
+            chunks.append(' '.join(words[i:i + self.window_size]))
+        return chunks
+
+# Example Usage
+text = "This is a long text to demonstrate sliding window chunking."
+chunker = SlidingWindowChunking(window_size=5, step=2)
+print(chunker.chunk(text))
+```
+
+### Combining Chunking with Cosine Similarity
+To enhance the relevance of extracted content, chunking strategies can be paired with cosine similarity techniques. Here’s an example workflow:
+
+**Code Example**:
+```python
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+class CosineSimilarityExtractor:
+    def __init__(self, query):
+        self.query = query
+        self.vectorizer = TfidfVectorizer()
+
+    def find_relevant_chunks(self, chunks):
+        vectors = self.vectorizer.fit_transform([self.query] + chunks)
+        similarities = cosine_similarity(vectors[0:1], vectors[1:]).flatten()
+        return [(chunks[i], similarities[i]) for i in range(len(chunks))]
+
+# Example Workflow
+text = """This is a sample document. It has multiple sentences. 
+We are testing chunking and similarity."""
+
+chunker = SlidingWindowChunking(window_size=5, step=3)
+chunks = chunker.chunk(text)
+query = "testing chunking"
+extractor = CosineSimilarityExtractor(query)
+relevant_chunks = extractor.find_relevant_chunks(chunks)
+
+print(relevant_chunks)
+```
diff --git a/docs/md_v2/extraction/clustring-strategies.md b/docs/md_v2/extraction/clustring-strategies.md
new file mode 100644
index 0000000..c32b1c9
--- /dev/null
+++ b/docs/md_v2/extraction/clustring-strategies.md
@@ -0,0 +1,222 @@
+# Cosine Strategy
+
+The Cosine Strategy in Crawl4AI uses similarity-based clustering to identify and extract relevant content sections from web pages. This strategy is particularly useful when you need to find and extract content based on semantic similarity rather than structural patterns.
+
+## How It Works
+
+The Cosine Strategy:
+1. Breaks down page content into meaningful chunks
+2. Converts text into vector representations
+3. Calculates similarity between chunks
+4. Clusters similar content together
+5. Ranks and filters content based on relevance
+
+## Basic Usage
+
+```python
+from crawl4ai import CosineStrategy
+
+strategy = CosineStrategy(
+    semantic_filter="product reviews",    # Target content type
+    word_count_threshold=10,             # Minimum words per cluster
+    sim_threshold=0.3                    # Similarity threshold
+)
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://example.com/reviews",
+        extraction_strategy=strategy
+    )
+    
+    content = result.extracted_content
+```
+
+## Configuration Options
+
+### Core Parameters
+
+```python
+CosineStrategy(
+    # Content Filtering
+    semantic_filter: str = None,       # Keywords/topic for content filtering
+    word_count_threshold: int = 10,    # Minimum words per cluster
+    sim_threshold: float = 0.3,        # Similarity threshold (0.0 to 1.0)
+    
+    # Clustering Parameters
+    max_dist: float = 0.2,            # Maximum distance for clustering
+    linkage_method: str = 'ward',      # Clustering linkage method
+    top_k: int = 3,                   # Number of top categories to extract
+    
+    # Model Configuration
+    model_name: str = 'sentence-transformers/all-MiniLM-L6-v2',  # Embedding model
+    
+    verbose: bool = False             # Enable logging
+)
+```
+
+### Parameter Details
+
+1. **semantic_filter**
+   - Sets the target topic or content type
+   - Use keywords relevant to your desired content
+   - Example: "technical specifications", "user reviews", "pricing information"
+
+2. **sim_threshold**
+   - Controls how similar content must be to be grouped together
+   - Higher values (e.g., 0.8) mean stricter matching
+   - Lower values (e.g., 0.3) allow more variation
+   ```python
+   # Strict matching
+   strategy = CosineStrategy(sim_threshold=0.8)
+   
+   # Loose matching
+   strategy = CosineStrategy(sim_threshold=0.3)
+   ```
+
+3. **word_count_threshold**
+   - Filters out short content blocks
+   - Helps eliminate noise and irrelevant content
+   ```python
+   # Only consider substantial paragraphs
+   strategy = CosineStrategy(word_count_threshold=50)
+   ```
+
+4. **top_k**
+   - Number of top content clusters to return
+   - Higher values return more diverse content
+   ```python
+   # Get top 5 most relevant content clusters
+   strategy = CosineStrategy(top_k=5)
+   ```
+
+## Use Cases
+
+### 1. Article Content Extraction
+```python
+strategy = CosineStrategy(
+    semantic_filter="main article content",
+    word_count_threshold=100,  # Longer blocks for articles
+    top_k=1                   # Usually want single main content
+)
+
+result = await crawler.arun(
+    url="https://example.com/blog/post",
+    extraction_strategy=strategy
+)
+```
+
+### 2. Product Review Analysis
+```python
+strategy = CosineStrategy(
+    semantic_filter="customer reviews and ratings",
+    word_count_threshold=20,   # Reviews can be shorter
+    top_k=10,                 # Get multiple reviews
+    sim_threshold=0.4         # Allow variety in review content
+)
+```
+
+### 3. Technical Documentation
+```python
+strategy = CosineStrategy(
+    semantic_filter="technical specifications documentation",
+    word_count_threshold=30,
+    sim_threshold=0.6,        # Stricter matching for technical content
+    max_dist=0.3             # Allow related technical sections
+)
+```
+
+## Advanced Features
+
+### Custom Clustering
+```python
+strategy = CosineStrategy(
+    linkage_method='complete',  # Alternative clustering method
+    max_dist=0.4,              # Larger clusters
+    model_name='sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2'  # Multilingual support
+)
+```
+
+### Content Filtering Pipeline
+```python
+strategy = CosineStrategy(
+    semantic_filter="pricing plans features",
+    word_count_threshold=15,
+    sim_threshold=0.5,
+    top_k=3
+)
+
+async def extract_pricing_features(url: str):
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url=url,
+            extraction_strategy=strategy
+        )
+        
+        if result.success:
+            content = json.loads(result.extracted_content)
+            return {
+                'pricing_features': content,
+                'clusters': len(content),
+                'similarity_scores': [item['score'] for item in content]
+            }
+```
+
+## Best Practices
+
+1. **Adjust Thresholds Iteratively**
+   - Start with default values
+   - Adjust based on results
+   - Monitor clustering quality
+
+2. **Choose Appropriate Word Count Thresholds**
+   - Higher for articles (100+)
+   - Lower for reviews/comments (20+)
+   - Medium for product descriptions (50+)
+
+3. **Optimize Performance**
+   ```python
+   strategy = CosineStrategy(
+       word_count_threshold=10,  # Filter early
+       top_k=5,                 # Limit results
+       verbose=True             # Monitor performance
+   )
+   ```
+
+4. **Handle Different Content Types**
+   ```python
+   # For mixed content pages
+   strategy = CosineStrategy(
+       semantic_filter="product features",
+       sim_threshold=0.4,      # More flexible matching
+       max_dist=0.3,          # Larger clusters
+       top_k=3                # Multiple relevant sections
+   )
+   ```
+
+## Error Handling
+
+```python
+try:
+    result = await crawler.arun(
+        url="https://example.com",
+        extraction_strategy=strategy
+    )
+    
+    if result.success:
+        content = json.loads(result.extracted_content)
+        if not content:
+            print("No relevant content found")
+    else:
+        print(f"Extraction failed: {result.error_message}")
+        
+except Exception as e:
+    print(f"Error during extraction: {str(e)}")
+```
+
+The Cosine Strategy is particularly effective when:
+- Content structure is inconsistent
+- You need semantic understanding
+- You want to find similar content blocks
+- Structure-based extraction (CSS/XPath) isn't reliable
+
+It works well with other strategies and can be used as a pre-processing step for LLM-based extraction.
\ No newline at end of file
diff --git a/docs/md_v2/extraction/llm-strategies.md b/docs/md_v2/extraction/llm-strategies.md
new file mode 100644
index 0000000..cba4d6e
--- /dev/null
+++ b/docs/md_v2/extraction/llm-strategies.md
@@ -0,0 +1,330 @@
+# Extracting JSON (LLM)
+
+In some cases, you need to extract **complex or unstructured** information from a webpage that a simple CSS/XPath schema cannot easily parse. Or you want **AI**-driven insights, classification, or summarization. For these scenarios, Crawl4AI provides an **LLM-based extraction strategy** that:
+
+1. Works with **any** large language model supported by [LiteLLM](https://github.com/BerriAI/litellm) (Ollama, OpenAI, Claude, and more).  
+2. Automatically splits content into chunks (if desired) to handle token limits, then combines results.  
+3. Lets you define a **schema** (like a Pydantic model) or a simpler “block” extraction approach.
+
+**Important**: LLM-based extraction can be slower and costlier than schema-based approaches. If your page data is highly structured, consider using [`JsonCssExtractionStrategy`](./no-llm-strategies.md) or [`JsonXPathExtractionStrategy`](./no-llm-strategies.md) first. But if you need AI to interpret or reorganize content, read on!
+
+---
+
+## 1. Why Use an LLM?
+
+- **Complex Reasoning**: If the site’s data is unstructured, scattered, or full of natural language context.  
+- **Semantic Extraction**: Summaries, knowledge graphs, or relational data that require comprehension.  
+- **Flexible**: You can pass instructions to the model to do more advanced transformations or classification.
+
+---
+
+## 2. Provider-Agnostic via LiteLLM
+
+You can use LLMConfig, to quickly configure multiple variations of LLMs and experiment with them to find the optimal one for your use case. You can read more about LLMConfig [here](/api/parameters).
+
+```python
+llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
+```
+
+Crawl4AI uses a “provider string” (e.g., `"openai/gpt-4o"`, `"ollama/llama2.0"`, `"aws/titan"`) to identify your LLM. **Any** model that LiteLLM supports is fair game. You just provide:
+
+- **`provider`**: The `/` identifier (e.g., `"openai/gpt-4"`, `"ollama/llama2"`, `"huggingface/google-flan"`, etc.).  
+- **`api_token`**: If needed (for OpenAI, HuggingFace, etc.); local models or Ollama might not require it.  
+- **`base_url`** (optional): If your provider has a custom endpoint.  
+
+This means you **aren’t locked** into a single LLM vendor. Switch or experiment easily.
+
+---
+
+## 3. How LLM Extraction Works
+
+### 3.1 Flow
+
+1. **Chunking** (optional): The HTML or markdown is split into smaller segments if it’s very long (based on `chunk_token_threshold`, overlap, etc.).  
+2. **Prompt Construction**: For each chunk, the library forms a prompt that includes your **`instruction`** (and possibly schema or examples).  
+3. **LLM Inference**: Each chunk is sent to the model in parallel or sequentially (depending on your concurrency).  
+4. **Combining**: The results from each chunk are merged and parsed into JSON.
+
+### 3.2 `extraction_type`
+
+- **`"schema"`**: The model tries to return JSON conforming to your Pydantic-based schema.  
+- **`"block"`**: The model returns freeform text, or smaller JSON structures, which the library collects.  
+
+For structured data, `"schema"` is recommended. You provide `schema=YourPydanticModel.model_json_schema()`.
+
+---
+
+## 4. Key Parameters
+
+Below is an overview of important LLM extraction parameters. All are typically set inside `LLMExtractionStrategy(...)`. You then put that strategy in your `CrawlerRunConfig(..., extraction_strategy=...)`.
+
+1. **`llm_config`** (LLMConfig): e.g., `"openai/gpt-4"`, `"ollama/llama2"`.
+2. **`schema`** (dict): A JSON schema describing the fields you want. Usually generated by `YourModel.model_json_schema()`.  
+3. **`extraction_type`** (str): `"schema"` or `"block"`.  
+4. **`instruction`** (str): Prompt text telling the LLM what you want extracted. E.g., “Extract these fields as a JSON array.”  
+5. **`chunk_token_threshold`** (int): Maximum tokens per chunk. If your content is huge, you can break it up for the LLM.  
+6. **`overlap_rate`** (float): Overlap ratio between adjacent chunks. E.g., `0.1` means 10% of each chunk is repeated to preserve context continuity.  
+7. **`apply_chunking`** (bool): Set `True` to chunk automatically. If you want a single pass, set `False`.  
+8. **`input_format`** (str): Determines **which** crawler result is passed to the LLM. Options include:  
+   - `"markdown"`: The raw markdown (default).  
+   - `"fit_markdown"`: The filtered “fit” markdown if you used a content filter.  
+   - `"html"`: The cleaned or raw HTML.  
+9. **`extra_args`** (dict): Additional LLM parameters like `temperature`, `max_tokens`, `top_p`, etc.  
+10. **`show_usage()`**: A method you can call to print out usage info (token usage per chunk, total cost if known).  
+
+**Example**:
+
+```python
+extraction_strategy = LLMExtractionStrategy(
+    llm_config = LLMConfig(provider="openai/gpt-4", api_token="YOUR_OPENAI_KEY"),
+    schema=MyModel.model_json_schema(),
+    extraction_type="schema",
+    instruction="Extract a list of items from the text with 'name' and 'price' fields.",
+    chunk_token_threshold=1200,
+    overlap_rate=0.1,
+    apply_chunking=True,
+    input_format="html",
+    extra_args={"temperature": 0.1, "max_tokens": 1000},
+    verbose=True
+)
+```
+
+---
+
+## 5. Putting It in `CrawlerRunConfig`
+
+**Important**: In Crawl4AI, all strategy definitions should go inside the `CrawlerRunConfig`, not directly as a param in `arun()`. Here’s a full example:
+
+```python
+import os
+import asyncio
+import json
+from pydantic import BaseModel, Field
+from typing import List
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
+from crawl4ai import LLMExtractionStrategy
+
+class Product(BaseModel):
+    name: str
+    price: str
+
+async def main():
+    # 1. Define the LLM extraction strategy
+    llm_strategy = LLMExtractionStrategy(
+        llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv('OPENAI_API_KEY')),
+        schema=Product.model_json_schema(), # Or use model_json_schema()
+        extraction_type="schema",
+        instruction="Extract all product objects with 'name' and 'price' from the content.",
+        chunk_token_threshold=1000,
+        overlap_rate=0.0,
+        apply_chunking=True,
+        input_format="markdown",   # or "html", "fit_markdown"
+        extra_args={"temperature": 0.0, "max_tokens": 800}
+    )
+
+    # 2. Build the crawler config
+    crawl_config = CrawlerRunConfig(
+        extraction_strategy=llm_strategy,
+        cache_mode=CacheMode.BYPASS
+    )
+
+    # 3. Create a browser config if needed
+    browser_cfg = BrowserConfig(headless=True)
+
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        # 4. Let's say we want to crawl a single page
+        result = await crawler.arun(
+            url="https://example.com/products",
+            config=crawl_config
+        )
+
+        if result.success:
+            # 5. The extracted content is presumably JSON
+            data = json.loads(result.extracted_content)
+            print("Extracted items:", data)
+            
+            # 6. Show usage stats
+            llm_strategy.show_usage()  # prints token usage
+        else:
+            print("Error:", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+## 6. Chunking Details
+
+### 6.1 `chunk_token_threshold`
+
+If your page is large, you might exceed your LLM’s context window. **`chunk_token_threshold`** sets the approximate max tokens per chunk. The library calculates word→token ratio using `word_token_rate` (often ~0.75 by default). If chunking is enabled (`apply_chunking=True`), the text is split into segments.
+
+### 6.2 `overlap_rate`
+
+To keep context continuous across chunks, we can overlap them. E.g., `overlap_rate=0.1` means each subsequent chunk includes 10% of the previous chunk’s text. This is helpful if your needed info might straddle chunk boundaries.
+
+### 6.3 Performance & Parallelism
+
+By chunking, you can potentially process multiple chunks in parallel (depending on your concurrency settings and the LLM provider). This reduces total time if the site is huge or has many sections.
+
+---
+
+## 7. Input Format
+
+By default, **LLMExtractionStrategy** uses `input_format="markdown"`, meaning the **crawler’s final markdown** is fed to the LLM. You can change to:
+
+- **`html`**: The cleaned HTML or raw HTML (depending on your crawler config) goes into the LLM.  
+- **`fit_markdown`**: If you used, for instance, `PruningContentFilter`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.  
+- **`markdown`**: Standard markdown output from the crawler’s `markdown_generator`.
+
+This setting is crucial: if the LLM instructions rely on HTML tags, pick `"html"`. If you prefer a text-based approach, pick `"markdown"`.
+
+```python
+LLMExtractionStrategy(
+    # ...
+    input_format="html",  # Instead of "markdown" or "fit_markdown"
+)
+```
+
+---
+
+## 8. Token Usage & Show Usage
+
+To keep track of tokens and cost, each chunk is processed with an LLM call. We record usage in:
+
+- **`usages`** (list): token usage per chunk or call.  
+- **`total_usage`**: sum of all chunk calls.  
+- **`show_usage()`**: prints a usage report (if the provider returns usage data).
+
+```python
+llm_strategy = LLMExtractionStrategy(...)
+# ...
+llm_strategy.show_usage()
+# e.g. “Total usage: 1241 tokens across 2 chunk calls”
+```
+
+If your model provider doesn't return usage info, these fields might be partial or empty.
+
+> **Tip:** `JsonCssExtractionStrategy.generate_schema()` also supports token usage tracking via an optional `usage` parameter. See [Token Usage Tracking in Schema Generation](./no-llm-strategies.md#token-usage-tracking) for details.
+
+---
+
+## 9. Example: Building a Knowledge Graph
+
+Below is a snippet combining **`LLMExtractionStrategy`** with a Pydantic schema for a knowledge graph. Notice how we pass an **`instruction`** telling the model what to parse.
+
+```python
+import os
+import json
+import asyncio
+from typing import List
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
+from crawl4ai import LLMExtractionStrategy
+
+class Entity(BaseModel):
+    name: str
+    description: str
+
+class Relationship(BaseModel):
+    entity1: Entity
+    entity2: Entity
+    description: str
+    relation_type: str
+
+class KnowledgeGraph(BaseModel):
+    entities: List[Entity]
+    relationships: List[Relationship]
+
+async def main():
+    # LLM extraction strategy
+    llm_strat = LLMExtractionStrategy(
+        llm_config = LLMConfig(provider="openai/gpt-4", api_token=os.getenv('OPENAI_API_KEY')),
+        schema=KnowledgeGraph.model_json_schema(),
+        extraction_type="schema",
+        instruction="Extract entities and relationships from the content. Return valid JSON.",
+        chunk_token_threshold=1400,
+        apply_chunking=True,
+        input_format="html",
+        extra_args={"temperature": 0.1, "max_tokens": 1500}
+    )
+
+    crawl_config = CrawlerRunConfig(
+        extraction_strategy=llm_strat,
+        cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        # Example page
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, config=crawl_config)
+
+        print("--- LLM RAW RESPONSE ---")
+        print(result.extracted_content)
+        print("--- END LLM RAW RESPONSE ---")
+
+        if result.success:
+            with open("kb_result.json", "w", encoding="utf-8") as f:
+                f.write(result.extracted_content)
+            llm_strat.show_usage()
+        else:
+            print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key Observations**:
+
+- **`extraction_type="schema"`** ensures we get JSON fitting our `KnowledgeGraph`.  
+- **`input_format="html"`** means we feed HTML to the model.  
+- **`instruction`** guides the model to output a structured knowledge graph.  
+
+---
+
+## 10. Best Practices & Caveats
+
+1. **Cost & Latency**: LLM calls can be slow or expensive. Consider chunking or smaller coverage if you only need partial data.  
+2. **Model Token Limits**: If your page + instruction exceed the context window, chunking is essential.  
+3. **Instruction Engineering**: Well-crafted instructions can drastically improve output reliability.  
+4. **Schema Strictness**: `"schema"` extraction tries to parse the model output as JSON. If the model returns invalid JSON, partial extraction might happen, or you might get an error.  
+5. **Parallel vs. Serial**: The library can process multiple chunks in parallel, but you must watch out for rate limits on certain providers.  
+6. **Check Output**: Sometimes, an LLM might omit fields or produce extraneous text. You may want to post-validate with Pydantic or do additional cleanup.
+
+---
+
+## 11. Conclusion
+
+**LLM-based extraction** in Crawl4AI is **provider-agnostic**, letting you choose from hundreds of models via LiteLLM. It’s perfect for **semantically complex** tasks or generating advanced structures like knowledge graphs. However, it’s **slower** and potentially costlier than schema-based approaches. Keep these tips in mind:
+
+- Put your LLM strategy **in `CrawlerRunConfig`**.  
+- Use **`input_format`** to pick which form (markdown, HTML, fit_markdown) the LLM sees.  
+- Tweak **`chunk_token_threshold`**, **`overlap_rate`**, and **`apply_chunking`** to handle large content efficiently.  
+- Monitor token usage with `show_usage()`.
+
+If your site’s data is consistent or repetitive, consider [`JsonCssExtractionStrategy`](./no-llm-strategies.md) first for speed and simplicity. But if you need an **AI-driven** approach, `LLMExtractionStrategy` offers a flexible, multi-provider solution for extracting structured JSON from any website.
+
+**Next Steps**:
+
+1. **Experiment with Different Providers**  
+   - Try switching the `provider` (e.g., `"ollama/llama2"`, `"openai/gpt-4o"`, etc.) to see differences in speed, accuracy, or cost.  
+   - Pass different `extra_args` like `temperature`, `top_p`, and `max_tokens` to fine-tune your results.
+
+2. **Performance Tuning**  
+   - If pages are large, tweak `chunk_token_threshold`, `overlap_rate`, or `apply_chunking` to optimize throughput.  
+   - Check the usage logs with `show_usage()` to keep an eye on token consumption and identify potential bottlenecks.
+
+3. **Validate Outputs**  
+   - If using `extraction_type="schema"`, parse the LLM’s JSON with a Pydantic model for a final validation step.  
+   - Log or handle any parse errors gracefully, especially if the model occasionally returns malformed JSON.
+
+4. **Explore Hooks & Automation**  
+   - Integrate LLM extraction with [hooks](../advanced/hooks-auth.md) for complex pre/post-processing.  
+   - Use a multi-step pipeline: crawl, filter, LLM-extract, then store or index results for further analysis.
+
+**Last Updated**: 2025-01-01
+
+---
+
+That’s it for **Extracting JSON (LLM)**—now you can harness AI to parse, classify, or reorganize data on the web. Happy crawling!
diff --git a/docs/md_v2/extraction/no-llm-strategies.md b/docs/md_v2/extraction/no-llm-strategies.md
new file mode 100644
index 0000000..63138b3
--- /dev/null
+++ b/docs/md_v2/extraction/no-llm-strategies.md
@@ -0,0 +1,946 @@
+# Extracting JSON (No LLM)
+
+One of Crawl4AI's **most powerful** features is extracting **structured JSON** from websites **without** relying on large language models. Crawl4AI offers several strategies for LLM-free extraction:
+
+1. **Schema-based extraction** with CSS or XPath selectors via `JsonCssExtractionStrategy` and `JsonXPathExtractionStrategy`
+2. **Regular expression extraction** with `RegexExtractionStrategy` for fast pattern matching
+
+These approaches let you extract data instantly—even from complex or nested HTML structures—without the cost, latency, or environmental impact of an LLM.
+
+**Why avoid LLM for basic extractions?**
+
+1. **Faster & Cheaper**: No API calls or GPU overhead.  
+2. **Lower Carbon Footprint**: LLM inference can be energy-intensive. Pattern-based extraction is practically carbon-free.  
+3. **Precise & Repeatable**: CSS/XPath selectors and regex patterns do exactly what you specify. LLM outputs can vary or hallucinate.  
+4. **Scales Readily**: For thousands of pages, pattern-based extraction runs quickly and in parallel.
+
+Below, we'll explore how to craft these schemas and use them with **JsonCssExtractionStrategy** (or **JsonXPathExtractionStrategy** if you prefer XPath). We'll also highlight advanced features like **nested fields** and **base element attributes**.
+
+---
+
+## 1. Intro to Schema-Based Extraction
+
+A schema defines:
+
+1. A **base selector** that identifies each "container" element on the page (e.g., a product row, a blog post card).  
+2. **Fields** describing which CSS/XPath selectors to use for each piece of data you want to capture (text, attribute, HTML block, etc.).  
+3. **Nested** or **list** types for repeated or hierarchical structures.  
+
+For example, if you have a list of products, each one might have a name, price, reviews, and "related products." This approach is faster and more reliable than an LLM for consistent, structured pages.
+
+---
+
+## 2. Simple Example: Crypto Prices
+
+Let's begin with a **simple** schema-based extraction using the `JsonCssExtractionStrategy`. Below is a snippet that extracts cryptocurrency prices from a site (similar to the legacy Coinbase example). Notice we **don't** call any LLM:
+
+```python
+import json
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai import JsonCssExtractionStrategy
+
+async def extract_crypto_prices():
+    # 1. Define a simple extraction schema
+    schema = {
+        "name": "Crypto Prices",
+        "baseSelector": "div.crypto-row",    # Repeated elements
+        "fields": [
+            {
+                "name": "coin_name",
+                "selector": "h2.coin-name",
+                "type": "text"
+            },
+            {
+                "name": "price",
+                "selector": "span.coin-price",
+                "type": "text"
+            }
+        ]
+    }
+
+    # 2. Create the extraction strategy
+    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+    # 3. Set up your crawler config (if needed)
+    config = CrawlerRunConfig(
+        # e.g., pass js_code or wait_for if the page is dynamic
+        # wait_for="css:.crypto-row:nth-child(20)"
+        cache_mode = CacheMode.BYPASS,
+        extraction_strategy=extraction_strategy,
+    )
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        # 4. Run the crawl and extraction
+        result = await crawler.arun(
+            url="https://example.com/crypto-prices",
+            
+            config=config
+        )
+
+        if not result.success:
+            print("Crawl failed:", result.error_message)
+            return
+
+        # 5. Parse the extracted JSON
+        data = json.loads(result.extracted_content)
+        print(f"Extracted {len(data)} coin entries")
+        print(json.dumps(data[0], indent=2) if data else "No data found")
+
+asyncio.run(extract_crypto_prices())
+```
+
+**Highlights**:
+
+- **`baseSelector`**: Tells us where each "item" (crypto row) is.
+- **`fields`**: Two fields (`coin_name`, `price`) using simple CSS selectors.
+- Each field defines a **`type`** (e.g., `text`, `attribute`, `html`, `regex`, etc.).
+- Optional keys: **`transform`**, **`default`**, **`attribute`**, **`pattern`**, and **`source`** (for sibling data — see [Extracting Sibling Data](#sibling-data)).
+
+No LLM is needed, and the performance is **near-instant** for hundreds or thousands of items.
+
+---
+
+### **XPath Example with `raw://` HTML**
+
+Below is a short example demonstrating **XPath** extraction plus the **`raw://`** scheme. We'll pass a **dummy HTML** directly (no network request) and define the extraction strategy in `CrawlerRunConfig`.
+
+```python
+import json
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai import JsonXPathExtractionStrategy
+
+async def extract_crypto_prices_xpath():
+    # 1. Minimal dummy HTML with some repeating rows
+    dummy_html = """
+    
+      
+        

+          
Bitcoin

+          $28,000
+        

+        

+          
Ethereum

+          $1,800
+        

+      
+    
+    """
+
+    # 2. Define the JSON schema (XPath version)
+    schema = {
+        "name": "Crypto Prices via XPath",
+        "baseSelector": "//div[@class='crypto-row']",
+        "fields": [
+            {
+                "name": "coin_name",
+                "selector": ".//h2[@class='coin-name']",
+                "type": "text"
+            },
+            {
+                "name": "price",
+                "selector": ".//span[@class='coin-price']",
+                "type": "text"
+            }
+        ]
+    }
+
+    # 3. Place the strategy in the CrawlerRunConfig
+    config = CrawlerRunConfig(
+        extraction_strategy=JsonXPathExtractionStrategy(schema, verbose=True)
+    )
+
+    # 4. Use raw:// scheme to pass dummy_html directly
+    raw_url = f"raw://{dummy_html}"
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url=raw_url,
+            config=config
+        )
+
+        if not result.success:
+            print("Crawl failed:", result.error_message)
+            return
+
+        data = json.loads(result.extracted_content)
+        print(f"Extracted {len(data)} coin rows")
+        if data:
+            print("First item:", data[0])
+
+asyncio.run(extract_crypto_prices_xpath())
+```
+
+**Key Points**:
+
+1. **`JsonXPathExtractionStrategy`** is used instead of `JsonCssExtractionStrategy`.  
+2. **`baseSelector`** and each field's `"selector"` use **XPath** instead of CSS.  
+3. **`raw://`** lets us pass `dummy_html` with no real network request—handy for local testing.  
+4. Everything (including the extraction strategy) is in **`CrawlerRunConfig`**.  
+
+That's how you keep the config self-contained, illustrate **XPath** usage, and demonstrate the **raw** scheme for direct HTML input—all while avoiding the old approach of passing `extraction_strategy` directly to `arun()`.
+
+---
+
+## 3. Advanced Schema & Nested Structures
+
+Real sites often have **nested** or repeated data—like categories containing products, which themselves have a list of reviews or features. For that, we can define **nested** or **list** (and even **nested_list**) fields.
+
+### Sample E-Commerce HTML
+
+We have a **sample e-commerce** HTML file on GitHub (example):
+```
+https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/examples/sample_ecommerce.html
+```
+This snippet includes categories, products, features, reviews, and related items. Let's see how to define a schema that fully captures that structure **without LLM**.
+
+```python
+schema = {
+    "name": "E-commerce Product Catalog",
+    "baseSelector": "div.category",
+    # (1) We can define optional baseFields if we want to extract attributes 
+    # from the category container
+    "baseFields": [
+        {"name": "data_cat_id", "type": "attribute", "attribute": "data-cat-id"}, 
+    ],
+    "fields": [
+        {
+            "name": "category_name",
+            "selector": "h2.category-name",
+            "type": "text"
+        },
+        {
+            "name": "products",
+            "selector": "div.product",
+            "type": "nested_list",    # repeated sub-objects
+            "fields": [
+                {
+                    "name": "name",
+                    "selector": "h3.product-name",
+                    "type": "text"
+                },
+                {
+                    "name": "price",
+                    "selector": "p.product-price",
+                    "type": "text"
+                },
+                {
+                    "name": "details",
+                    "selector": "div.product-details",
+                    "type": "nested",  # single sub-object
+                    "fields": [
+                        {
+                            "name": "brand",
+                            "selector": "span.brand",
+                            "type": "text"
+                        },
+                        {
+                            "name": "model",
+                            "selector": "span.model",
+                            "type": "text"
+                        }
+                    ]
+                },
+                {
+                    "name": "features",
+                    "selector": "ul.product-features li",
+                    "type": "list",
+                    "fields": [
+                        {"name": "feature", "type": "text"} 
+                    ]
+                },
+                {
+                    "name": "reviews",
+                    "selector": "div.review",
+                    "type": "nested_list",
+                    "fields": [
+                        {
+                            "name": "reviewer", 
+                            "selector": "span.reviewer", 
+                            "type": "text"
+                        },
+                        {
+                            "name": "rating", 
+                            "selector": "span.rating", 
+                            "type": "text"
+                        },
+                        {
+                            "name": "comment", 
+                            "selector": "p.review-text", 
+                            "type": "text"
+                        }
+                    ]
+                },
+                {
+                    "name": "related_products",
+                    "selector": "ul.related-products li",
+                    "type": "list",
+                    "fields": [
+                        {
+                            "name": "name", 
+                            "selector": "span.related-name", 
+                            "type": "text"
+                        },
+                        {
+                            "name": "price", 
+                            "selector": "span.related-price", 
+                            "type": "text"
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}
+```
+
+Key Takeaways:
+
+- **Nested vs. List**:  
+  - **`type: "nested"`** means a **single** sub-object (like `details`).  
+  - **`type: "list"`** means multiple items that are **simple** dictionaries or single text fields.  
+  - **`type: "nested_list"`** means repeated **complex** objects (like `products` or `reviews`).
+- **Base Fields**: We can extract **attributes** from the container element via `"baseFields"`. For instance, `"data_cat_id"` might be `data-cat-id="elect123"`.  
+- **Transforms**: We can also define a `transform` if we want to lower/upper case, strip whitespace, or even run a custom function.
+
+### Running the Extraction
+
+```python
+import json
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai import JsonCssExtractionStrategy
+
+ecommerce_schema = {
+    # ... the advanced schema from above ...
+}
+
+async def extract_ecommerce_data():
+    strategy = JsonCssExtractionStrategy(ecommerce_schema, verbose=True)
+    
+    config = CrawlerRunConfig()
+    
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/examples/sample_ecommerce.html",
+            extraction_strategy=strategy,
+            config=config
+        )
+
+        if not result.success:
+            print("Crawl failed:", result.error_message)
+            return
+        
+        # Parse the JSON output
+        data = json.loads(result.extracted_content)
+        print(json.dumps(data, indent=2) if data else "No data found.")
+
+asyncio.run(extract_ecommerce_data())
+```
+
+If all goes well, you get a **structured** JSON array with each "category," containing an array of `products`. Each product includes `details`, `features`, `reviews`, etc. All of that **without** an LLM.
+
+---
+
+## 4. RegexExtractionStrategy - Fast Pattern-Based Extraction
+
+Crawl4AI now offers a powerful new zero-LLM extraction strategy: `RegexExtractionStrategy`. This strategy provides lightning-fast extraction of common data types like emails, phone numbers, URLs, dates, and more using pre-compiled regular expressions.
+
+### Key Features
+
+- **Zero LLM Dependency**: Extracts data without any AI model calls
+- **Blazing Fast**: Uses pre-compiled regex patterns for maximum performance
+- **Built-in Patterns**: Includes ready-to-use patterns for common data types
+- **Custom Patterns**: Add your own regex patterns for domain-specific extraction
+- **LLM-Assisted Pattern Generation**: Optionally use an LLM once to generate optimized patterns, then reuse them without further LLM calls
+
+### Simple Example: Extracting Common Entities
+
+The easiest way to start is by using the built-in pattern catalog:
+
+```python
+import json
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    RegexExtractionStrategy
+)
+
+async def extract_with_regex():
+    # Create a strategy using built-in patterns for URLs and currencies
+    strategy = RegexExtractionStrategy(
+        pattern = RegexExtractionStrategy.Url | RegexExtractionStrategy.Currency
+    )
+    
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=config
+        )
+        
+        if result.success:
+            data = json.loads(result.extracted_content)
+            for item in data[:5]:  # Show first 5 matches
+                print(f"{item['label']}: {item['value']}")
+            print(f"Total matches: {len(data)}")
+
+asyncio.run(extract_with_regex())
+```
+
+### Available Built-in Patterns
+
+`RegexExtractionStrategy` provides these common patterns as IntFlag attributes for easy combining:
+
+```python
+# Use individual patterns
+strategy = RegexExtractionStrategy(pattern=RegexExtractionStrategy.Email)
+
+# Combine multiple patterns
+strategy = RegexExtractionStrategy(
+    pattern = (
+        RegexExtractionStrategy.Email | 
+        RegexExtractionStrategy.PhoneUS | 
+        RegexExtractionStrategy.Url
+    )
+)
+
+# Use all available patterns
+strategy = RegexExtractionStrategy(pattern=RegexExtractionStrategy.All)
+```
+
+Available patterns include:
+- `Email` - Email addresses
+- `PhoneIntl` - International phone numbers
+- `PhoneUS` - US-format phone numbers
+- `Url` - HTTP/HTTPS URLs
+- `IPv4` - IPv4 addresses
+- `IPv6` - IPv6 addresses
+- `Uuid` - UUIDs
+- `Currency` - Currency values (USD, EUR, etc.)
+- `Percentage` - Percentage values
+- `Number` - Numeric values
+- `DateIso` - ISO format dates
+- `DateUS` - US format dates
+- `Time24h` - 24-hour format times
+- `PostalUS` - US postal codes
+- `PostalUK` - UK postal codes
+- `HexColor` - HTML hex color codes
+- `TwitterHandle` - Twitter handles
+- `Hashtag` - Hashtags
+- `MacAddr` - MAC addresses
+- `Iban` - International bank account numbers
+- `CreditCard` - Credit card numbers
+
+### Custom Pattern Example
+
+For more targeted extraction, you can provide custom patterns:
+
+```python
+import json
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    RegexExtractionStrategy
+)
+
+async def extract_prices():
+    # Define a custom pattern for US Dollar prices
+    price_pattern = {"usd_price": r"\$\s?\d{1,3}(?:,\d{3})*(?:\.\d{2})?"}
+    
+    # Create strategy with custom pattern
+    strategy = RegexExtractionStrategy(custom=price_pattern)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com/products",
+            config=config
+        )
+        
+        if result.success:
+            data = json.loads(result.extracted_content)
+            for item in data:
+                print(f"Found price: {item['value']}")
+
+asyncio.run(extract_prices())
+```
+
+### LLM-Assisted Pattern Generation
+
+For complex or site-specific patterns, you can use an LLM once to generate an optimized pattern, then save and reuse it without further LLM calls:
+
+```python
+import json
+import asyncio
+from pathlib import Path
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    RegexExtractionStrategy,
+    LLMConfig
+)
+
+async def extract_with_generated_pattern():
+    cache_dir = Path("./pattern_cache")
+    cache_dir.mkdir(exist_ok=True)
+    pattern_file = cache_dir / "price_pattern.json"
+    
+    # 1. Generate or load pattern
+    if pattern_file.exists():
+        pattern = json.load(pattern_file.open())
+        print(f"Using cached pattern: {pattern}")
+    else:
+        print("Generating pattern via LLM...")
+        
+        # Configure LLM
+        llm_config = LLMConfig(
+            provider="openai/gpt-4o-mini",
+            api_token="env:OPENAI_API_KEY",
+        )
+        
+        # Get sample HTML for context
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://example.com/products")
+            html = result.markdown.fit_html
+        
+        # Generate pattern (one-time LLM usage)
+        pattern = RegexExtractionStrategy.generate_pattern(
+            label="price",
+            html=html,
+            query="Product prices in USD format",
+            llm_config=llm_config,
+        )
+        
+        # Cache pattern for future use
+        json.dump(pattern, pattern_file.open("w"), indent=2)
+    
+    # 2. Use pattern for extraction (no LLM calls)
+    strategy = RegexExtractionStrategy(custom=pattern)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com/products",
+            config=config
+        )
+        
+        if result.success:
+            data = json.loads(result.extracted_content)
+            for item in data[:10]:
+                print(f"Extracted: {item['value']}")
+            print(f"Total matches: {len(data)}")
+
+asyncio.run(extract_with_generated_pattern())
+```
+
+This pattern allows you to:
+1. Use an LLM once to generate a highly optimized regex for your specific site
+2. Save the pattern to disk for reuse 
+3. Extract data using only regex (no further LLM calls) in production
+
+### Extraction Results Format
+
+The `RegexExtractionStrategy` returns results in a consistent format:
+
+```json
+[
+  {
+    "url": "https://example.com",
+    "label": "email",
+    "value": "contact@example.com",
+    "span": [145, 163]
+  },
+  {
+    "url": "https://example.com",
+    "label": "url",
+    "value": "https://support.example.com",
+    "span": [210, 235]
+  }
+]
+```
+
+Each match includes:
+- `url`: The source URL
+- `label`: The pattern name that matched (e.g., "email", "phone_us")
+- `value`: The extracted text
+- `span`: The start and end positions in the source content
+
+---
+
+## 5. Why "No LLM" Is Often Better
+
+1. **Zero Hallucination**: Pattern-based extraction doesn't guess text. It either finds it or not.  
+2. **Guaranteed Structure**: The same schema or regex yields consistent JSON across many pages, so your downstream pipeline can rely on stable keys.  
+3. **Speed**: LLM-based extraction can be 10–1000x slower for large-scale crawling.  
+4. **Scalable**: Adding or updating a field is a matter of adjusting the schema or regex, not re-tuning a model.
+
+**When might you consider an LLM?** Possibly if the site is extremely unstructured or you want AI summarization. But always try a schema or regex approach first for repeated or consistent data patterns.
+
+---
+
+## 6. Base Element Attributes & Additional Fields
+
+It's easy to **extract attributes** (like `href`, `src`, or `data-xxx`) from your base or nested elements using:
+
+```json
+{
+  "name": "href",
+  "type": "attribute",
+  "attribute": "href",
+  "default": null
+}
+```
+
+You can define them in **`baseFields`** (extracted from the main container element) or in each field's sub-lists. This is especially helpful if you need an item's link or ID stored in the parent `
`.
+
+---
+
+## 7. Putting It All Together: Larger Example
+
+Consider a blog site. We have a schema that extracts the **URL** from each post card (via `baseFields` with an `"attribute": "href"`), plus the title, date, summary, and author:
+
+```python
+schema = {
+  "name": "Blog Posts",
+  "baseSelector": "a.blog-post-card",
+  "baseFields": [
+    {"name": "post_url", "type": "attribute", "attribute": "href"}
+  ],
+  "fields": [
+    {"name": "title", "selector": "h2.post-title", "type": "text", "default": "No Title"},
+    {"name": "date", "selector": "time.post-date", "type": "text", "default": ""},
+    {"name": "summary", "selector": "p.post-summary", "type": "text", "default": ""},
+    {"name": "author", "selector": "span.post-author", "type": "text", "default": ""}
+  ]
+}
+```
+
+Then run with `JsonCssExtractionStrategy(schema)` to get an array of blog post objects, each with `"post_url"`, `"title"`, `"date"`, `"summary"`, `"author"`.
+
+---
+
+## 8. Extracting Sibling Data with `source` {#sibling-data}
+
+Some websites split a single logical item across **sibling elements** rather than nesting everything inside one container. A classic example is Hacker News, where each submission spans two adjacent `` rows:
+
+```html
+  
+  1.
+  Example Title
+
+                             
+  
+    100 points
+    johndoe
+  
+
+```
+
+Normally, field selectors only search **descendants** of the base element — siblings are unreachable. The `source` field key solves this by navigating to a sibling element before running the selector.
+
+### Syntax
+
+```
+"source": "+ "
+```
+
+- **`+ tr`** — next sibling ``
+- **`+ div.details`** — next sibling `
` with class `details`
+- **`+ .subtext`** — next sibling with class `subtext`
+
+### Example: Hacker News
+
+```python
+schema = {
+    "name": "HN Submissions",
+    "baseSelector": "tr.athing.submission",
+    "fields": [
+        {"name": "rank", "selector": "span.rank", "type": "text"},
+        {"name": "title", "selector": "span.titleline a", "type": "text"},
+        {"name": "url", "selector": "span.titleline a", "type": "attribute", "attribute": "href"},
+        {"name": "score", "selector": "span.score", "type": "text", "source": "+ tr"},
+        {"name": "author", "selector": "a.hnuser", "type": "text", "source": "+ tr"},
+    ],
+}
+
+strategy = JsonCssExtractionStrategy(schema)
+```
+
+The `score` and `author` fields first navigate to the next sibling ``, then run their selectors inside that element. Fields without `source` work as before — searching descendants of the base element.
+
+`source` works with all field types (`text`, `attribute`, `nested`, `list`, etc.) and with both `JsonCssExtractionStrategy` and `JsonXPathExtractionStrategy`. If the sibling isn't found, the field returns its `default` value.
+
+---
+
+## 9. Tips & Best Practices
+
+1. **Inspect the DOM** in Chrome DevTools or Firefox's Inspector to find stable selectors.  
+2. **Start Simple**: Verify you can extract a single field. Then add complexity like nested objects or lists.  
+3. **Test** your schema on partial HTML or a test page before a big crawl.  
+4. **Combine with JS Execution** if the site loads content dynamically. You can pass `js_code` or `wait_for` in `CrawlerRunConfig`.  
+5. **Look at Logs** when `verbose=True`: if your selectors are off or your schema is malformed, it'll often show warnings.  
+6. **Use baseFields** if you need attributes from the container element (e.g., `href`, `data-id`), especially for the "parent" item.  
+7. **Performance**: For large pages, make sure your selectors are as narrow as possible.
+8. **Consider Using Regex First**: For simple data types like emails, URLs, and dates, `RegexExtractionStrategy` is often the fastest approach.
+
+---
+
+## 10. Schema Generation Utility
+
+While manually crafting schemas is powerful and precise, Crawl4AI now offers a convenient utility to **automatically generate** extraction schemas using LLM. This is particularly useful when:
+
+1. You're dealing with a new website structure and want a quick starting point
+2. You need to extract complex nested data structures
+3. You want to avoid the learning curve of CSS/XPath selector syntax
+
+### Using the Schema Generator
+
+The schema generator is available as a static method on both `JsonCssExtractionStrategy` and `JsonXPathExtractionStrategy`. You can choose between OpenAI's GPT-4 or the open-source Ollama for schema generation:
+
+```python
+from crawl4ai import JsonCssExtractionStrategy, JsonXPathExtractionStrategy
+from crawl4ai import LLMConfig
+
+# Sample HTML with product information
+html = """
+

+    
Gaming Laptop

+    
$999.99

+    

+        
    
+            
  • 16GB RAM
    • 
+            
  • 1TB SSD
    • 
+        

+    

+

+"""
+
+# Option 1: Using OpenAI (requires API token)
+css_schema = JsonCssExtractionStrategy.generate_schema(
+    html,
+    schema_type="css",
+    llm_config = LLMConfig(provider="openai/gpt-4o",api_token="your-openai-token")
+)
+
+# Option 2: Using Ollama (open source, no token needed)
+xpath_schema = JsonXPathExtractionStrategy.generate_schema(
+    html,
+    schema_type="xpath",
+    llm_config = LLMConfig(provider="ollama/llama3.3", api_token=None)  # Not needed for Ollama
+)
+
+# Use the generated schema for fast, repeated extractions
+strategy = JsonCssExtractionStrategy(css_schema)
+```
+
+### Schema Validation
+
+By default, `generate_schema` **validates** the generated schema against the HTML to ensure that it actually extracts the data you expect. If the schema doesn't produce results, it automatically refines the selectors before returning.
+
+You can control this with the `validate` parameter:
+
+```python
+# Default: validated (recommended)
+schema = JsonCssExtractionStrategy.generate_schema(
+    url="https://news.ycombinator.com",
+    query="Extract each story: title, url, score, author",
+)
+
+# Skip validation if you want raw LLM output
+schema = JsonCssExtractionStrategy.generate_schema(
+    url="https://news.ycombinator.com",
+    query="Extract each story: title, url, score, author",
+    validate=False,
+)
+```
+
+The generator also understands sibling layouts — for sites like Hacker News where data is split across sibling elements, it will automatically use the [`source` field](#sibling-data) to reach sibling data.
+
+### Token Usage Tracking
+
+`generate_schema` may make multiple LLM calls internally (field inference, schema generation, validation retries). To track the total token consumption across all of these calls, pass a `TokenUsage` accumulator:
+
+```python
+from crawl4ai import JsonCssExtractionStrategy
+from crawl4ai.models import TokenUsage
+
+usage = TokenUsage()
+
+schema = JsonCssExtractionStrategy.generate_schema(
+    url="https://news.ycombinator.com",
+    query="Extract each story: title, url, score, author",
+    usage=usage,
+)
+
+print(f"Prompt tokens:     {usage.prompt_tokens}")
+print(f"Completion tokens: {usage.completion_tokens}")
+print(f"Total tokens:      {usage.total_tokens}")
+```
+
+The `usage` parameter is optional — omitting it changes nothing (fully backward-compatible). You can also reuse the same accumulator across multiple calls to get a grand total:
+
+```python
+usage = TokenUsage()
+schema1 = JsonCssExtractionStrategy.generate_schema(url=url1, query=q1, usage=usage)
+schema2 = JsonCssExtractionStrategy.generate_schema(url=url2, query=q2, usage=usage)
+print(f"Grand total: {usage.total_tokens} tokens")
+```
+
+Both `generate_schema` (sync) and `agenerate_schema` (async) support the `usage` parameter.
+
+### LLM Provider Options
+
+1. **OpenAI GPT-4 (`openai/gpt4o`)**
+   - Default provider
+   - Requires an API token
+   - Generally provides more accurate schemas
+   - Set via environment variable: `OPENAI_API_KEY`
+
+2. **Ollama (`ollama/llama3.3`)**
+   - Open source alternative
+   - No API token required
+   - Self-hosted option
+   - Good for development and testing
+
+### Benefits of Schema Generation
+
+1. **One-Time Cost**: While schema generation uses LLM, it's a one-time cost. The generated schema can be reused for unlimited extractions without further LLM calls.
+2. **Smart Pattern Recognition**: The LLM analyzes the HTML structure and identifies common patterns, often producing more robust selectors than manual attempts.
+3. **Automatic Nesting**: Complex nested structures are automatically detected and properly represented in the schema.
+4. **Learning Tool**: The generated schemas serve as excellent examples for learning how to write your own schemas.
+
+### Best Practices
+
+1. **Review Generated Schemas**: While the generator is smart, always review and test the generated schema before using it in production.
+2. **Provide Representative HTML**: The better your sample HTML represents the overall structure, the more accurate the generated schema will be.
+3. **Consider Both CSS and XPath**: Try both schema types and choose the one that works best for your specific case.
+4. **Cache Generated Schemas**: Since generation uses LLM, save successful schemas for reuse.
+5. **API Token Security**: Never hardcode API tokens. Use environment variables or secure configuration management.
+6. **Choose Provider Wisely**:
+   - Use OpenAI for production-quality schemas
+   - Use Ollama for development, testing, or when you need a self-hosted solution
+
+### Multi-Sample Schema Generation
+
+When scraping multiple pages with varying DOM structures (e.g., product pages where table rows appear in different positions), single-sample schema generation may produce **fragile selectors** like `tr:nth-child(6)` that break on other pages.
+
+**The Problem:**
+```
+Page A: Manufacturer is in row 6  → selector: tr:nth-child(6) td a
+Page B: Manufacturer is in row 5  → selector FAILS
+Page C: Manufacturer is in row 7  → selector FAILS
+```
+
+**The Solution:** Provide multiple HTML samples so the LLM identifies stable patterns that work across all pages.
+
+```python
+from crawl4ai import JsonCssExtractionStrategy, LLMConfig
+
+# Collect HTML samples from different pages
+html_sample_1 = """
+
+  
+  
+
BrandApple
ManufacturerApple Inc

+"""
+
+html_sample_2 = """
+
+  
+  
+
ManufacturerSamsung
BrandGalaxy

+"""
+
+html_sample_3 = """
+
+  
+  
+  
+
ModelPixel 8
BrandGoogle
ManufacturerGoogle LLC

+"""
+
+# Combine samples with labels
+combined_html = """
+## HTML Sample 1 (Product A):
+```html
+""" + html_sample_1 + """
+```
+
+## HTML Sample 2 (Product B):
+```html
+""" + html_sample_2 + """
+```
+
+## HTML Sample 3 (Product C):
+```html
+""" + html_sample_3 + """
+```
+"""
+
+# Provide instructions for stable selectors
+query = """
+IMPORTANT: I'm providing 3 HTML samples from different product pages.
+The manufacturer field appears in different row positions across pages.
+Generate selectors using stable attributes like href patterns (e.g., a[href*='/m/'])
+instead of fragile positional selectors like nth-child().
+Extract: manufacturer name and link.
+"""
+
+# Generate schema with multi-sample awareness
+schema = JsonCssExtractionStrategy.generate_schema(
+    html=combined_html,
+    query=query,
+    schema_type="CSS",
+    llm_config=LLMConfig(provider="openai/gpt-4o", api_token="your-token")
+)
+
+# The generated schema will use stable selectors like:
+# a[href*="/m/"] instead of tr:nth-child(6) td a
+print(schema)
+```
+
+**Key Points for Multi-Sample Queries:**
+
+1. **Format samples clearly** - Use markdown headers and code blocks to separate samples
+2. **State the number of samples** - "I'm providing 3 HTML samples..."
+3. **Explain the variation** - "...the manufacturer field appears in different row positions"
+4. **Request stable selectors** - "Use href patterns, data attributes, or class names instead of nth-child"
+
+**Stable vs Fragile Selectors:**
+
+| Fragile (single sample) | Stable (multi-sample) |
+|------------------------|----------------------|
+| `tr:nth-child(6) td a` | `a[href*="/m/"]` |
+| `div:nth-child(3) .price` | `.price, [data-price]` |
+| `ul li:first-child` | `li[data-featured="true"]` |
+
+This approach lets you generate schemas once that work reliably across hundreds of similar pages with varying structures.
+
+---
+
+## 11. Conclusion
+
+With Crawl4AI's LLM-free extraction strategies - `JsonCssExtractionStrategy`, `JsonXPathExtractionStrategy`, and now `RegexExtractionStrategy` - you can build powerful pipelines that:
+
+- Scrape any consistent site for structured data.  
+- Support nested objects, repeating lists, or pattern-based extraction.  
+- Scale to thousands of pages quickly and reliably.
+
+**Choosing the Right Strategy**:
+
+- Use **`RegexExtractionStrategy`** for fast extraction of common data types like emails, phones, URLs, dates, etc.
+- Use **`JsonCssExtractionStrategy`** or **`JsonXPathExtractionStrategy`** for structured data with clear HTML patterns
+- If you need both: first extract structured data with JSON strategies, then use regex on specific fields
+
+**Remember**: For repeated, structured data, you don't need to pay for or wait on an LLM. Well-crafted schemas and regex patterns get you the data faster, cleaner, and cheaper—**the real power** of Crawl4AI.
+
+**Last Updated**: 2025-05-02
+
+---
+
+That's it for **Extracting JSON (No LLM)**! You've seen how schema-based approaches (either CSS or XPath) and regex patterns can handle everything from simple lists to deeply nested product catalogs—instantly, with minimal overhead. Enjoy building robust scrapers that produce consistent, structured JSON for your data pipelines!
\ No newline at end of file
diff --git a/docs/md_v2/favicon.ico b/docs/md_v2/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..42e1dd25a2f77db21d3044b0e1476188d9a9e2f3
GIT binary patch
literal 3449
zcmb_dXH=707X1=%GHE2!eDZ^xoS;n)EIpgc3wTllJH((xX!I6p`LRs*NH=
zIs($9OA`{x#CdCG{>++RbJx1-p1sdm=lr;P?E?THfCQkX28ag^#Df7qiWtFQe`6O2
z0CW&JIQ(yHK?VRk*8m{F$UyT31q%g{y`in8W_b)be4^8n}Jl>_lnbNNHtRbL+v
zBF0bv6z2gDn-Ql001NQ~015WjIh@8A3~3E_D_&r+bZvRKOR
zlxZA+aKw&c#b@AvgF0qxQ6a9u9`Wi6RKb86g6O0Ql!I}!$DRCiIADwCh9}-XV3U*S
z-QR$2*Z>!^{vdvXIt%}t1VhGNk<9isi#g9EG#k8OwgzMAn3Ir{lxS(8=GE6ru16vYm$?3p|kO6QCbc!@iKoPed=TRc&LO
znoYD{PPz<7yf3<<-^zDv79kwxFVEZ3?8DaS=jhx4B?*eC-QE7iLjolkIudX?w_>0w
ztK?i-6`syCZ7$bcp2^!rir)Q7+t}dji(1~q*))rUq1IYqsf)eLhR^ehnS_U(TQSR9
zIdh5$VZP^$t1kWy%KoL?LvxuM0%fGvwzg`RY=7*?Qx)XXt-G!Qeb?fML|=X0I$U2(
zlUYvMB(|Y3@-Ht)v68aKy$XL7tzAvySPN@kI!o%ypvi@Rpnki78_PNq3C+Y{`zCDD5_qM~(mVsX-MZC;XDboJ
zQtVAj+z%i*-&7c_6bz|nscy}beQ#nW0LaVlQ>MZ%{3#T!_}Y?hES?2mOAvXP85Krm
z0(_+q5GfY;Q`Dx!L(&6-sqE<)7c1!%=S!({(0VoDj+xNdq!5!gUEZLruAxv
z8{&g&jH@q<2WSSo51$tb(>(jl+(vYu-`!%;-ECch7HQo1M``FA9eD{|^U{8onbkld
zE!YPlsr`{!TJp%GlP^Abj2?TJYRTNV9Cr-IXYcls@vUf+alx8L=j4NU{ucuuhnkGA
zg>BzgEc@v2oArO+wu_
zI>}_qx}T5B`rl+$CReDc)=_b^aGWkt^c@oasm^UkjD>wCyK;?XaMl+_*a;T>P--vW
zTgUoJFX3a=XUYk8%g4y}{bOmu4WW+WJ9$U%>dZbs9cL2ASu%n`WADkYaDP}e7h=$P
zgOEMo-`O+tFd4P$*nXb!(U$W30vE=yzb6&44WqVPw8e8=G8IQ#TO;V31!w#D$>9{|
z&16@#hD)~X6lQ-dTKl$wHjr5>2|_>lHbW7k;FFUYf1kMtPo(ba9fAEwjj?2JAil7uV#9FI^uFK-}mxt-TqlU|80GE
z)91rvrzu4r8{i2W-MFF1x3sghNPYT{A+CMN5qR7;^ZRT!FU2z#l_1okY6sB-lV19b$&Q6bpkkrB(19p`~72Xy;7UvqRLZo
z$bET)j3XX;>atJcca5ri$@?oMVZFB6umC*yk`zyfG=e($k~nzd32%cJV#AktwrRjD@X;31mL*
zkpK8#wGP;H&X}qB)a0DO*fo{NU{)wy+0tSacnL;oYk%QH+~h+l>@MfNu_BFu6e)n#
zhp*vq=-mhLroZcpWoN&=8v=bQ2GT00!S9DJ9(llXB~>{U!5Goliyb>SEeLg~?!Lo<
zZD!Wu?UG)A6h%Ij-5uUU;Q#@Xo^*uGl>S*EtE|Lq^@49*LFj^cJ<(h;3
zGTzsxfg3f!wV-evaew+;W#we`FBQm|B@aGP3lK)S@F7EGCzAPb*0yBMyEab3d`|W9
z%sXKS^REb73)e?XOk=IvMX+}--MV2zc`jWj0&KP}*TS`7V#4fIgfa5#vlbge1PQ8wvzIo>6wGea$@;8>H*g7?H5{CQrv_lgWG
z_my?Nnzxb8tH6keFMftP>Dqe0Vo1&0w}A)L
z%_k{7cr||2(SS<=74Vh$8^$^CB#Du^G7L4iEuou6-ai{$vB_FdCHZunRe#*ve420S
zfX3XH(_B~hxGmI*A#i~{SXY?$#_6YA
z*9O~N%6&+|<4|&Qul#BDV@r4aVgp@@drQ<##SzLAO72-&r4=t|qRjy-4iEyz>1~-uv#*XH*N(G6DT1D|+0hRr9H^6X3HzZ((bimnF##J9Q4J
z+w^U}QQr6#x%ecei-dQz*OwXV>G|I}oBSBBH0!)Dm$!yhx4u@m+C
zpmqaeUsX0}p4h|SxGlb>|gG&m7#tlXi#m8ak|D;$6Hf
zE|o2E&dQ6o3Ktb{3KjF`-v4TJDJ2a7Q{15)dU6BpqiWLuX>KPwm=bSfg#BQ3JU12>}FcPzs4AiaugUcoPT&g$FMBfPuuQTw96=z1kLO0f{Z;
zl3wJryPprUr)STeJ*7;t+1=TH{_mS_{`u!$p|vJ5O(3|@^NnNJJ#Ab7h8z>rPb}k*
z`R0L56nM@d@HEgMhvQXX94(i?xEzmTTF=nJh5dLtgR~z@j-WtcpsCF7%Gf(uKq)=o
z{(Av%nITNqS8xj^AS4ip0*(Waz`&sC%bB%5yglGTm@x!kJ3u&M`Uv=C@DXqa52xX5
z-wyldv#?K`0@ths`ul-U2xD&pVQl*Q!FS$;ec~kSGiSk?8X%o^mCSA7+*i7s1mxbm
zaOTbfQzg;eBfF2gtm0<
zbc6&VVW58im^cy6p~K({C4gf?ZrciZ^eC820wW{7skq=PSHk)F8$e2kjt+!YzXkU6
z0P#35JnY7UM@4OEj$Uz#IwYw9Y9XjYA?lJ67M;65)peVy^CtE3<+K3mWRlvl%(!Hd
z7Qonw6;z!!sk*vZbp8T$NlDfpKrJP8JnrI|u~`L#1fDzr=g$YKs(|iph*Kv`FqM_T
zK6nV2Fab!X5&H0b;Ph!AnKVNmJpz|5h5hYbAQAz3{zYi*8lbruNF=~F-UOb14mfuX
z;{17_NSOf7Q6L_tjz+0x&ZN5DNHs9T*v?(F!Xa7$)RmP~*RE6D>ZWRKqpq%|28
{Q2l+2>e`>wsg&uDgjw{}ZmNMHs>WvO@^XVQs)A;gfY#K-#YU>qQmVRos=kMe
zZU2;(l*ZwlIaH0!hD38Kb!8=Wc{$aMCaPOq24wDBV=E0qf
zyX>AA$gF3=7|xnyzE@7Tu*iBvUzT6r(V2x)GoGl8>78_
zRLw2abLJYK$wUsgOTc122@_O$;XUtw}b+sF`2_KHR!ugrZ(^Qd3OaS0`vfSgxZVp@{nCBZ!)Kyj4CD`&2Rc{|vODpxfd07>7
z-V%5^62-=_qQZdm-e>HS8d^fol7c#wqH6fVta(ctbyXGh?AcT;Z6;4#y+&P8;o3{a
zcGNO@{{dCQRq7ct#tK41e|_(2~}&G
z*&X#)Xic9^OA1=y5M!TzLDfHKlt`thqY;BuC=p@dLa_WLh`V(I~{7yO3*F1MM9UmIY_oGH}u)h`Kt6ix>Tg
z$g>OL7PS;6!DLeL)Tvaz{Z7?=o2tE&vDcOrOvU>36;vHJsk&~n=$Buqr%a&^hpB}(
z6jhG3jBzva*4bN5KEL;g)2|E$gCDQ`#|K0hfT^k;^HQ
z#iT3%9z?)XlsXyL{)#yJBC~_McjTr}$6JDVfr6ehsoO(1FCBCADo~b>=Z7PxLJs=w
zmp#{B7W`|>QDrPbegFpa6}Fd!K4c3=|N8oEgT5Df_uG#WC^W-o`4xT|nc$`RAG*CS
UCSNaJPyhe`07*qoM6N<$f)VKA_5c6?

literal 0
HcmV?d00001

diff --git a/docs/md_v2/img/favicon-x-32x32.png b/docs/md_v2/img/favicon-x-32x32.png
new file mode 100644
index 0000000000000000000000000000000000000000..437675b1c13ddaa9a4102929d5f12a4bef22a3e1
GIT binary patch
literal 1484
zcmV;-1vC1IP)ASSAaNzC2#nOBgZg{#2iLB#
z*+fx5#8j8Hnk+31UU;EDH-{oeQDA&Lym~cy?|n8JW>16&A;U@H5F?aAo?~KSZT=hm
z_!C%O#?X*FcrdhgpWMEUkr5OH0E8hr9ZpZL&40_$QLL?@*;If0F|>Cd78X#eS!@Y_
zGMo(CzXpIHK)1`Osq}~M_4v4~uCm!w_wF_J?2&)&v(XR$Y09lz(;vRqk31rs4wFPJ
zE;jb;kvo5LcvuDlpD_XDd&-t_FyP3D-1$46nUOr_&=5K+dTOe7;R2EbMS-F~lJw4>
z*W0$q$_g6|DT?^mG5PZ^jEp$%fS4YFDyiX_UKAW{s^5N(j~_=ZLR6FX(%|W*d*?ov
zeqZ|i-nq{QPdz2=HfuG+as1lr>X%P?$47?`f63s5rC@4)lbR>TApk#4v5+1dQS0UHA_-fQ#U
z#mA26BoUINEg?%;HPTkHsG7TNG!jaIlq^dxT`G3%LU#oKlHu_B_4wdH+hSU;gTQ+e
zz_Cb5?{FIG!PJ`dIvO#konDb)h#;Uxq&z?>QUQQeE;uW}-8&*Ak!8KJpB6iJqT7WC
zL}9f%*!f)Vv$K+AAgvDpp-v||aUz>JDydJpAd$#w)KfiU>%S^)6m4UV0I&
z7NR<8VdLPz;Af|}6-E-UjppDdgFreRy<S}!Wh`M{1lUpFdhbPmoujt1fL$42|JW%uyS=s=S=h^Ag#k0@IgEnd*$olL_
zb@L`SZIYt!c`*#7)6q{(_fMaO2=bhpHwQOv)Zdx22coA-rIQV<6aZv7l!5?70g`b;
z^kCg+1W*OiO93DZ0brn6kL>vj_=(K~sMf`eKvpjV0%%QNq^ck&Gc!2lVdda*i*np$
z+2l9o73QY5_(_J%3RO|#k_YLynrV~5dxQdG#
zj_m%c^2f?j5jT^X8D}bLCglzVfGTI#|I1woh8fF
m@bkl$8y<-Y^p!(J0N_8LhVS`>Jsr#d0000=%GHE2!eDZ^xoS;n)EIpgc3wTllJH((xX!I6p`LRs*NH=
zIs($9OA`{x#CdCG{>++RbJx1-p1sdm=lr;P?E?THfCQkX28ag^#Df7qiWtFQe`6O2
z0CW&JIQ(yHK?VRk*8m{F$UyT31q%g{y`in8W_b)be4^8n}Jl>_lnbNNHtRbL+v
zBF0bv6z2gDn-Ql001NQ~015WjIh@8A3~3E_D_&r+bZvRKOR
zlxZA+aKw&c#b@AvgF0qxQ6a9u9`Wi6RKb86g6O0Ql!I}!$DRCiIADwCh9}-XV3U*S
z-QR$2*Z>!^{vdvXIt%}t1VhGNk<9isi#g9EG#k8OwgzMAn3Ir{lxS(8=GE6ru16vYm$?3p|kO6QCbc!@iKoPed=TRc&LO
znoYD{PPz<7yf3<<-^zDv79kwxFVEZ3?8DaS=jhx4B?*eC-QE7iLjolkIudX?w_>0w
ztK?i-6`syCZ7$bcp2^!rir)Q7+t}dji(1~q*))rUq1IYqsf)eLhR^ehnS_U(TQSR9
zIdh5$VZP^$t1kWy%KoL?LvxuM0%fGvwzg`RY=7*?Qx)XXt-G!Qeb?fML|=X0I$U2(
zlUYvMB(|Y3@-Ht)v68aKy$XL7tzAvySPN@kI!o%ypvi@Rpnki78_PNq3C+Y{`zCDD5_qM~(mVsX-MZC;XDboJ
zQtVAj+z%i*-&7c_6bz|nscy}beQ#nW0LaVlQ>MZ%{3#T!_}Y?hES?2mOAvXP85Krm
z0(_+q5GfY;Q`Dx!L(&6-sqE<)7c1!%=S!({(0VoDj+xNdq!5!gUEZLruAxv
z8{&g&jH@q<2WSSo51$tb(>(jl+(vYu-`!%;-ECch7HQo1M``FA9eD{|^U{8onbkld
zE!YPlsr`{!TJp%GlP^Abj2?TJYRTNV9Cr-IXYcls@vUf+alx8L=j4NU{ucuuhnkGA
zg>BzgEc@v2oArO+wu_
zI>}_qx}T5B`rl+$CReDc)=_b^aGWkt^c@oasm^UkjD>wCyK;?XaMl+_*a;T>P--vW
zTgUoJFX3a=XUYk8%g4y}{bOmu4WW+WJ9$U%>dZbs9cL2ASu%n`WADkYaDP}e7h=$P
zgOEMo-`O+tFd4P$*nXb!(U$W30vE=yzb6&44WqVPw8e8=G8IQ#TO;V31!w#D$>9{|
z&16@#hD)~X6lQ-dTKl$wHjr5>2|_>lHbW7k;FFUYf1kMtPo(ba9fAEwjj?2JAil7uV#9FI^uFK-}mxt-TqlU|80GE
z)91rvrzu4r8{i2W-MFF1x3sghNPYT{A+CMN5qR7;^ZRT!FU2z#l_1okY6sB-lV19b$&Q6bpkkrB(19p`~72Xy;7UvqRLZo
z$bET)j3XX;>atJcca5ri$@?oMVZFB6umC*yk`zyfG=e($k~nzd32%cJV#AktwrRjD@X;31mL*
zkpK8#wGP;H&X}qB)a0DO*fo{NU{)wy+0tSacnL;oYk%QH+~h+l>@MfNu_BFu6e)n#
zhp*vq=-mhLroZcpWoN&=8v=bQ2GT00!S9DJ9(llXB~>{U!5Goliyb>SEeLg~?!Lo<
zZD!Wu?UG)A6h%Ij-5uUU;Q#@Xo^*uGl>S*EtE|Lq^@49*LFj^cJ<(h;3
zGTzsxfg3f!wV-evaew+;W#we`FBQm|B@aGP3lK)S@F7EGCzAPb*0yBMyEab3d`|W9
z%sXKS^REb73)e?XOk=IvMX+}--MV2zc`jWj0&KP}*TS`7V#4fIgfa5#vlbge1PQ8wvzIo>6wGea$@;8>H*g7?H5{CQrv_lgWG
z_my?Nnzxb8tH6keFMftP>Dqe0Vo1&0w}A)L
z%_k{7cr||2(SS<=74Vh$8^$^CB#Du^G7L4iEuou6-ai{$vB_FdCHZunRe#*ve420S
zfX3XH(_B~hxGmI*A#i~{SXY?$#_6YA
z*9O~N%6&+|<4|&Qul#BDV@r4aVgp@@drQ<##SzLAO72-&r4=t|qRjy-4iEyz>1~-uv#*XH*N(G6DT1D|+0hRr9H^6X3HzZ((bimnF##J9Q4J
z+w^U}QQr6#x%ecei-dQz*OwXV>G|I}oBSBBH0!)Dm$!yhx4u@m+C
zpmqaeUsX0}p4h|SxGlb>|gG&m7#tlXi#m8ak|D;$6Hf
zE|o2E&dQ6o3Ktb{3KjF`-v4TJDJ2a7Q{15)dU6BpqiWLuX>KPwm=bSfg#BQ3
+
+  

+    
+      unclecode%2Fcrawl4ai | Trendshift
+    
+
+  

+
+  

+    
+      GitHub Stars
+    
+    
+      GitHub Forks
+    
+    
+      PyPI version
+    
+  

+
+  

+    
+      Python Version
+    
+    
+      Downloads
+    
+    
+      License
+    
+  

+  

+    
+      Follow on X
+    
+    
+      Follow on LinkedIn
+    
+    
+      Join our Discord
+    
+  

+  
+

+
+---
+#### 🚀 Crawl4AI Cloud API — Closed Beta (Launching Soon)
+Reliable, large-scale web extraction, now built to be _**drastically more cost-effective**_ than any of the existing solutions.
+
+👉 **Apply [here](https://forms.gle/E9MyPaNXACnAMaqG7) for early access**  
+_We’ll be onboarding in phases and working closely with early users.
+Limited slots._
+
+---
+
+Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for large language models, AI agents, and data pipelines. Fully open source, flexible, and built for real-time performance, **Crawl4AI** empowers developers with unmatched speed, precision, and deployment ease.
+
+> Enjoy using Crawl4AI? Consider **[becoming a sponsor](https://github.com/sponsors/unclecode)** to support ongoing development and community growth!
+
+## 🆕 AI Assistant Skill Now Available!
+
+

+  
🤖 Crawl4AI Skill for Claude & AI Assistants

+  
Supercharge your AI coding assistant with complete Crawl4AI knowledge! Download our comprehensive skill package that includes:

+  
    
+    
  • 📚 Complete SDK reference (23K+ words)
    • 
+    
  • 🚀 Ready-to-use extraction scripts
    • 
+    
  • ⚡ Schema generation for efficient scraping
    • 
+    
  • 🔧 Version 0.7.4 compatible
    • 
+  

+  
+  

+    Works with Claude, Cursor, Windsurf, and other AI coding assistants. Import the .zip file into your AI assistant's skill/knowledge system.
+  

+

+
+## 🎯 New: Adaptive Web Crawling
+
+Crawl4AI now features intelligent adaptive crawling that knows when to stop! Using advanced information foraging algorithms, it determines when sufficient information has been gathered to answer your query.
+
+[Learn more about Adaptive Crawling →](core/adaptive-crawling.md)
+
+
+## Quick Start
+
+Here's a quick example to show you how easy it is to use Crawl4AI with its asynchronous capabilities:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+    # Create an instance of AsyncWebCrawler
+    async with AsyncWebCrawler() as crawler:
+        # Run the crawler on a URL
+        result = await crawler.arun(url="https://crawl4ai.com")
+
+        # Print the extracted content
+        print(result.markdown)
+
+# Run the async main function
+asyncio.run(main())
+```
+
+---
+
+## Video Tutorial
+
+

+  
+

+
+---
+
+## What Does Crawl4AI Do?
+
+Crawl4AI is a feature-rich crawler and scraper that aims to:
+
+1. **Generate Clean Markdown**: Perfect for RAG pipelines or direct ingestion into LLMs.  
+2. **Structured Extraction**: Parse repeated patterns with CSS, XPath, or LLM-based extraction.  
+3. **Advanced Browser Control**: Hooks, proxies, stealth modes, session re-use—fine-grained control.  
+4. **High Performance**: Parallel crawling, chunk-based extraction, real-time use cases.  
+5. **Open Source**: No forced API keys, no paywalls—everyone can access their data.  
+
+**Core Philosophies**:
+- **Democratize Data**: Free to use, transparent, and highly configurable.  
+- **LLM Friendly**: Minimally processed, well-structured text, images, and metadata, so AI models can easily consume it.
+
+---
+
+## Documentation Structure
+
+To help you get started, we’ve organized our docs into clear sections:
+
+- **Setup & Installation**  
+  Basic instructions to install Crawl4AI via pip or Docker.  
+- **Quick Start**  
+  A hands-on introduction showing how to do your first crawl, generate Markdown, and do a simple extraction.  
+- **Core**  
+  Deeper guides on single-page crawling, advanced browser/crawler parameters, content filtering, and caching.  
+- **Advanced**  
+  Explore link & media handling, lazy loading, hooking & authentication, proxies, session management, and more.  
+- **Extraction**  
+  Detailed references for no-LLM (CSS, XPath) vs. LLM-based strategies, chunking, and clustering approaches.  
+- **API Reference**  
+  Find the technical specifics of each class and method, including `AsyncWebCrawler`, `arun()`, and `CrawlResult`.
+
+Throughout these sections, you’ll find code samples you can **copy-paste** into your environment. If something is missing or unclear, raise an issue or PR.
+
+---
+
+## How You Can Support
+
+- **Star & Fork**: If you find Crawl4AI helpful, star the repo on GitHub or fork it to add your own features.  
+- **File Issues**: Encounter a bug or missing feature? Let us know by filing an issue, so we can improve.  
+- **Pull Requests**: Whether it’s a small fix, a big feature, or better docs—contributions are always welcome.  
+- **Join Discord**: Come chat about web scraping, crawling tips, or AI workflows with the community.  
+- **Spread the Word**: Mention Crawl4AI in your blog posts, talks, or on social media.  
+
+**Our mission**: to empower everyone—students, researchers, entrepreneurs, data scientists—to access, parse, and shape the world’s data with speed, cost-efficiency, and creative freedom.
+
+---
+
+## Quick Links
+
+- **[GitHub Repo](https://github.com/unclecode/crawl4ai)**  
+- **[Installation Guide](./core/installation.md)**  
+- **[Quick Start](./core/quickstart.md)**  
+- **[API Reference](./api/async-webcrawler.md)**  
+- **[Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)**  
+
+Thank you for joining me on this journey. Let’s keep building an **open, democratic** approach to data extraction and AI together.
+
+Happy Crawling!  
+— *Unclecode, Founder & Maintainer of Crawl4AI*  
diff --git a/docs/md_v2/marketplace/README.md b/docs/md_v2/marketplace/README.md
new file mode 100644
index 0000000..75e1b5c
--- /dev/null
+++ b/docs/md_v2/marketplace/README.md
@@ -0,0 +1,66 @@
+# Crawl4AI Marketplace
+
+A terminal-themed marketplace for tools, integrations, and resources related to Crawl4AI.
+
+## Setup
+
+### Backend
+
+1. Install dependencies:
+```bash
+cd backend
+pip install -r requirements.txt
+```
+
+2. Generate dummy data:
+```bash
+python dummy_data.py
+```
+
+3. Run the server:
+```bash
+python server.py
+```
+
+The API will be available at http://localhost:8100
+
+### Frontend
+
+1. Open `frontend/index.html` in your browser
+2. Or serve via MkDocs as part of the documentation site
+
+## Database Schema
+
+The marketplace uses SQLite with automatic migration from `schema.yaml`. Tables include:
+- **apps**: Tools and integrations
+- **articles**: Reviews, tutorials, and news
+- **categories**: App categories
+- **sponsors**: Sponsored content
+
+## API Endpoints
+
+- `GET /api/apps` - List apps with filters
+- `GET /api/articles` - List articles
+- `GET /api/categories` - Get all categories
+- `GET /api/sponsors` - Get active sponsors
+- `GET /api/search?q=query` - Search across content
+- `GET /api/stats` - Marketplace statistics
+
+## Features
+
+- **Smart caching**: LocalStorage with TTL (1 hour)
+- **Terminal theme**: Consistent with Crawl4AI branding
+- **Responsive design**: Works on all devices
+- **Fast search**: Debounced with 300ms delay
+- **CORS protected**: Only crawl4ai.com and localhost
+
+## Admin Panel
+
+Coming soon - for now, edit the database directly or modify `dummy_data.py`
+
+## Deployment
+
+For production deployment on EC2:
+1. Update `API_BASE` in `marketplace.js` to production URL
+2. Run FastAPI with proper production settings (use gunicorn/uvicorn)
+3. Set up nginx proxy if needed
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/admin/admin.css b/docs/md_v2/marketplace/admin/admin.css
new file mode 100644
index 0000000..66b975a
--- /dev/null
+++ b/docs/md_v2/marketplace/admin/admin.css
@@ -0,0 +1,759 @@
+/* Admin Dashboard - C4AI Terminal Style */
+
+/* Utility Classes */
+.hidden {
+    display: none !important;
+}
+
+/* Brand Colors */
+:root {
+    --c4ai-cyan: #50ffff;
+    --c4ai-green: #50ff50;
+    --c4ai-yellow: #ffff50;
+    --c4ai-pink: #ff50ff;
+    --c4ai-blue: #5050ff;
+}
+
+.admin-container {
+    min-height: 100vh;
+    background: var(--bg-dark);
+}
+
+/* Login Screen */
+.login-screen {
+    min-height: 100vh;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background: linear-gradient(135deg, #070708 0%, #1a1a2e 100%);
+}
+
+.login-box {
+    background: var(--bg-secondary);
+    border: 2px solid var(--primary-cyan);
+    padding: 3rem;
+    width: 400px;
+    box-shadow: 0 0 40px rgba(80, 255, 255, 0.2);
+    text-align: center;
+}
+
+.login-logo {
+    height: 60px;
+    margin-bottom: 2rem;
+    filter: brightness(1.2);
+}
+
+.login-box h1 {
+    color: var(--primary-cyan);
+    font-size: 1.5rem;
+    margin-bottom: 2rem;
+}
+
+#login-form input {
+    width: 100%;
+    padding: 0.75rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    font-family: inherit;
+    margin-bottom: 1rem;
+}
+
+#login-form input:focus {
+    outline: none;
+    border-color: var(--primary-cyan);
+}
+
+#login-form button {
+    width: 100%;
+    padding: 0.75rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    border: none;
+    color: var(--bg-dark);
+    font-weight: 600;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+#login-form button:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+    transform: translateY(-2px);
+}
+
+.error-msg {
+    color: var(--error);
+    font-size: 0.875rem;
+    margin-top: 1rem;
+}
+
+/* Admin Dashboard */
+.admin-dashboard.hidden {
+    display: none;
+}
+
+.admin-header {
+    background: var(--bg-secondary);
+    border-bottom: 2px solid var(--primary-cyan);
+    padding: 1rem 0;
+}
+
+.header-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.header-left {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.header-logo {
+    height: 35px;
+}
+
+.admin-header h1 {
+    font-size: 1.25rem;
+    color: var(--primary-cyan);
+}
+
+.header-right {
+    display: flex;
+    align-items: center;
+    gap: 2rem;
+}
+
+.admin-user {
+    color: var(--text-secondary);
+}
+
+.logout-btn {
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--error);
+    color: var(--error);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.logout-btn:hover {
+    background: rgba(255, 60, 116, 0.1);
+}
+
+/* Layout */
+.admin-layout {
+    display: flex;
+    max-width: 1800px;
+    margin: 0 auto;
+    min-height: calc(100vh - 60px);
+}
+
+/* Sidebar */
+.admin-sidebar {
+    width: 250px;
+    background: var(--bg-secondary);
+    border-right: 1px solid var(--border-color);
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
+.sidebar-nav {
+    padding: 1rem 0;
+}
+
+.nav-btn {
+    width: 100%;
+    padding: 1rem 1.5rem;
+    background: transparent;
+    border: none;
+    border-left: 3px solid transparent;
+    color: var(--text-secondary);
+    text-align: left;
+    cursor: pointer;
+    transition: all 0.2s;
+    display: flex;
+    align-items: center;
+    gap: 0.75rem;
+}
+
+.nav-btn:hover {
+    background: rgba(80, 255, 255, 0.05);
+    color: var(--primary-cyan);
+}
+
+.nav-btn.active {
+    border-left-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.1);
+    color: var(--primary-cyan);
+}
+
+.nav-icon {
+    font-size: 1.25rem;
+    margin-right: 0.25rem;
+    display: inline-block;
+    width: 1.5rem;
+    text-align: center;
+}
+
+.nav-btn[data-section="stats"] .nav-icon {
+    color: var(--c4ai-cyan);
+}
+
+.nav-btn[data-section="apps"] .nav-icon {
+    color: var(--c4ai-green);
+}
+
+.nav-btn[data-section="articles"] .nav-icon {
+    color: var(--c4ai-yellow);
+}
+
+.nav-btn[data-section="categories"] .nav-icon {
+    color: var(--c4ai-pink);
+}
+
+.nav-btn[data-section="sponsors"] .nav-icon {
+    color: var(--c4ai-blue);
+}
+
+.sidebar-actions {
+    padding: 1rem;
+    border-top: 1px solid var(--border-color);
+}
+
+.action-btn {
+    width: 100%;
+    padding: 0.75rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+    margin-bottom: 0.5rem;
+    transition: all 0.2s;
+}
+
+.action-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Main Content */
+.admin-main {
+    flex: 1;
+    padding: 2rem;
+    overflow-y: auto;
+}
+
+.content-section {
+    display: none;
+}
+
+.content-section.active {
+    display: block;
+}
+
+/* Stats Grid */
+.stats-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1.5rem;
+    margin-bottom: 3rem;
+}
+
+.stat-card {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02));
+    border: 1px solid rgba(80, 255, 255, 0.3);
+    padding: 1.5rem;
+    display: flex;
+    gap: 1.5rem;
+}
+
+.stat-icon {
+    font-size: 2rem;
+    width: 3rem;
+    height: 3rem;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    border: 2px solid;
+    border-radius: 4px;
+}
+
+.stat-card:nth-child(1) .stat-icon {
+    color: var(--c4ai-cyan);
+    border-color: var(--c4ai-cyan);
+}
+
+.stat-card:nth-child(2) .stat-icon {
+    color: var(--c4ai-green);
+    border-color: var(--c4ai-green);
+}
+
+.stat-card:nth-child(3) .stat-icon {
+    color: var(--c4ai-yellow);
+    border-color: var(--c4ai-yellow);
+}
+
+.stat-card:nth-child(4) .stat-icon {
+    color: var(--c4ai-pink);
+    border-color: var(--c4ai-pink);
+}
+
+.stat-number {
+    font-size: 2rem;
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.stat-label {
+    color: var(--text-secondary);
+}
+
+.stat-detail {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+    margin-top: 0.5rem;
+}
+
+/* Quick Actions */
+.quick-actions {
+    display: flex;
+    gap: 1rem;
+}
+
+.quick-btn {
+    padding: 0.75rem 1.5rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.quick-btn:hover {
+    background: rgba(80, 255, 255, 0.1);
+    transform: translateY(-2px);
+}
+
+/* Section Headers */
+.section-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 2rem;
+}
+
+.section-header h2 {
+    font-size: 1.5rem;
+    color: var(--text-primary);
+}
+
+.header-actions {
+    display: flex;
+    gap: 1rem;
+}
+
+.search-input {
+    padding: 0.5rem 1rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    width: 250px;
+}
+
+.search-input:focus {
+    outline: none;
+    border-color: var(--primary-cyan);
+}
+
+.filter-select {
+    padding: 0.5rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+}
+
+.add-btn {
+    padding: 0.5rem 1rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    border: none;
+    color: var(--bg-dark);
+    font-weight: 600;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.add-btn:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+    transform: translateY(-2px);
+}
+
+/* Data Tables */
+.data-table {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    overflow-x: auto;
+}
+
+.data-table table {
+    width: 100%;
+    border-collapse: collapse;
+}
+
+.data-table th {
+    background: var(--bg-tertiary);
+    padding: 1rem;
+    text-align: left;
+    color: var(--primary-cyan);
+    font-weight: 600;
+    border-bottom: 2px solid var(--border-color);
+    position: sticky;
+    top: 0;
+    z-index: 10;
+}
+
+.data-table td {
+    padding: 1rem;
+    border-bottom: 1px solid var(--border-color);
+}
+
+.data-table tr:hover {
+    background: rgba(80, 255, 255, 0.03);
+}
+
+/* Table Actions */
+.table-actions {
+    display: flex;
+    gap: 0.5rem;
+}
+
+.table-logo {
+    width: 48px;
+    height: 48px;
+    object-fit: contain;
+    border-radius: 6px;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 4px;
+}
+
+.btn-edit, .btn-delete, .btn-duplicate {
+    padding: 0.25rem 0.5rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+    font-size: 0.875rem;
+}
+
+.btn-edit:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+.btn-delete:hover {
+    border-color: var(--error);
+    color: var(--error);
+}
+
+.btn-duplicate:hover {
+    border-color: var(--accent-pink);
+    color: var(--accent-pink);
+}
+
+/* Badges in Tables */
+.badge {
+    padding: 0.25rem 0.5rem;
+    font-size: 0.75rem;
+    text-transform: uppercase;
+}
+
+.badge.featured {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+.badge.sponsored {
+    background: var(--warning);
+    color: var(--bg-dark);
+}
+
+.badge.active {
+    background: var(--success);
+    color: var(--bg-dark);
+}
+
+/* Modal Enhancements */
+.modal-content.large {
+    max-width: 1000px;
+    width: 90%;
+    max-height: 90vh;
+}
+
+.modal-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 1.5rem;
+    border-bottom: 1px solid var(--border-color);
+}
+
+.modal-body {
+    padding: 1.5rem;
+    overflow-y: auto;
+    max-height: calc(90vh - 140px);
+}
+
+.modal-footer {
+    display: flex;
+    justify-content: flex-end;
+    gap: 1rem;
+    padding: 1rem 1.5rem;
+    border-top: 1px solid var(--border-color);
+}
+
+.btn-cancel, .btn-save {
+    padding: 0.5rem 1.5rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.btn-cancel {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+}
+
+.btn-cancel:hover {
+    border-color: var(--error);
+    color: var(--error);
+}
+
+.btn-save {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    border: none;
+    color: var(--bg-dark);
+    font-weight: 600;
+}
+
+.btn-save:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+}
+
+/* Form Styles */
+.form-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+    gap: 1.5rem;
+}
+
+.form-group {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.form-group label {
+    color: var(--text-secondary);
+    font-size: 0.875rem;
+}
+
+.form-group input,
+.form-group select,
+.form-group textarea {
+    padding: 0.5rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    font-family: inherit;
+}
+
+.form-group input:focus,
+.form-group select:focus,
+.form-group textarea:focus {
+    outline: none;
+    border-color: var(--primary-cyan);
+}
+
+.form-group.full-width {
+    grid-column: 1 / -1;
+}
+
+.checkbox-group {
+    display: flex;
+    gap: 2rem;
+}
+
+.checkbox-label {
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    cursor: pointer;
+}
+
+.sponsor-form {
+    grid-template-columns: 200px repeat(2, minmax(220px, 1fr));
+    align-items: flex-start;
+    grid-auto-flow: dense;
+}
+
+.sponsor-logo-group {
+    grid-row: span 3;
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+}
+
+.span-two {
+    grid-column: span 2;
+}
+
+.logo-upload {
+    position: relative;
+    width: 180px;
+}
+
+.image-preview {
+    width: 180px;
+    height: 180px;
+    border: 1px dashed var(--border-color);
+    border-radius: 12px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background: var(--bg-tertiary);
+    overflow: hidden;
+}
+
+.image-preview.empty {
+    color: var(--text-secondary);
+    font-size: 0.75rem;
+    text-align: center;
+    padding: 0.75rem;
+}
+
+.image-preview img {
+    max-width: 100%;
+    max-height: 100%;
+    object-fit: contain;
+}
+
+.upload-btn {
+    position: absolute;
+    left: 50%;
+    bottom: 12px;
+    transform: translateX(-50%);
+    padding: 0.35rem 1rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    border: none;
+    border-radius: 999px;
+    color: var(--bg-dark);
+    font-size: 0.75rem;
+    font-weight: 600;
+    cursor: pointer;
+    box-shadow: 0 6px 18px rgba(80, 255, 255, 0.25);
+}
+
+.upload-btn:hover {
+    box-shadow: 0 8px 22px rgba(80, 255, 255, 0.35);
+}
+
+.logo-upload input[type="file"] {
+    display: none;
+}
+
+.upload-hint {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    margin: 0;
+}
+
+@media (max-width: 960px) {
+    .sponsor-form {
+        grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
+    }
+
+    .sponsor-logo-group {
+        grid-column: 1 / -1;
+        grid-row: auto;
+        flex-direction: row;
+        align-items: center;
+        gap: 1.5rem;
+    }
+
+    .logo-upload {
+        width: 160px;
+    }
+
+    .span-two {
+        grid-column: 1 / -1;
+    }
+}
+
+/* Rich Text Editor */
+.editor-toolbar {
+    display: flex;
+    gap: 0.5rem;
+    padding: 0.5rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    border-bottom: none;
+}
+
+.editor-btn {
+    padding: 0.25rem 0.5rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+}
+
+.editor-btn:hover {
+    background: rgba(80, 255, 255, 0.1);
+    border-color: var(--primary-cyan);
+}
+
+.editor-content {
+    min-height: 300px;
+    padding: 1rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    font-family: 'Dank Mono', Monaco, monospace;
+}
+
+/* Responsive */
+@media (max-width: 1024px) {
+    .admin-layout {
+        flex-direction: column;
+    }
+
+    .admin-sidebar {
+        width: 100%;
+        border-right: none;
+        border-bottom: 1px solid var(--border-color);
+    }
+
+    .sidebar-nav {
+        display: flex;
+        overflow-x: auto;
+        padding: 0;
+    }
+
+    .nav-btn {
+        border-left: none;
+        border-bottom: 3px solid transparent;
+        white-space: nowrap;
+    }
+
+    .nav-btn.active {
+        border-bottom-color: var(--primary-cyan);
+    }
+
+    .sidebar-actions {
+        display: none;
+    }
+}
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/admin/admin.js b/docs/md_v2/marketplace/admin/admin.js
new file mode 100644
index 0000000..7bdc3fc
--- /dev/null
+++ b/docs/md_v2/marketplace/admin/admin.js
@@ -0,0 +1,933 @@
+// Admin Dashboard - Smart & Powerful
+const { API_BASE, API_ORIGIN } = (() => {
+    const cleanOrigin = (value) => value ? value.replace(/\/$/, '') : '';
+    const params = new URLSearchParams(window.location.search);
+    const overrideParam = cleanOrigin(params.get('api_origin'));
+
+    let storedOverride = '';
+    try {
+        storedOverride = cleanOrigin(localStorage.getItem('marketplace_api_origin'));
+    } catch (error) {
+        storedOverride = '';
+    }
+
+    let origin = overrideParam || storedOverride;
+
+    if (overrideParam && overrideParam !== storedOverride) {
+        try {
+            localStorage.setItem('marketplace_api_origin', overrideParam);
+        } catch (error) {
+            // ignore storage errors (private mode, etc.)
+        }
+    }
+
+    const { protocol, hostname, port } = window.location;
+    const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname);
+
+    if (!origin && isLocalHost && port !== '8100') {
+        origin = `${protocol}//127.0.0.1:8100`;
+    }
+
+    if (origin) {
+        const normalized = cleanOrigin(origin);
+        return { API_BASE: `${normalized}/marketplace/api`, API_ORIGIN: normalized };
+    }
+
+    return { API_BASE: '/marketplace/api', API_ORIGIN: '' };
+})();
+
+const resolveAssetUrl = (path) => {
+    if (!path) return '';
+    if (/^https?:\/\//i.test(path)) return path;
+    if (path.startsWith('/') && API_ORIGIN) {
+        return `${API_ORIGIN}${path}`;
+    }
+    return path;
+};
+
+class AdminDashboard {
+    constructor() {
+        this.token = localStorage.getItem('admin_token');
+        this.currentSection = 'stats';
+        this.data = {
+            apps: [],
+            articles: [],
+            categories: [],
+            sponsors: []
+        };
+        this.editingItem = null;
+        this.init();
+    }
+
+    async init() {
+        // Check auth
+        if (!this.token) {
+            this.showLogin();
+            return;
+        }
+
+        // Try to load stats to verify token
+        try {
+            await this.loadStats();
+            this.showDashboard();
+            this.setupEventListeners();
+            await this.loadAllData();
+        } catch (error) {
+            if (error.status === 401) {
+                this.showLogin();
+            }
+        }
+    }
+
+    showLogin() {
+        document.getElementById('login-screen').classList.remove('hidden');
+        document.getElementById('admin-dashboard').classList.add('hidden');
+
+        // Set up login button click handler
+        const loginBtn = document.getElementById('login-btn');
+        if (loginBtn) {
+            loginBtn.onclick = async () => {
+                const password = document.getElementById('password').value;
+                await this.login(password);
+            };
+        }
+    }
+
+    async login(password) {
+        try {
+            const response = await fetch(`${API_BASE}/admin/login`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ password })
+            });
+
+            if (!response.ok) throw new Error('Invalid password');
+
+            const data = await response.json();
+            this.token = data.token;
+            localStorage.setItem('admin_token', this.token);
+
+            document.getElementById('login-screen').classList.add('hidden');
+            this.showDashboard();
+            this.setupEventListeners();
+            await this.loadAllData();
+        } catch (error) {
+            document.getElementById('login-error').textContent = 'Invalid password';
+            document.getElementById('password').value = '';
+        }
+    }
+
+    showDashboard() {
+        document.getElementById('login-screen').classList.add('hidden');
+        document.getElementById('admin-dashboard').classList.remove('hidden');
+    }
+
+    setupEventListeners() {
+        // Navigation
+        document.querySelectorAll('.nav-btn').forEach(btn => {
+            btn.onclick = () => this.switchSection(btn.dataset.section);
+        });
+
+        // Logout
+        document.getElementById('logout-btn').onclick = () => this.logout();
+
+        // Export/Backup
+        document.getElementById('export-btn').onclick = () => this.exportData();
+        document.getElementById('backup-btn').onclick = () => this.backupDatabase();
+
+        // Search
+        ['apps', 'articles'].forEach(type => {
+            const searchInput = document.getElementById(`${type}-search`);
+            if (searchInput) {
+                searchInput.oninput = (e) => this.filterTable(type, e.target.value);
+            }
+        });
+
+        // Category filter
+        const categoryFilter = document.getElementById('apps-filter');
+        if (categoryFilter) {
+            categoryFilter.onchange = (e) => this.filterByCategory(e.target.value);
+        }
+
+        // Save button in modal
+        document.getElementById('save-btn').onclick = () => this.saveItem();
+    }
+
+    async loadAllData() {
+        try {
+            await this.loadStats();
+        } catch (e) {
+            console.error('Failed to load stats:', e);
+        }
+
+        try {
+            await this.loadApps();
+        } catch (e) {
+            console.error('Failed to load apps:', e);
+        }
+
+        try {
+            await this.loadArticles();
+        } catch (e) {
+            console.error('Failed to load articles:', e);
+        }
+
+        try {
+            await this.loadCategories();
+        } catch (e) {
+            console.error('Failed to load categories:', e);
+        }
+
+        try {
+            await this.loadSponsors();
+        } catch (e) {
+            console.error('Failed to load sponsors:', e);
+        }
+
+        this.populateCategoryFilter();
+    }
+
+    async apiCall(endpoint, options = {}) {
+        const isFormData = options.body instanceof FormData;
+        const headers = {
+            'Authorization': `Bearer ${this.token}`,
+            ...options.headers
+        };
+
+        if (!isFormData && !headers['Content-Type']) {
+            headers['Content-Type'] = 'application/json';
+        }
+
+        const response = await fetch(`${API_BASE}${endpoint}`, {
+            ...options,
+            headers
+        });
+
+        if (response.status === 401) {
+            this.logout();
+            throw { status: 401 };
+        }
+
+        if (!response.ok) throw new Error(`API Error: ${response.status}`);
+        return response.json();
+    }
+
+    async loadStats() {
+        const stats = await this.apiCall(`/admin/stats?_=${Date.now()}`, {
+            cache: 'no-store'
+        });
+
+        document.getElementById('stat-apps').textContent = stats.apps.total;
+        document.getElementById('stat-featured').textContent = stats.apps.featured;
+        document.getElementById('stat-sponsored').textContent = stats.apps.sponsored;
+        document.getElementById('stat-articles').textContent = stats.articles;
+        document.getElementById('stat-sponsors').textContent = stats.sponsors.active;
+        document.getElementById('stat-views').textContent = this.formatNumber(stats.total_views);
+    }
+
+    async loadApps() {
+        this.data.apps = await this.apiCall(`/apps?limit=100&_=${Date.now()}`, {
+            cache: 'no-store'
+        });
+        this.renderAppsTable(this.data.apps);
+    }
+
+    async loadArticles() {
+        this.data.articles = await this.apiCall(`/articles?limit=100&_=${Date.now()}`, {
+            cache: 'no-store'
+        });
+        this.renderArticlesTable(this.data.articles);
+    }
+
+    async loadCategories() {
+        const cacheBuster = Date.now();
+        this.data.categories = await this.apiCall(`/categories?_=${cacheBuster}`, {
+            cache: 'no-store'
+        });
+        this.renderCategoriesTable(this.data.categories);
+    }
+
+    async loadSponsors() {
+        const cacheBuster = Date.now();
+        this.data.sponsors = await this.apiCall(`/sponsors?limit=100&_=${cacheBuster}`, {
+            cache: 'no-store'
+        });
+        this.renderSponsorsTable(this.data.sponsors);
+    }
+
+    renderAppsTable(apps) {
+        const table = document.getElementById('apps-table');
+        table.innerHTML = `
+            
+                
+                    
+                        
+                        
+                        
+                        
+                        
+                        
+                        
+                        
+                    
+                
+                
+                    ${apps.map(app => `
+                        
+                            
+                            
+                            
+                            
+                            
+                            
+                            
+                            
+                        
+                    `).join('')}
+                
+            
IDNameCategoryTypeRatingDownloadsStatusActions
${app.id}${app.name}${app.category}${app.type}◆ ${app.rating}/5${this.formatNumber(app.downloads)}
+                                ${app.featured ? 'Featured' : ''}
+                                ${app.sponsored ? '' : ''}
+                            
+                                

+                                    
+                                    
+                                    
+                                

+                            

+        `;
+    }
+
+    renderArticlesTable(articles) {
+        const table = document.getElementById('articles-table');
+        table.innerHTML = `
+            
+                
+                    
+                        
+                        
+                        
+                        
+                        
+                        
+                        
+                    
+                
+                
+                    ${articles.map(article => `
+                        
+                            
+                            
+                            
+                            
+                            
+                            
+                            
+                        
+                    `).join('')}
+                
+            
IDTitleCategoryAuthorPublishedViewsActions
${article.id}${article.title}${article.category}${article.author}${new Date(article.published_date).toLocaleDateString()}${this.formatNumber(article.views)}
+                                

+                                    
+                                    
+                                    
+                                

+                            

+        `;
+    }
+
+    renderCategoriesTable(categories) {
+        const table = document.getElementById('categories-table');
+        table.innerHTML = `
+            
+                
+                    
+                        
+                        
+                        
+                        
+                        
+                    
+                
+                
+                    ${categories.map(cat => `
+                        
+                            
+                            
+                            
+                            
+                            
+                        
+                    `).join('')}
+                
+            
OrderIconNameDescriptionActions
${cat.order_index}${cat.icon}${cat.name}${cat.description}
+                                

+                                    
+                                    
+                                

+                            

+        `;
+    }
+
+    renderSponsorsTable(sponsors) {
+        const table = document.getElementById('sponsors-table');
+        table.innerHTML = `
+            
+                
+                    
+                        
+                        
+                        
+                        
+                        
+                        
+                        
+                        
+                    
+                
+                
+                    ${sponsors.map(sponsor => `
+                        
+                            
+                            
+                            
+                            
+                            
+                            
+                            
+                            
+                        
+                    `).join('')}
+                
+            
IDLogoCompanyTierStartEndStatusActions
${sponsor.id}${sponsor.logo_url ? `` : '-'}${sponsor.company_name}${sponsor.tier}${new Date(sponsor.start_date).toLocaleDateString()}${new Date(sponsor.end_date).toLocaleDateString()}${sponsor.active ? 'Active' : 'Inactive'}
+                                

+                                    
+                                    
+                                

+                            

+        `;
+    }
+
+    showAddForm(type) {
+        this.editingItem = null;
+        this.showModal(type, null);
+    }
+
+    async editItem(type, id) {
+        const item = this.data[type].find(i => i.id === id);
+        if (item) {
+            this.editingItem = item;
+            this.showModal(type, item);
+        }
+    }
+
+    async duplicateItem(type, id) {
+        const item = this.data[type].find(i => i.id === id);
+        if (item) {
+            const newItem = { ...item };
+            delete newItem.id;
+            newItem.name = `${newItem.name || newItem.title} (Copy)`;
+            if (newItem.slug) newItem.slug = `${newItem.slug}-copy-${Date.now()}`;
+
+            this.editingItem = null;
+            this.showModal(type, newItem);
+        }
+    }
+
+    showModal(type, item) {
+        const modal = document.getElementById('form-modal');
+        const title = document.getElementById('modal-title');
+        const body = document.getElementById('modal-body');
+
+        title.textContent = item ? `Edit ${type.slice(0, -1)}` : `Add New ${type.slice(0, -1)}`;
+
+        if (type === 'apps') {
+            body.innerHTML = this.getAppForm(item);
+        } else if (type === 'articles') {
+            body.innerHTML = this.getArticleForm(item);
+        } else if (type === 'categories') {
+            body.innerHTML = this.getCategoryForm(item);
+        } else if (type === 'sponsors') {
+            body.innerHTML = this.getSponsorForm(item);
+        }
+
+        modal.classList.remove('hidden');
+        modal.dataset.type = type;
+
+        if (type === 'sponsors') {
+            this.setupLogoUploadHandlers();
+        }
+    }
+
+    getAppForm(app) {
+        return `
+            

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                    Markdown support: **bold**, *italic*, [links](url), # headers, code blocks, lists
+                

+                

+                    
+                    
+                    Single markdown field with installation, examples, and complete guide. Code blocks get auto copy buttons.
+                

+                

+                    
+                    
+                    Full documentation with API reference, examples, best practices, etc.
+                

+            

+        `;
+    }
+
+    getArticleForm(article) {
+        return `
+            

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+            

+        `;
+    }
+
+    getCategoryForm(category) {
+        return `
+            

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+                

+                    
+                    
+                

+            

+        `;
+    }
+
+    getSponsorForm(sponsor) {
+        const existingFile = sponsor?.logo_url ? sponsor.logo_url.split('/').pop().split('?')[0] : '';
+        return `
+            
+        `;
+    }
+
+    async saveItem() {
+        const modal = document.getElementById('form-modal');
+        const type = modal.dataset.type;
+
+        try {
+            if (type === 'sponsors') {
+                const fileInput = document.getElementById('form-logo-file');
+                if (fileInput && fileInput.files && fileInput.files[0]) {
+                    const formData = new FormData();
+                    formData.append('file', fileInput.files[0]);
+                    formData.append('folder', 'sponsors');
+
+                    const uploadResponse = await this.apiCall('/admin/upload-image', {
+                        method: 'POST',
+                        body: formData
+                    });
+
+                    if (!uploadResponse.url) {
+                        throw new Error('Image upload failed');
+                    }
+
+                    document.getElementById('form-logo-url').value = uploadResponse.url;
+                }
+            }
+
+            const data = this.collectFormData(type);
+
+            if (this.editingItem) {
+                await this.apiCall(`/admin/${type}/${this.editingItem.id}`, {
+                    method: 'PUT',
+                    body: JSON.stringify(data)
+                });
+            } else {
+                await this.apiCall(`/admin/${type}`, {
+                    method: 'POST',
+                    body: JSON.stringify(data)
+                });
+            }
+
+            this.closeModal();
+            await this[`load${type.charAt(0).toUpperCase() + type.slice(1)}`]();
+            await this.loadStats();
+        } catch (error) {
+            alert('Error saving item: ' + error.message);
+        }
+    }
+
+    collectFormData(type) {
+        const data = {};
+
+        if (type === 'apps') {
+            data.name = document.getElementById('form-name').value;
+            data.slug = document.getElementById('form-slug').value || this.generateSlug(data.name);
+            data.description = document.getElementById('form-description').value;
+            data.category = document.getElementById('form-category').value;
+            data.type = document.getElementById('form-type').value;
+            const rating = parseFloat(document.getElementById('form-rating').value);
+            const downloads = parseInt(document.getElementById('form-downloads').value, 10);
+            data.rating = Number.isFinite(rating) ? rating : 0;
+            data.downloads = Number.isFinite(downloads) ? downloads : 0;
+            data.image = document.getElementById('form-image').value;
+            data.website_url = document.getElementById('form-website').value;
+            data.github_url = document.getElementById('form-github').value;
+            data.pricing = document.getElementById('form-pricing').value;
+            data.contact_email = document.getElementById('form-email').value;
+            data.featured = document.getElementById('form-featured').checked ? 1 : 0;
+            data.sponsored = document.getElementById('form-sponsored').checked ? 1 : 0;
+            data.long_description = document.getElementById('form-long-description').value;
+            data.integration_guide = document.getElementById('form-integration').value;
+            data.documentation = document.getElementById('form-documentation').value;
+        } else if (type === 'articles') {
+            data.title = document.getElementById('form-title').value;
+            data.slug = this.generateSlug(data.title);
+            data.author = document.getElementById('form-author').value;
+            data.category = document.getElementById('form-category').value;
+            data.featured_image = document.getElementById('form-image').value;
+            data.content = document.getElementById('form-content').value;
+        } else if (type === 'categories') {
+            data.name = document.getElementById('form-name').value;
+            data.slug = this.generateSlug(data.name);
+            data.icon = document.getElementById('form-icon').value;
+            data.description = document.getElementById('form-description').value;
+            const orderIndex = parseInt(document.getElementById('form-order').value, 10);
+            data.order_index = Number.isFinite(orderIndex) ? orderIndex : 0;
+        } else if (type === 'sponsors') {
+            data.company_name = document.getElementById('form-name').value;
+            data.logo_url = document.getElementById('form-logo-url').value;
+            data.tier = document.getElementById('form-tier').value;
+            data.landing_url = document.getElementById('form-landing').value;
+            data.banner_url = document.getElementById('form-banner').value;
+            data.start_date = document.getElementById('form-start').value;
+            data.end_date = document.getElementById('form-end').value;
+            data.active = document.getElementById('form-active').checked ? 1 : 0;
+        }
+
+        return data;
+    }
+
+    setupLogoUploadHandlers() {
+        const fileInput = document.getElementById('form-logo-file');
+        const preview = document.getElementById('form-logo-preview');
+        const logoUrlInput = document.getElementById('form-logo-url');
+        const trigger = document.getElementById('form-logo-button');
+        const fileNameEl = document.getElementById('form-logo-filename');
+
+        if (!fileInput || !preview || !logoUrlInput) return;
+
+        const setFileName = (text) => {
+            if (fileNameEl) {
+                fileNameEl.textContent = text;
+            }
+        };
+
+        const setEmptyState = () => {
+            preview.innerHTML = 'No logo uploaded';
+            preview.classList.add('empty');
+            setFileName('No file selected');
+        };
+
+        const setExistingState = () => {
+            if (logoUrlInput.value) {
+                const existingFile = logoUrlInput.value.split('/').pop().split('?')[0];
+                preview.innerHTML = `Logo preview`;
+                preview.classList.remove('empty');
+                setFileName(existingFile ? `Current: ${existingFile}` : 'Current logo');
+            } else {
+                setEmptyState();
+            }
+        };
+
+        setExistingState();
+
+        if (trigger) {
+            trigger.onclick = () => fileInput.click();
+        }
+
+        fileInput.addEventListener('change', (event) => {
+            const file = event.target.files && event.target.files[0];
+
+            if (!file) {
+                setExistingState();
+                return;
+            }
+
+            setFileName(file.name);
+
+            const reader = new FileReader();
+            reader.onload = () => {
+                preview.innerHTML = `Logo preview`;
+                preview.classList.remove('empty');
+            };
+            reader.readAsDataURL(file);
+        });
+    }
+
+    async deleteItem(type, id) {
+        if (!confirm(`Are you sure you want to delete this ${type.slice(0, -1)}?`)) return;
+
+        try {
+            await this.apiCall(`/admin/${type}/${id}`, { method: 'DELETE' });
+            await this[`load${type.charAt(0).toUpperCase() + type.slice(1)}`]();
+            await this.loadStats();
+        } catch (error) {
+            alert('Error deleting item: ' + error.message);
+        }
+    }
+
+    async deleteCategory(id) {
+        const hasApps = this.data.apps.some(app =>
+            app.category === this.data.categories.find(c => c.id === id)?.name
+        );
+
+        if (hasApps) {
+            alert('Cannot delete category with existing apps');
+            return;
+        }
+
+        await this.deleteItem('categories', id);
+    }
+
+    closeModal() {
+        document.getElementById('form-modal').classList.add('hidden');
+        this.editingItem = null;
+    }
+
+    switchSection(section) {
+        // Update navigation
+        document.querySelectorAll('.nav-btn').forEach(btn => {
+            btn.classList.toggle('active', btn.dataset.section === section);
+        });
+
+        // Show section
+        document.querySelectorAll('.content-section').forEach(sec => {
+            sec.classList.remove('active');
+        });
+        document.getElementById(`${section}-section`).classList.add('active');
+
+        this.currentSection = section;
+    }
+
+    filterTable(type, query) {
+        const items = this.data[type].filter(item => {
+            const searchText = Object.values(item).join(' ').toLowerCase();
+            return searchText.includes(query.toLowerCase());
+        });
+
+        if (type === 'apps') {
+            this.renderAppsTable(items);
+        } else if (type === 'articles') {
+            this.renderArticlesTable(items);
+        }
+    }
+
+    filterByCategory(category) {
+        const apps = category
+            ? this.data.apps.filter(app => app.category === category)
+            : this.data.apps;
+        this.renderAppsTable(apps);
+    }
+
+    populateCategoryFilter() {
+        const filter = document.getElementById('apps-filter');
+        if (!filter) return;
+
+        filter.innerHTML = '';
+        this.data.categories.forEach(cat => {
+            filter.innerHTML += ``;
+        });
+    }
+
+    async exportData() {
+        const data = {
+            apps: this.data.apps,
+            articles: this.data.articles,
+            categories: this.data.categories,
+            sponsors: this.data.sponsors,
+            exported: new Date().toISOString()
+        };
+
+        const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
+        const url = URL.createObjectURL(blob);
+        const a = document.createElement('a');
+        a.href = url;
+        a.download = `marketplace-export-${Date.now()}.json`;
+        a.click();
+    }
+
+    async backupDatabase() {
+        // In production, this would download the SQLite file
+        alert('Database backup would be implemented on the server side');
+    }
+
+    generateSlug(text) {
+        return text.toLowerCase()
+            .replace(/[^\w\s-]/g, '')
+            .replace(/\s+/g, '-')
+            .replace(/-+/g, '-')
+            .trim();
+    }
+
+    formatNumber(num) {
+        if (num >= 1000000) return (num / 1000000).toFixed(1) + 'M';
+        if (num >= 1000) return (num / 1000).toFixed(1) + 'K';
+        return num.toString();
+    }
+
+    logout() {
+        localStorage.removeItem('admin_token');
+        this.token = null;
+        this.showLogin();
+    }
+}
+
+// Initialize
+const admin = new AdminDashboard();
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/admin/index.html b/docs/md_v2/marketplace/admin/index.html
new file mode 100644
index 0000000..0d30238
--- /dev/null
+++ b/docs/md_v2/marketplace/admin/index.html
@@ -0,0 +1,215 @@
+
+
+
+    
+    
+    Admin Dashboard - Crawl4AI Marketplace
+    
+    
+
+
+    

+        
+        
+
+        
+        
+
+        
+        
+    

+
+    
+
+
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/app-detail.css b/docs/md_v2/marketplace/app-detail.css
new file mode 100644
index 0000000..25a8cf6
--- /dev/null
+++ b/docs/md_v2/marketplace/app-detail.css
@@ -0,0 +1,683 @@
+/* App Detail Page Styles */
+
+.app-detail-container {
+    min-height: 100vh;
+    background: var(--bg-dark);
+}
+
+/* Back Button */
+.header-nav {
+    display: flex;
+    align-items: center;
+}
+
+.back-btn {
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+    font-size: 0.875rem;
+}
+
+.back-btn:hover {
+    border-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.1);
+}
+
+/* App Hero Section */
+.app-hero {
+    max-width: 1800px;
+    margin: 2rem auto;
+    padding: 0 2rem;
+}
+
+.app-hero-content {
+    display: grid;
+    grid-template-columns: 1fr 2fr;
+    gap: 3rem;
+    background: linear-gradient(135deg, #1a1a2e, #0f0f1e);
+    border: 2px solid var(--primary-cyan);
+    padding: 2rem;
+    box-shadow: 0 0 30px rgba(80, 255, 255, 0.15),
+                inset 0 0 20px rgba(80, 255, 255, 0.05);
+}
+
+.app-hero-image {
+    width: 100%;
+    height: 300px;
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05));
+    background-size: cover;
+    background-position: center;
+    border: 1px solid var(--border-color);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 4rem;
+    color: var(--primary-cyan);
+}
+
+.app-badges {
+    display: flex;
+    gap: 0.5rem;
+    margin-bottom: 1rem;
+}
+
+.app-badge {
+    padding: 0.3rem 0.6rem;
+    background: var(--bg-tertiary);
+    color: var(--text-secondary);
+    font-size: 0.75rem;
+    text-transform: uppercase;
+    font-weight: 600;
+}
+
+.app-badge.featured {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3);
+}
+
+.app-badge.sponsored {
+    background: linear-gradient(135deg, var(--warning), #ff8c00);
+    color: var(--bg-dark);
+    box-shadow: 0 2px 10px rgba(245, 158, 11, 0.3);
+}
+
+.app-hero-info h1 {
+    font-size: 2.5rem;
+    color: var(--primary-cyan);
+    margin: 0.5rem 0;
+    text-shadow: 0 0 20px rgba(80, 255, 255, 0.5);
+}
+
+.app-tagline {
+    font-size: 1.1rem;
+    color: var(--text-secondary);
+    margin-bottom: 2rem;
+}
+
+/* Stats */
+.app-stats {
+    display: flex;
+    gap: 2rem;
+    margin: 2rem 0;
+    padding: 1rem 0;
+    border-top: 1px solid var(--border-color);
+    border-bottom: 1px solid var(--border-color);
+}
+
+.stat {
+    display: flex;
+    flex-direction: column;
+    gap: 0.25rem;
+}
+
+.stat-value {
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.stat-label {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+}
+
+/* Action Buttons */
+.app-actions {
+    display: flex;
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.action-btn {
+    padding: 0.75rem 1.5rem;
+    border: 1px solid var(--border-color);
+    background: transparent;
+    color: var(--text-primary);
+    text-decoration: none;
+    display: inline-flex;
+    align-items: center;
+    gap: 0.5rem;
+    transition: all 0.2s;
+    cursor: pointer;
+    font-family: inherit;
+    font-size: 0.9rem;
+}
+
+.action-btn.primary {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    border-color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.action-btn.primary:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+    transform: translateY(-2px);
+}
+
+.action-btn.secondary {
+    border-color: var(--accent-pink);
+    color: var(--accent-pink);
+}
+
+.action-btn.secondary:hover {
+    background: rgba(243, 128, 245, 0.1);
+    box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2);
+}
+
+.action-btn.ghost {
+    border-color: var(--border-color);
+    color: var(--text-secondary);
+}
+
+.action-btn.ghost:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Pricing */
+.pricing-info {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+    font-size: 1.1rem;
+}
+
+.pricing-label {
+    color: var(--text-tertiary);
+}
+
+.pricing-value {
+    color: var(--warning);
+    font-weight: 600;
+}
+
+/* Navigation Tabs */
+.tabs {
+    display: flex;
+    flex-direction: row;
+    gap: 0;
+    border-bottom: 2px solid var(--border-color);
+    margin-bottom: 0;
+    background: var(--bg-tertiary);
+}
+
+.tab-btn {
+    padding: 1rem 2rem;
+    background: transparent;
+    border: none;
+    border-bottom: 3px solid transparent;
+    color: var(--text-secondary);
+    cursor: pointer;
+    transition: all 0.2s;
+    font-family: inherit;
+    font-size: 0.95rem;
+    margin-bottom: -2px;
+    white-space: nowrap;
+    font-weight: 500;
+}
+
+.tab-btn:hover {
+    color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.05);
+}
+
+.tab-btn.active {
+    color: var(--primary-cyan);
+    border-bottom-color: var(--primary-cyan);
+    background: var(--bg-secondary);
+}
+
+.app-nav {
+    max-width: 1800px;
+    margin: 2rem auto 0;
+    padding: 0 2rem;
+    display: flex;
+    gap: 1rem;
+    border-bottom: 2px solid var(--border-color);
+}
+
+.nav-tab {
+    padding: 1rem 1.5rem;
+    background: transparent;
+    border: none;
+    border-bottom: 2px solid transparent;
+    color: var(--text-secondary);
+    cursor: pointer;
+    transition: all 0.2s;
+    font-family: inherit;
+    font-size: 0.9rem;
+    margin-bottom: -2px;
+}
+
+.nav-tab:hover {
+    color: var(--primary-cyan);
+}
+
+.nav-tab.active {
+    color: var(--primary-cyan);
+    border-bottom-color: var(--primary-cyan);
+}
+
+/* Main Content Wrapper */
+.app-main {
+    max-width: 1800px;
+    margin: 2rem auto;
+    padding: 0 2rem;
+}
+
+/* Content Sections */
+.app-content {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 0;
+}
+
+.tab-content {
+    display: none !important;
+    padding: 2rem;
+}
+
+.tab-content.active {
+    display: block !important;
+}
+
+/* Overview Layout */
+.overview-columns {
+    display: grid;
+    grid-template-columns: 2fr 1fr;
+    gap: 2rem;
+}
+
+.overview-main h2, .overview-main h3 {
+    color: var(--primary-cyan);
+    margin-top: 2rem;
+    margin-bottom: 1rem;
+}
+
+.overview-main h2:first-child {
+    margin-top: 0;
+}
+
+.overview-main h2 {
+    font-size: 1.8rem;
+    border-bottom: 2px solid var(--border-color);
+    padding-bottom: 0.5rem;
+}
+
+.overview-main h3 {
+    font-size: 1.3rem;
+}
+
+.features-list {
+    list-style: none;
+    padding: 0;
+}
+
+.features-list li {
+    padding: 0.5rem 0;
+    padding-left: 1.5rem;
+    position: relative;
+    color: var(--text-secondary);
+}
+
+.features-list li:before {
+    content: "▸";
+    position: absolute;
+    left: 0;
+    color: var(--primary-cyan);
+}
+
+.use-cases p {
+    color: var(--text-secondary);
+    line-height: 1.6;
+}
+
+/* Sidebar */
+.sidebar {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+}
+
+.sidebar-card {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+}
+
+.sidebar-card h3 {
+    font-size: 1.1rem;
+    color: var(--primary-cyan);
+    margin: 0 0 1rem 0;
+    border-bottom: 1px solid var(--border-color);
+    padding-bottom: 0.5rem;
+}
+
+.stats-grid {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 1rem;
+}
+
+.stats-grid > div {
+    text-align: center;
+}
+
+.metadata {
+    margin: 0;
+}
+
+.metadata div {
+    display: flex;
+    justify-content: space-between;
+    padding: 0.75rem 0;
+    border-bottom: 1px solid var(--border-color);
+}
+
+.metadata dt {
+    color: var(--text-tertiary);
+    font-weight: normal;
+}
+
+.metadata dd {
+    color: var(--text-primary);
+    margin: 0;
+    font-weight: 600;
+}
+
+.sidebar-card p {
+    color: var(--text-secondary);
+    margin: 0;
+}
+
+/* Integration Content */
+.integration-content {
+    max-width: 100%;
+}
+
+.integration-content h2 {
+    font-size: 1.8rem;
+    color: var(--primary-cyan);
+    margin: 0 0 2rem 0;
+    padding-bottom: 0.5rem;
+    border-bottom: 2px solid var(--border-color);
+}
+
+.integration-content h3 {
+    font-size: 1.3rem;
+    color: var(--text-primary);
+    margin: 2rem 0 1rem;
+}
+
+.docs-content {
+    max-width: 100%;
+}
+
+.docs-content h2 {
+    font-size: 1.8rem;
+    color: var(--primary-cyan);
+    margin: 0 0 1.5rem 0;
+    padding-bottom: 0.5rem;
+    border-bottom: 2px solid var(--border-color);
+}
+
+.docs-content h3 {
+    font-size: 1.3rem;
+    color: var(--text-primary);
+    margin: 2rem 0 1rem;
+}
+
+.docs-content h4 {
+    font-size: 1.1rem;
+    color: var(--accent-pink);
+    margin: 1.5rem 0 0.5rem;
+}
+
+.docs-content p {
+    color: var(--text-secondary);
+    line-height: 1.6;
+    margin-bottom: 1rem;
+}
+
+.docs-content code {
+    background: var(--bg-tertiary);
+    padding: 0.2rem 0.4rem;
+    color: var(--primary-cyan);
+    font-family: 'Dank Mono', Monaco, monospace;
+    font-size: 0.9em;
+}
+
+/* Code Blocks */
+.code-block {
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    margin: 1rem 0;
+    overflow: hidden;
+    position: relative;
+}
+
+.code-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 0.5rem 1rem;
+    background: var(--bg-tertiary);
+    border-bottom: 1px solid var(--border-color);
+}
+
+.code-lang {
+    color: var(--primary-cyan);
+    font-size: 0.875rem;
+    text-transform: uppercase;
+}
+
+.copy-btn {
+    position: absolute;
+    top: 0.5rem;
+    right: 0.5rem;
+    padding: 0.4rem 0.8rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+    font-size: 0.75rem;
+    transition: all 0.2s;
+    z-index: 10;
+}
+
+.copy-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+    background: var(--bg-secondary);
+}
+
+.code-block pre {
+    margin: 0;
+    padding: 1rem;
+    overflow-x: auto;
+}
+
+.code-block code {
+    background: transparent;
+    padding: 0;
+    color: var(--text-secondary);
+    font-size: 0.875rem;
+    line-height: 1.5;
+}
+
+/* Markdown rendered code blocks */
+.integration-content pre,
+.docs-content pre {
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    margin: 1rem 0;
+    padding: 1rem;
+    padding-top: 2.5rem; /* Space for copy button */
+    overflow-x: auto;
+    position: relative;
+    max-height: none; /* Remove any height restrictions */
+    height: auto; /* Allow content to expand */
+}
+
+.integration-content pre code,
+.docs-content pre code {
+    background: transparent;
+    padding: 0;
+    color: var(--text-secondary);
+    font-size: 0.875rem;
+    line-height: 1.5;
+    white-space: pre; /* Preserve whitespace and line breaks */
+    display: block;
+}
+
+/* Feature Grid */
+.feature-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.feature-card {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+    transition: all 0.2s;
+}
+
+.feature-card:hover {
+    border-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.05);
+}
+
+.feature-card h4 {
+    margin-top: 0;
+}
+
+/* Info Box */
+.info-box {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.05), rgba(243, 128, 245, 0.03));
+    border: 1px solid var(--primary-cyan);
+    border-left: 4px solid var(--primary-cyan);
+    padding: 1.5rem;
+    margin: 2rem 0;
+}
+
+.info-box h4 {
+    margin-top: 0;
+    color: var(--primary-cyan);
+}
+
+/* Support Grid */
+.support-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.support-card {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+    text-align: center;
+}
+
+.support-card h3 {
+    color: var(--primary-cyan);
+    margin-bottom: 0.5rem;
+}
+
+/* Related Apps */
+.related-apps {
+    max-width: 1800px;
+    margin: 4rem auto;
+    padding: 0 2rem;
+}
+
+.related-apps h2 {
+    font-size: 1.5rem;
+    color: var(--text-primary);
+    margin-bottom: 1.5rem;
+}
+
+.related-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+    gap: 1rem;
+}
+
+.related-app-card {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 1rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.related-app-card:hover {
+    border-color: var(--primary-cyan);
+    transform: translateY(-2px);
+}
+
+/* Responsive */
+@media (max-width: 1024px) {
+    .app-hero-content {
+        grid-template-columns: 1fr;
+    }
+
+    .app-stats {
+        justify-content: space-around;
+    }
+
+    .overview-columns {
+        grid-template-columns: 1fr;
+    }
+}
+
+@media (max-width: 768px) {
+    .app-hero-info h1 {
+        font-size: 2rem;
+    }
+
+    .app-actions {
+        flex-direction: column;
+    }
+
+    .tabs {
+        overflow-x: auto;
+        -webkit-overflow-scrolling: touch;
+    }
+
+    .tab-btn {
+        padding: 0.75rem 1.5rem;
+        font-size: 0.875rem;
+    }
+
+    .app-nav {
+        overflow-x: auto;
+        gap: 0;
+    }
+
+    .nav-tab {
+        white-space: nowrap;
+    }
+
+    .feature-grid,
+    .support-grid {
+        grid-template-columns: 1fr;
+    }
+
+    .tab-content {
+        padding: 1rem;
+    }
+
+    .app-main {
+        padding: 0 1rem;
+    }
+}
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/app-detail.html b/docs/md_v2/marketplace/app-detail.html
new file mode 100644
index 0000000..574f9f4
--- /dev/null
+++ b/docs/md_v2/marketplace/app-detail.html
@@ -0,0 +1,175 @@
+
+
+
+    
+    
+    App Details - Crawl4AI Marketplace
+    
+    
+
+
+    

+        
+        

+            

+                

+                    

+                        
+                        

+                            [
+                            Marketplace
+                            ]
+                        

+                    

+                

+                

+                    ← Back to Marketplace
+                

+            

+        

+
+        
+        

+            

+                

+                    
+                

+                

+                    

+                        Open Source
+                        
+                        
+                    

+                    
App Name

+                    
App description goes here

+
+                    

+                        

+                            ★★★★★
+                            Rating
+                        

+                        

+                            0
+                            Downloads
+                        

+                        

+                            Category
+                            Category
+                        

+                    

+
+                    

+                        Visit Website
+                        View GitHub
+                        
+                    

+                

+            

+        

+
+        
+        

+            

+                

+                    
+                    
+                    
+                

+
+                

+                    

+                        

+                            
Overview content goes here.

+                        

+
+                        
+                    

+                

+
+                

+                    

+                    

+                

+
+                
+
+                
+            

+
+        

+
+        
+        

+            
Related Apps

+            
+        

+    

+
+    
+
+
diff --git a/docs/md_v2/marketplace/app-detail.js b/docs/md_v2/marketplace/app-detail.js
new file mode 100644
index 0000000..29fcb7e
--- /dev/null
+++ b/docs/md_v2/marketplace/app-detail.js
@@ -0,0 +1,318 @@
+// App Detail Page JavaScript
+const { API_BASE, API_ORIGIN } = (() => {
+    const { hostname, port, protocol } = window.location;
+    const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname);
+
+    if (isLocalHost && port && port !== '8100') {
+        const origin = `${protocol}//127.0.0.1:8100`;
+        return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin };
+    }
+
+    return { API_BASE: '/marketplace/api', API_ORIGIN: '' };
+})();
+
+class AppDetailPage {
+    constructor() {
+        this.appSlug = this.getAppSlugFromURL();
+        this.appData = null;
+        this.init();
+    }
+
+    getAppSlugFromURL() {
+        const params = new URLSearchParams(window.location.search);
+        return params.get('app') || '';
+    }
+
+    async init() {
+        if (!this.appSlug) {
+            window.location.href = 'index.html';
+            return;
+        }
+
+        await this.loadAppDetails();
+        this.setupEventListeners();
+        await this.loadRelatedApps();
+    }
+
+    async loadAppDetails() {
+        try {
+            const response = await fetch(`${API_BASE}/apps/${this.appSlug}`);
+            if (!response.ok) throw new Error('App not found');
+
+            this.appData = await response.json();
+            this.renderAppDetails();
+        } catch (error) {
+            console.error('Error loading app details:', error);
+            // Fallback to loading all apps and finding the right one
+            try {
+                const response = await fetch(`${API_BASE}/apps`);
+                const apps = await response.json();
+                this.appData = apps.find(app => app.slug === this.appSlug || app.name.toLowerCase().replace(/\s+/g, '-') === this.appSlug);
+                if (this.appData) {
+                    this.renderAppDetails();
+                } else {
+                    window.location.href = 'index.html';
+                }
+            } catch (err) {
+                console.error('Error loading apps:', err);
+                window.location.href = 'index.html';
+            }
+        }
+    }
+
+    renderAppDetails() {
+        if (!this.appData) return;
+
+        // Update title
+        document.title = `${this.appData.name} - Crawl4AI Marketplace`;
+
+        // Hero image
+        const appImage = document.getElementById('app-image');
+        if (this.appData.image) {
+            appImage.style.backgroundImage = `url('${this.appData.image}')`;
+            appImage.innerHTML = '';
+        } else {
+            appImage.innerHTML = `[${this.appData.category || 'APP'}]`;
+        }
+
+        // Basic info
+        document.getElementById('app-name').textContent = this.appData.name;
+        document.getElementById('app-description').textContent = this.appData.description;
+        document.getElementById('app-type').textContent = this.appData.type || 'Open Source';
+        document.getElementById('app-category').textContent = this.appData.category;
+
+        // Badges
+        if (this.appData.featured) {
+            document.getElementById('app-featured').style.display = 'inline-block';
+        }
+        if (this.appData.sponsored) {
+            document.getElementById('app-sponsored').style.display = 'inline-block';
+        }
+
+        // Stats
+        const rating = this.appData.rating || 0;
+        const stars = '★'.repeat(Math.floor(rating)) + '☆'.repeat(5 - Math.floor(rating));
+        document.getElementById('app-rating').textContent = stars + ` ${rating}/5`;
+        document.getElementById('app-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
+
+        // Action buttons
+        const websiteBtn = document.getElementById('app-website');
+        const githubBtn = document.getElementById('app-github');
+
+        if (this.appData.website_url) {
+            websiteBtn.href = this.appData.website_url;
+        } else {
+            websiteBtn.style.display = 'none';
+        }
+
+        if (this.appData.github_url) {
+            githubBtn.href = this.appData.github_url;
+        } else {
+            githubBtn.style.display = 'none';
+        }
+
+        // Contact
+        document.getElementById('app-contact') && (document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available');
+
+        // Sidebar info
+        document.getElementById('sidebar-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
+        document.getElementById('sidebar-rating').textContent = (this.appData.rating || 0).toFixed(1);
+        document.getElementById('sidebar-category').textContent = this.appData.category || '-';
+        document.getElementById('sidebar-type').textContent = this.appData.type || '-';
+        document.getElementById('sidebar-status').textContent = this.appData.status || 'Active';
+        document.getElementById('sidebar-pricing').textContent = this.appData.pricing || 'Free';
+        document.getElementById('sidebar-contact').textContent = this.appData.contact_email || 'contact@example.com';
+
+        // Render tab contents from database fields
+        this.renderTabContents();
+    }
+
+    renderTabContents() {
+        // Overview tab - use long_description from database
+        const overviewDiv = document.getElementById('app-overview');
+        if (overviewDiv) {
+            if (this.appData.long_description) {
+                overviewDiv.innerHTML = this.renderMarkdown(this.appData.long_description);
+            } else {
+                overviewDiv.innerHTML = `
${this.appData.description || 'No overview available.'}
`;
+            }
+        }
+
+        // Integration tab - use integration_guide field from database
+        const integrationDiv = document.getElementById('app-integration');
+        if (integrationDiv) {
+            if (this.appData.integration_guide) {
+                integrationDiv.innerHTML = this.renderMarkdown(this.appData.integration_guide);
+                // Add copy buttons to all code blocks
+                this.addCopyButtonsToCodeBlocks(integrationDiv);
+            } else {
+                integrationDiv.innerHTML = '
Integration guide not yet available. Please check the official website for details.
';
+            }
+        }
+
+        // Documentation tab - use documentation field from database
+        const docsDiv = document.getElementById('app-docs');
+        if (docsDiv) {
+            if (this.appData.documentation) {
+                docsDiv.innerHTML = this.renderMarkdown(this.appData.documentation);
+                // Add copy buttons to all code blocks
+                this.addCopyButtonsToCodeBlocks(docsDiv);
+            } else {
+                docsDiv.innerHTML = '
Documentation coming soon.
';
+            }
+        }
+    }
+
+    addCopyButtonsToCodeBlocks(container) {
+        // Find all code blocks and add copy buttons
+        const codeBlocks = container.querySelectorAll('pre code');
+        codeBlocks.forEach(codeBlock => {
+            const pre = codeBlock.parentElement;
+
+            // Skip if already has a copy button
+            if (pre.querySelector('.copy-btn')) return;
+
+            // Create copy button
+            const copyBtn = document.createElement('button');
+            copyBtn.className = 'copy-btn';
+            copyBtn.textContent = 'Copy';
+            copyBtn.onclick = () => {
+                navigator.clipboard.writeText(codeBlock.textContent).then(() => {
+                    copyBtn.textContent = '✓ Copied!';
+                    setTimeout(() => {
+                        copyBtn.textContent = 'Copy';
+                    }, 2000);
+                });
+            };
+
+            // Add button to pre element
+            pre.style.position = 'relative';
+            pre.insertBefore(copyBtn, codeBlock);
+        });
+    }
+
+    renderMarkdown(text) {
+        if (!text) return '';
+
+        // Store code blocks temporarily to protect them from processing
+        const codeBlocks = [];
+        let processed = text.replace(/```(\w+)?\n([\s\S]*?)```/g, (match, lang, code) => {
+            const placeholder = `___CODE_BLOCK_${codeBlocks.length}___`;
+            codeBlocks.push(`
${this.escapeHtml(code)}
`);
+            return placeholder;
+        });
+
+        // Store inline code temporarily
+        const inlineCodes = [];
+        processed = processed.replace(/`([^`]+)`/g, (match, code) => {
+            const placeholder = `___INLINE_CODE_${inlineCodes.length}___`;
+            inlineCodes.push(`${this.escapeHtml(code)}`);
+            return placeholder;
+        });
+
+        // Now process the rest of the markdown
+        processed = processed
+            // Headers
+            .replace(/^### (.*$)/gim, '
$1
')
+            .replace(/^## (.*$)/gim, '
$1
')
+            .replace(/^# (.*$)/gim, '
$1
')
+            // Bold
+            .replace(/\*\*(.*?)\*\*/g, '$1')
+            // Italic
+            .replace(/\*(.*?)\*/g, '$1')
+            // Links
+            .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1')
+            // Line breaks
+            .replace(/\n\n/g, '
')
+            .replace(/\n/g, '
')
+            // Lists
+            .replace(/^\* (.*)$/gim, '
  • $1
    • ')
+            .replace(/^- (.*)$/gim, '
  • $1
    • ')
+            // Wrap in paragraphs
+            .replace(/^(?!<[h|p|pre|ul|ol|li])/gim, '
    ')
+            .replace(/(?])$/gim, '
    ');
+
+        // Restore inline code
+        inlineCodes.forEach((code, i) => {
+            processed = processed.replace(`___INLINE_CODE_${i}___`, code);
+        });
+
+        // Restore code blocks
+        codeBlocks.forEach((block, i) => {
+            processed = processed.replace(`___CODE_BLOCK_${i}___`, block);
+        });
+
+        return processed;
+    }
+
+    escapeHtml(text) {
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+
+    formatNumber(num) {
+        if (num >= 1000000) {
+            return (num / 1000000).toFixed(1) + 'M';
+        } else if (num >= 1000) {
+            return (num / 1000).toFixed(1) + 'K';
+        }
+        return num.toString();
+    }
+
+    setupEventListeners() {
+        // Tab switching
+        const tabs = document.querySelectorAll('.tab-btn');
+
+        tabs.forEach(tab => {
+            tab.addEventListener('click', () => {
+                // Update active tab button
+                tabs.forEach(t => t.classList.remove('active'));
+                tab.classList.add('active');
+
+                // Show corresponding content
+                const tabName = tab.dataset.tab;
+
+                // Hide all tab contents
+                const allTabContents = document.querySelectorAll('.tab-content');
+                allTabContents.forEach(content => {
+                    content.classList.remove('active');
+                });
+
+                // Show the selected tab content
+                const targetTab = document.getElementById(`${tabName}-tab`);
+                if (targetTab) {
+                    targetTab.classList.add('active');
+                }
+            });
+        });
+    }
+
+    async loadRelatedApps() {
+        try {
+            const response = await fetch(`${API_BASE}/apps?category=${encodeURIComponent(this.appData.category)}&limit=4`);
+            const apps = await response.json();
+
+            const relatedApps = apps.filter(app => app.slug !== this.appSlug).slice(0, 3);
+            const grid = document.getElementById('related-apps-grid');
+
+            grid.innerHTML = relatedApps.map(app => `
+                
    
+                    
    ${app.name}
    
+                    
    ${app.description.substring(0, 100)}...
    
+                    
    
+                        ${app.type}
+                        ★ ${app.rating}/5
+                    
    
+                
    
+            `).join('');
+        } catch (error) {
+            console.error('Error loading related apps:', error);
+        }
+    }
+}
+
+// Initialize when DOM is loaded
+document.addEventListener('DOMContentLoaded', () => {
+    new AppDetailPage();
+});
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/backend/.env.example b/docs/md_v2/marketplace/backend/.env.example
new file mode 100644
index 0000000..7d46f19
--- /dev/null
+++ b/docs/md_v2/marketplace/backend/.env.example
@@ -0,0 +1,14 @@
+# Marketplace Configuration
+# Copy this to .env and update with your values
+
+# Admin password (required)
+MARKETPLACE_ADMIN_PASSWORD=change_this_password
+
+# JWT secret key (required) - generate with: python3 -c "import secrets; print(secrets.token_urlsafe(32))"
+MARKETPLACE_JWT_SECRET=change_this_to_a_secure_random_key
+
+# Database path (optional, defaults to ./marketplace.db)
+MARKETPLACE_DB_PATH=./marketplace.db
+
+# Token expiry in hours (optional, defaults to 4)
+MARKETPLACE_TOKEN_EXPIRY=4
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/backend/config.py b/docs/md_v2/marketplace/backend/config.py
new file mode 100644
index 0000000..29bb55d
--- /dev/null
+++ b/docs/md_v2/marketplace/backend/config.py
@@ -0,0 +1,59 @@
+"""
+Marketplace Configuration - Loads from .env file
+"""
+import os
+import sys
+import hashlib
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Load .env file
+env_path = Path(__file__).parent / '.env'
+if not env_path.exists():
+    print("\n❌ ERROR: No .env file found!")
+    print("Please copy .env.example to .env and update with your values:")
+    print(f"  cp {Path(__file__).parent}/.env.example {Path(__file__).parent}/.env")
+    print("\nThen edit .env with your secure values.")
+    sys.exit(1)
+
+load_dotenv(env_path)
+
+# Required environment variables
+required_vars = ['MARKETPLACE_ADMIN_PASSWORD', 'MARKETPLACE_JWT_SECRET']
+missing_vars = [var for var in required_vars if not os.getenv(var)]
+
+if missing_vars:
+    print(f"\n❌ ERROR: Missing required environment variables: {', '.join(missing_vars)}")
+    print("Please check your .env file and ensure all required variables are set.")
+    sys.exit(1)
+
+class Config:
+    """Configuration loaded from environment variables"""
+
+    # Admin authentication - hashed from password in .env
+    ADMIN_PASSWORD_HASH = hashlib.sha256(
+        os.getenv('MARKETPLACE_ADMIN_PASSWORD').encode()
+    ).hexdigest()
+
+    # JWT secret for token generation
+    JWT_SECRET_KEY = os.getenv('MARKETPLACE_JWT_SECRET')
+
+    # Database path
+    DATABASE_PATH = os.getenv('MARKETPLACE_DB_PATH', './marketplace.db')
+
+    # Token expiry in hours
+    TOKEN_EXPIRY_HOURS = int(os.getenv('MARKETPLACE_TOKEN_EXPIRY', '4'))
+
+    # CORS origins - hardcoded as they don't contain secrets
+    ALLOWED_ORIGINS = [
+        "http://localhost:8000",
+        "http://localhost:8080",
+        "http://localhost:8100",
+        "http://127.0.0.1:8000",
+        "http://127.0.0.1:8080",
+        "http://127.0.0.1:8100",
+        "https://crawl4ai.com",
+        "https://www.crawl4ai.com",
+        "https://docs.crawl4ai.com",
+        "https://market.crawl4ai.com"
+    ]
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/backend/database.py b/docs/md_v2/marketplace/backend/database.py
new file mode 100644
index 0000000..8ccfbaf
--- /dev/null
+++ b/docs/md_v2/marketplace/backend/database.py
@@ -0,0 +1,117 @@
+import sqlite3
+import yaml
+import json
+from pathlib import Path
+from typing import Dict, List, Any
+
+class DatabaseManager:
+    def __init__(self, db_path=None, schema_path='schema.yaml'):
+        self.schema = self._load_schema(schema_path)
+        # Use provided path or fallback to schema default
+        self.db_path = db_path or self.schema['database']['name']
+        self.conn = None
+        self._init_database()
+
+    def _load_schema(self, path: str) -> Dict:
+        with open(path, 'r') as f:
+            return yaml.safe_load(f)
+
+    def _init_database(self):
+        """Auto-create/migrate database from schema"""
+        self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        self.conn.row_factory = sqlite3.Row
+
+        for table_name, table_def in self.schema['tables'].items():
+            self._create_or_update_table(table_name, table_def['columns'])
+
+    def _create_or_update_table(self, table_name: str, columns: Dict):
+        cursor = self.conn.cursor()
+
+        # Check if table exists
+        cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,))
+        table_exists = cursor.fetchone() is not None
+
+        if not table_exists:
+            # Create table
+            col_defs = []
+            for col_name, col_spec in columns.items():
+                col_def = f"{col_name} {col_spec['type']}"
+                if col_spec.get('primary'):
+                    col_def += " PRIMARY KEY"
+                if col_spec.get('autoincrement'):
+                    col_def += " AUTOINCREMENT"
+                if col_spec.get('unique'):
+                    col_def += " UNIQUE"
+                if col_spec.get('required'):
+                    col_def += " NOT NULL"
+                if 'default' in col_spec:
+                    default = col_spec['default']
+                    if default == 'CURRENT_TIMESTAMP':
+                        col_def += f" DEFAULT {default}"
+                    elif isinstance(default, str):
+                        col_def += f" DEFAULT '{default}'"
+                    else:
+                        col_def += f" DEFAULT {default}"
+                col_defs.append(col_def)
+
+            create_sql = f"CREATE TABLE {table_name} ({', '.join(col_defs)})"
+            cursor.execute(create_sql)
+        else:
+            # Check for new columns and add them
+            cursor.execute(f"PRAGMA table_info({table_name})")
+            existing_columns = {row[1] for row in cursor.fetchall()}
+
+            for col_name, col_spec in columns.items():
+                if col_name not in existing_columns:
+                    col_def = f"{col_spec['type']}"
+                    if 'default' in col_spec:
+                        default = col_spec['default']
+                        if default == 'CURRENT_TIMESTAMP':
+                            col_def += f" DEFAULT {default}"
+                        elif isinstance(default, str):
+                            col_def += f" DEFAULT '{default}'"
+                        else:
+                            col_def += f" DEFAULT {default}"
+
+                    cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_def}")
+
+        self.conn.commit()
+
+    def get_all(self, table: str, limit: int = 100, offset: int = 0, where: str = None) -> List[Dict]:
+        cursor = self.conn.cursor()
+        query = f"SELECT * FROM {table}"
+        if where:
+            query += f" WHERE {where}"
+        query += f" LIMIT {limit} OFFSET {offset}"
+
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        return [dict(row) for row in rows]
+
+    def search(self, query: str, tables: List[str] = None) -> Dict[str, List[Dict]]:
+        if not tables:
+            tables = list(self.schema['tables'].keys())
+
+        results = {}
+        cursor = self.conn.cursor()
+
+        for table in tables:
+            # Search in text columns
+            columns = self.schema['tables'][table]['columns']
+            text_cols = [col for col, spec in columns.items()
+                        if spec['type'] == 'TEXT' and col != 'id']
+
+            if text_cols:
+                where_clause = ' OR '.join([f"{col} LIKE ?" for col in text_cols])
+                params = [f'%{query}%'] * len(text_cols)
+
+                cursor.execute(f"SELECT * FROM {table} WHERE {where_clause} LIMIT 10", params)
+                rows = cursor.fetchall()
+                if rows:
+                    results[table] = [dict(row) for row in rows]
+
+        return results
+
+    def close(self):
+        if self.conn:
+            self.conn.close()
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/backend/dummy_data.py b/docs/md_v2/marketplace/backend/dummy_data.py
new file mode 100644
index 0000000..3e7f46f
--- /dev/null
+++ b/docs/md_v2/marketplace/backend/dummy_data.py
@@ -0,0 +1,267 @@
+import sqlite3
+import json
+import random
+from datetime import datetime, timedelta
+from database import DatabaseManager
+
+def generate_slug(text):
+    return text.lower().replace(' ', '-').replace('&', 'and')
+
+def generate_dummy_data():
+    db = DatabaseManager()
+    conn = db.conn
+    cursor = conn.cursor()
+
+    # Clear existing data
+    for table in ['apps', 'articles', 'categories', 'sponsors']:
+        cursor.execute(f"DELETE FROM {table}")
+
+    # Categories
+    categories = [
+        ("Browser Automation", "⚙", "Tools for browser automation and control"),
+        ("Proxy Services", "🔒", "Proxy providers and rotation services"),
+        ("LLM Integration", "🤖", "AI/LLM tools and integrations"),
+        ("Data Processing", "📊", "Data extraction and processing tools"),
+        ("Cloud Infrastructure", "☁", "Cloud browser and computing services"),
+        ("Developer Tools", "🛠", "Development and testing utilities")
+    ]
+
+    for i, (name, icon, desc) in enumerate(categories):
+        cursor.execute("""
+            INSERT INTO categories (name, slug, icon, description, order_index)
+            VALUES (?, ?, ?, ?, ?)
+        """, (name, generate_slug(name), icon, desc, i))
+
+    # Apps with real Unsplash images
+    apps_data = [
+        # Browser Automation
+        ("Playwright Cloud", "Browser Automation", "Paid", True, True,
+         "Scalable browser automation in the cloud with Playwright", "https://playwright.cloud",
+         None, "$99/month starter", 4.8, 12500,
+         "https://images.unsplash.com/photo-1633356122544-f134324a6cee?w=800&h=400&fit=crop"),
+
+        ("Selenium Grid Hub", "Browser Automation", "Freemium", False, False,
+         "Distributed Selenium grid for parallel testing", "https://seleniumhub.io",
+         "https://github.com/seleniumhub/grid", "Free - $299/month", 4.2, 8400,
+         "https://images.unsplash.com/photo-1555066931-4365d14bab8c?w=800&h=400&fit=crop"),
+
+        ("Puppeteer Extra", "Browser Automation", "Open Source", True, False,
+         "Enhanced Puppeteer with stealth plugins and more", "https://puppeteer-extra.dev",
+         "https://github.com/berstend/puppeteer-extra", "Free", 4.6, 15200,
+         "https://images.unsplash.com/photo-1461749280684-dccba630e2f6?w=800&h=400&fit=crop"),
+
+        # Proxy Services
+        ("BrightData", "Proxy Services", "Paid", True, True,
+         "Premium proxy network with 72M+ IPs worldwide", "https://brightdata.com",
+         None, "Starting $500/month", 4.7, 9800,
+         "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=800&h=400&fit=crop"),
+
+        ("SmartProxy", "Proxy Services", "Paid", False, True,
+         "Residential and datacenter proxies with rotation", "https://smartproxy.com",
+         None, "Starting $75/month", 4.3, 7600,
+         "https://images.unsplash.com/photo-1544197150-b99a580bb7a8?w=800&h=400&fit=crop"),
+
+        ("ProxyMesh", "Proxy Services", "Freemium", False, False,
+         "Rotating proxy servers with sticky sessions", "https://proxymesh.com",
+         None, "$10-$50/month", 4.0, 4200,
+         "https://images.unsplash.com/photo-1451187580459-43490279c0fa?w=800&h=400&fit=crop"),
+
+        # LLM Integration
+        ("LangChain Crawl", "LLM Integration", "Open Source", True, False,
+         "LangChain integration for Crawl4AI workflows", "https://langchain-crawl.dev",
+         "https://github.com/langchain/crawl", "Free", 4.5, 18900,
+         "https://images.unsplash.com/photo-1677442136019-21780ecad995?w=800&h=400&fit=crop"),
+
+        ("GPT Scraper", "LLM Integration", "Freemium", False, False,
+         "Extract structured data using GPT models", "https://gptscraper.ai",
+         None, "Free - $99/month", 4.1, 5600,
+         "https://images.unsplash.com/photo-1655720828018-edd2daec9349?w=800&h=400&fit=crop"),
+
+        ("Claude Extract", "LLM Integration", "Paid", True, True,
+         "Professional extraction using Claude AI", "https://claude-extract.com",
+         None, "$199/month", 4.9, 3200,
+         "https://images.unsplash.com/photo-1686191128892-3b09ad503b4f?w=800&h=400&fit=crop"),
+
+        # Data Processing
+        ("DataMiner Pro", "Data Processing", "Paid", False, False,
+         "Advanced data extraction and transformation", "https://dataminer.pro",
+         None, "$149/month", 4.2, 6700,
+         "https://images.unsplash.com/photo-1551288049-bebda4e38f71?w=800&h=400&fit=crop"),
+
+        ("ScraperAPI", "Data Processing", "Freemium", True, True,
+         "Simple API for web scraping with proxy rotation", "https://scraperapi.com",
+         None, "Free - $299/month", 4.6, 22300,
+         "https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=800&h=400&fit=crop"),
+
+        ("Apify", "Data Processing", "Freemium", False, False,
+         "Web scraping and automation platform", "https://apify.com",
+         None, "$49-$499/month", 4.4, 14500,
+         "https://images.unsplash.com/photo-1504639725590-34d0984388bd?w=800&h=400&fit=crop"),
+
+        # Cloud Infrastructure
+        ("BrowserCloud", "Cloud Infrastructure", "Paid", True, True,
+         "Managed headless browsers in the cloud", "https://browsercloud.io",
+         None, "$199/month", 4.5, 8900,
+         "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=800&h=400&fit=crop"),
+
+        ("LambdaTest", "Cloud Infrastructure", "Freemium", False, False,
+         "Cross-browser testing on cloud", "https://lambdatest.com",
+         None, "Free - $99/month", 4.1, 11200,
+         "https://images.unsplash.com/photo-1451187580459-43490279c0fa?w=800&h=400&fit=crop"),
+
+        ("Browserless", "Cloud Infrastructure", "Freemium", True, False,
+         "Headless browser automation API", "https://browserless.io",
+         None, "$50-$500/month", 4.7, 19800,
+         "https://images.unsplash.com/photo-1639762681485-074b7f938ba0?w=800&h=400&fit=crop"),
+
+        # Developer Tools
+        ("Crawl4AI VSCode", "Developer Tools", "Open Source", True, False,
+         "VSCode extension for Crawl4AI development", "https://marketplace.visualstudio.com",
+         "https://github.com/crawl4ai/vscode", "Free", 4.8, 34500,
+         "https://images.unsplash.com/photo-1629654297299-c8506221ca97?w=800&h=400&fit=crop"),
+
+        ("Postman Collection", "Developer Tools", "Open Source", False, False,
+         "Postman collection for Crawl4AI API testing", "https://postman.com/crawl4ai",
+         "https://github.com/crawl4ai/postman", "Free", 4.3, 7800,
+         "https://images.unsplash.com/photo-1599507593499-a3f7d7d97667?w=800&h=400&fit=crop"),
+
+        ("Debug Toolkit", "Developer Tools", "Open Source", False, False,
+         "Debugging tools for crawler development", "https://debug.crawl4ai.com",
+         "https://github.com/crawl4ai/debug", "Free", 4.0, 4300,
+         "https://images.unsplash.com/photo-1515879218367-8466d910aaa4?w=800&h=400&fit=crop"),
+    ]
+
+    for name, category, type_, featured, sponsored, desc, url, github, pricing, rating, downloads, image in apps_data:
+        screenshots = json.dumps([
+            f"https://images.unsplash.com/photo-{random.randint(1500000000000, 1700000000000)}-{random.randint(1000000000000, 9999999999999)}?w=800&h=600&fit=crop",
+            f"https://images.unsplash.com/photo-{random.randint(1500000000000, 1700000000000)}-{random.randint(1000000000000, 9999999999999)}?w=800&h=600&fit=crop"
+        ])
+        cursor.execute("""
+            INSERT INTO apps (name, slug, description, category, type, featured, sponsored,
+                            website_url, github_url, pricing, rating, downloads, image, screenshots, logo_url,
+                            integration_guide, contact_email, views)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (name, generate_slug(name), desc, category, type_, featured, sponsored,
+             url, github, pricing, rating, downloads, image, screenshots,
+             f"https://ui-avatars.com/api/?name={name}&background=50ffff&color=070708&size=128",
+             f"# {name} Integration\n\n```python\nfrom crawl4ai import AsyncWebCrawler\n# Integration code coming soon...\n```",
+             f"contact@{generate_slug(name)}.com",
+             random.randint(100, 5000)))
+
+    # Articles with real images
+    articles_data = [
+        ("Browser Automation Showdown: Playwright vs Puppeteer vs Selenium",
+         "Review", "John Doe", ["Playwright Cloud", "Puppeteer Extra"],
+         ["browser-automation", "comparison", "2024"],
+         "https://images.unsplash.com/photo-1587620962725-abab7fe55159?w=1200&h=630&fit=crop"),
+
+        ("Top 5 Proxy Services for Web Scraping in 2024",
+         "Comparison", "Jane Smith", ["BrightData", "SmartProxy", "ProxyMesh"],
+         ["proxy", "web-scraping", "guide"],
+         "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=1200&h=630&fit=crop"),
+
+        ("Integrating LLMs with Crawl4AI: A Complete Guide",
+         "Tutorial", "Crawl4AI Team", ["LangChain Crawl", "GPT Scraper", "Claude Extract"],
+         ["llm", "integration", "tutorial"],
+         "https://images.unsplash.com/photo-1677442136019-21780ecad995?w=1200&h=630&fit=crop"),
+
+        ("Building Scalable Crawlers with Cloud Infrastructure",
+         "Tutorial", "Mike Johnson", ["BrowserCloud", "Browserless"],
+         ["cloud", "scalability", "architecture"],
+         "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=1200&h=630&fit=crop"),
+
+        ("What's New in Crawl4AI Marketplace",
+         "News", "Crawl4AI Team", [],
+         ["marketplace", "announcement", "news"],
+         "https://images.unsplash.com/photo-1556075798-4825dfaaf498?w=1200&h=630&fit=crop"),
+
+        ("Cost Analysis: Self-Hosted vs Cloud Browser Solutions",
+         "Comparison", "Sarah Chen", ["BrowserCloud", "LambdaTest", "Browserless"],
+         ["cost", "cloud", "comparison"],
+         "https://images.unsplash.com/photo-1554224155-8d04cb21cd6c?w=1200&h=630&fit=crop"),
+
+        ("Getting Started with Browser Automation",
+         "Tutorial", "Crawl4AI Team", ["Playwright Cloud", "Selenium Grid Hub"],
+         ["beginner", "tutorial", "automation"],
+         "https://images.unsplash.com/photo-1498050108023-c5249f4df085?w=1200&h=630&fit=crop"),
+
+        ("The Future of Web Scraping: AI-Powered Extraction",
+         "News", "Dr. Alan Turing", ["Claude Extract", "GPT Scraper"],
+         ["ai", "future", "trends"],
+         "https://images.unsplash.com/photo-1593720213428-28a5b9e94613?w=1200&h=630&fit=crop")
+    ]
+
+    for title, category, author, related_apps, tags, image in articles_data:
+        # Get app IDs for related apps
+        related_ids = []
+        for app_name in related_apps:
+            cursor.execute("SELECT id FROM apps WHERE name = ?", (app_name,))
+            result = cursor.fetchone()
+            if result:
+                related_ids.append(result[0])
+
+        content = f"""# {title}
+
+By {author} | {datetime.now().strftime('%B %d, %Y')}
+
+## Introduction
+
+This is a comprehensive article about {title.lower()}. Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+
+## Key Points
+
+- Important point about the topic
+- Another crucial insight
+- Technical details and specifications
+- Performance comparisons
+
+## Conclusion
+
+In summary, this article explored various aspects of the topic. Stay tuned for more updates!
+"""
+
+        cursor.execute("""
+            INSERT INTO articles (title, slug, content, author, category, related_apps,
+                                featured_image, tags, views)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (title, generate_slug(title), content, author, category,
+             json.dumps(related_ids), image, json.dumps(tags),
+             random.randint(200, 10000)))
+
+    # Sponsors
+    sponsors_data = [
+        ("BrightData", "Gold", "https://brightdata.com",
+         "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=728&h=90&fit=crop"),
+        ("ScraperAPI", "Gold", "https://scraperapi.com",
+         "https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=728&h=90&fit=crop"),
+        ("BrowserCloud", "Silver", "https://browsercloud.io",
+         "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=728&h=90&fit=crop"),
+        ("Claude Extract", "Silver", "https://claude-extract.com",
+         "https://images.unsplash.com/photo-1686191128892-3b09ad503b4f?w=728&h=90&fit=crop"),
+        ("SmartProxy", "Bronze", "https://smartproxy.com",
+         "https://images.unsplash.com/photo-1544197150-b99a580bb7a8?w=728&h=90&fit=crop")
+    ]
+
+    for company, tier, landing_url, banner in sponsors_data:
+        start_date = datetime.now() - timedelta(days=random.randint(1, 30))
+        end_date = datetime.now() + timedelta(days=random.randint(30, 180))
+
+        cursor.execute("""
+            INSERT INTO sponsors (company_name, logo_url, tier, banner_url,
+                                landing_url, active, start_date, end_date)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        """, (company,
+             f"https://ui-avatars.com/api/?name={company}&background=09b5a5&color=fff&size=200",
+             tier, banner, landing_url, 1,
+             start_date.isoformat(), end_date.isoformat()))
+
+    conn.commit()
+    print("✓ Dummy data generated successfully!")
+    print(f"  - {len(categories)} categories")
+    print(f"  - {len(apps_data)} apps")
+    print(f"  - {len(articles_data)} articles")
+    print(f"  - {len(sponsors_data)} sponsors")
+
+if __name__ == "__main__":
+    generate_dummy_data()
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/backend/requirements.txt b/docs/md_v2/marketplace/backend/requirements.txt
new file mode 100644
index 0000000..1401b0e
--- /dev/null
+++ b/docs/md_v2/marketplace/backend/requirements.txt
@@ -0,0 +1,5 @@
+fastapi
+uvicorn
+pyyaml
+python-multipart
+python-dotenv
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/backend/schema.yaml b/docs/md_v2/marketplace/backend/schema.yaml
new file mode 100644
index 0000000..c5f443d
--- /dev/null
+++ b/docs/md_v2/marketplace/backend/schema.yaml
@@ -0,0 +1,75 @@
+database:
+  name: marketplace.db
+
+tables:
+  apps:
+    columns:
+      id: {type: INTEGER, primary: true, autoincrement: true}
+      name: {type: TEXT, required: true}
+      slug: {type: TEXT, unique: true}
+      description: {type: TEXT}
+      long_description: {type: TEXT}
+      logo_url: {type: TEXT}
+      image: {type: TEXT}
+      screenshots: {type: JSON, default: '[]'}
+      category: {type: TEXT}
+      type: {type: TEXT, default: 'Open Source'}
+      status: {type: TEXT, default: 'Active'}
+      website_url: {type: TEXT}
+      github_url: {type: TEXT}
+      demo_url: {type: TEXT}
+      video_url: {type: TEXT}
+      documentation_url: {type: TEXT}
+      support_url: {type: TEXT}
+      discord_url: {type: TEXT}
+      pricing: {type: TEXT}
+      rating: {type: REAL, default: 0.0}
+      downloads: {type: INTEGER, default: 0}
+      featured: {type: BOOLEAN, default: 0}
+      sponsored: {type: BOOLEAN, default: 0}
+      integration_guide: {type: TEXT}
+      documentation: {type: TEXT}
+      examples: {type: TEXT}
+      installation_command: {type: TEXT}
+      requirements: {type: TEXT}
+      changelog: {type: TEXT}
+      tags: {type: JSON, default: '[]'}
+      added_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
+      updated_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
+      contact_email: {type: TEXT}
+      views: {type: INTEGER, default: 0}
+
+  articles:
+    columns:
+      id: {type: INTEGER, primary: true, autoincrement: true}
+      title: {type: TEXT, required: true}
+      slug: {type: TEXT, unique: true}
+      content: {type: TEXT}
+      author: {type: TEXT, default: 'Crawl4AI Team'}
+      category: {type: TEXT}
+      related_apps: {type: JSON, default: '[]'}
+      featured_image: {type: TEXT}
+      published_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
+      tags: {type: JSON, default: '[]'}
+      views: {type: INTEGER, default: 0}
+
+  categories:
+    columns:
+      id: {type: INTEGER, primary: true, autoincrement: true}
+      name: {type: TEXT, unique: true}
+      slug: {type: TEXT, unique: true}
+      icon: {type: TEXT}
+      description: {type: TEXT}
+      order_index: {type: INTEGER, default: 0}
+
+  sponsors:
+    columns:
+      id: {type: INTEGER, primary: true, autoincrement: true}
+      company_name: {type: TEXT, required: true}
+      logo_url: {type: TEXT}
+      tier: {type: TEXT, default: 'Bronze'}
+      banner_url: {type: TEXT}
+      landing_url: {type: TEXT}
+      active: {type: BOOLEAN, default: 1}
+      start_date: {type: DATETIME}
+      end_date: {type: DATETIME}
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/backend/server.py b/docs/md_v2/marketplace/backend/server.py
new file mode 100644
index 0000000..f4935eb
--- /dev/null
+++ b/docs/md_v2/marketplace/backend/server.py
@@ -0,0 +1,497 @@
+from fastapi import FastAPI, HTTPException, Query, Depends, Body, UploadFile, File, Form, APIRouter
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from typing import Optional, Dict, Any
+import json
+import hashlib
+import secrets
+import re
+from pathlib import Path
+from database import DatabaseManager
+from datetime import datetime, timedelta
+
+# Import configuration (will exit if .env not found or invalid)
+from config import Config
+
+app = FastAPI(title="Crawl4AI Marketplace API")
+router = APIRouter(prefix="/marketplace/api")
+
+# Security setup
+security = HTTPBearer()
+tokens = {}  # In production, use Redis or database for token storage
+
+# CORS configuration
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=Config.ALLOWED_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+    allow_headers=["*"],
+    max_age=3600
+)
+
+# Initialize database with configurable path
+db = DatabaseManager(Config.DATABASE_PATH)
+
+BASE_DIR = Path(__file__).parent
+UPLOAD_ROOT = BASE_DIR / "uploads"
+UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
+
+app.mount("/uploads", StaticFiles(directory=UPLOAD_ROOT), name="uploads")
+
+ALLOWED_IMAGE_TYPES = {
+    "image/png": ".png",
+    "image/jpeg": ".jpg",
+    "image/webp": ".webp",
+    "image/svg+xml": ".svg"
+}
+ALLOWED_UPLOAD_FOLDERS = {"sponsors"}
+MAX_UPLOAD_SIZE = 2 * 1024 * 1024  # 2 MB
+
+def json_response(data, cache_time=3600):
+    """Helper to return JSON with cache headers"""
+    return JSONResponse(
+        content=data,
+        headers={
+            "Cache-Control": f"public, max-age={cache_time}",
+            "X-Content-Type-Options": "nosniff"
+        }
+    )
+
+
+def to_int(value, default=0):
+    """Coerce incoming values to integers, falling back to default."""
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return int(value)
+    if isinstance(value, (int, float)):
+        return int(value)
+
+    if isinstance(value, str):
+        stripped = value.strip()
+        if not stripped:
+            return default
+
+        match = re.match(r"^-?\d+", stripped)
+        if match:
+            try:
+                return int(match.group())
+            except ValueError:
+                return default
+    return default
+
+# ============= PUBLIC ENDPOINTS =============
+
+@router.get("/apps")
+async def get_apps(
+    category: Optional[str] = None,
+    type: Optional[str] = None,
+    featured: Optional[bool] = None,
+    sponsored: Optional[bool] = None,
+    limit: int = Query(default=20, le=10000),
+    offset: int = Query(default=0)
+):
+    """Get apps with optional filters"""
+    where_clauses = []
+    if category:
+        where_clauses.append(f"category = '{category}'")
+    if type:
+        where_clauses.append(f"type = '{type}'")
+    if featured is not None:
+        where_clauses.append(f"featured = {1 if featured else 0}")
+    if sponsored is not None:
+        where_clauses.append(f"sponsored = {1 if sponsored else 0}")
+
+    where = " AND ".join(where_clauses) if where_clauses else None
+    apps = db.get_all('apps', limit=limit, offset=offset, where=where)
+
+    # Parse JSON fields
+    for app in apps:
+        if app.get('screenshots'):
+            app['screenshots'] = json.loads(app['screenshots'])
+
+    return json_response(apps)
+
+@router.get("/apps/{slug}")
+async def get_app(slug: str):
+    """Get single app by slug"""
+    apps = db.get_all('apps', where=f"slug = '{slug}'", limit=1)
+    if not apps:
+        raise HTTPException(status_code=404, detail="App not found")
+
+    app = apps[0]
+    if app.get('screenshots'):
+        app['screenshots'] = json.loads(app['screenshots'])
+
+    return json_response(app)
+
+@router.get("/articles")
+async def get_articles(
+    category: Optional[str] = None,
+    limit: int = Query(default=20, le=10000),
+    offset: int = Query(default=0)
+):
+    """Get articles with optional category filter"""
+    where = f"category = '{category}'" if category else None
+    articles = db.get_all('articles', limit=limit, offset=offset, where=where)
+
+    # Parse JSON fields
+    for article in articles:
+        if article.get('related_apps'):
+            article['related_apps'] = json.loads(article['related_apps'])
+        if article.get('tags'):
+            article['tags'] = json.loads(article['tags'])
+
+    return json_response(articles)
+
+@router.get("/articles/{slug}")
+async def get_article(slug: str):
+    """Get single article by slug"""
+    articles = db.get_all('articles', where=f"slug = '{slug}'", limit=1)
+    if not articles:
+        raise HTTPException(status_code=404, detail="Article not found")
+
+    article = articles[0]
+    if article.get('related_apps'):
+        article['related_apps'] = json.loads(article['related_apps'])
+    if article.get('tags'):
+        article['tags'] = json.loads(article['tags'])
+
+    return json_response(article)
+
+@router.get("/categories")
+async def get_categories():
+    """Get all categories ordered by index"""
+    categories = db.get_all('categories', limit=50)
+    for category in categories:
+        category['order_index'] = to_int(category.get('order_index'), 0)
+    categories.sort(key=lambda x: x.get('order_index', 0))
+    return json_response(categories, cache_time=7200)
+
+@router.get("/sponsors")
+async def get_sponsors(active: Optional[bool] = True):
+    """Get sponsors, default active only"""
+    where = f"active = {1 if active else 0}" if active is not None else None
+    sponsors = db.get_all('sponsors', where=where, limit=20)
+
+    # Filter by date if active
+    if active:
+        now = datetime.now().isoformat()
+        sponsors = [s for s in sponsors
+                   if (not s.get('start_date') or s['start_date'] <= now) and
+                      (not s.get('end_date') or s['end_date'] >= now)]
+
+    return json_response(sponsors)
+
+@router.get("/search")
+async def search(q: str = Query(min_length=2)):
+    """Search across apps and articles"""
+    if len(q) < 2:
+        return json_response({})
+
+    results = db.search(q, tables=['apps', 'articles'])
+
+    # Parse JSON fields in results
+    for table, items in results.items():
+        for item in items:
+            if table == 'apps' and item.get('screenshots'):
+                item['screenshots'] = json.loads(item['screenshots'])
+            elif table == 'articles':
+                if item.get('related_apps'):
+                    item['related_apps'] = json.loads(item['related_apps'])
+                if item.get('tags'):
+                    item['tags'] = json.loads(item['tags'])
+
+    return json_response(results, cache_time=1800)
+
+@router.get("/stats")
+async def get_stats():
+    """Get marketplace statistics"""
+    stats = {
+        "total_apps": len(db.get_all('apps', limit=10000)),
+        "total_articles": len(db.get_all('articles', limit=10000)),
+        "total_categories": len(db.get_all('categories', limit=1000)),
+        "active_sponsors": len(db.get_all('sponsors', where="active = 1", limit=1000))
+    }
+    return json_response(stats, cache_time=1800)
+
+# ============= ADMIN AUTHENTICATION =============
+
+def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    """Verify admin authentication token"""
+    token = credentials.credentials
+    if token not in tokens or tokens[token] < datetime.now():
+        raise HTTPException(status_code=401, detail="Invalid or expired token")
+    return token
+
+
+@router.post("/admin/upload-image", dependencies=[Depends(verify_token)])
+async def upload_image(file: UploadFile = File(...), folder: str = Form("sponsors")):
+    """Upload image files for admin assets"""
+    folder = (folder or "").strip().lower()
+    if folder not in ALLOWED_UPLOAD_FOLDERS:
+        raise HTTPException(status_code=400, detail="Invalid upload folder")
+
+    if file.content_type not in ALLOWED_IMAGE_TYPES:
+        raise HTTPException(status_code=400, detail="Unsupported file type")
+
+    contents = await file.read()
+    if len(contents) > MAX_UPLOAD_SIZE:
+        raise HTTPException(status_code=400, detail="File too large (max 2MB)")
+
+    extension = ALLOWED_IMAGE_TYPES[file.content_type]
+    filename = f"{datetime.now().strftime('%Y%m%d%H%M%S')}_{secrets.token_hex(8)}{extension}"
+
+    target_dir = UPLOAD_ROOT / folder
+    target_dir.mkdir(parents=True, exist_ok=True)
+    target_path = target_dir / filename
+    target_path.write_bytes(contents)
+
+    return {"url": f"/uploads/{folder}/{filename}"}
+
+@router.post("/admin/login")
+async def admin_login(password: str = Body(..., embed=True)):
+    """Admin login with password"""
+    provided_hash = hashlib.sha256(password.encode()).hexdigest()
+
+    if provided_hash != Config.ADMIN_PASSWORD_HASH:
+        # Log failed attempt in production
+        print(f"Failed login attempt at {datetime.now()}")
+        raise HTTPException(status_code=401, detail="Invalid password")
+
+    # Generate secure token
+    token = secrets.token_urlsafe(32)
+    tokens[token] = datetime.now() + timedelta(hours=Config.TOKEN_EXPIRY_HOURS)
+
+    return {
+        "token": token,
+        "expires_in": Config.TOKEN_EXPIRY_HOURS * 3600
+    }
+
+# ============= ADMIN ENDPOINTS =============
+
+@router.get("/admin/stats", dependencies=[Depends(verify_token)])
+async def get_admin_stats():
+    """Get detailed admin statistics"""
+    stats = {
+        "apps": {
+            "total": len(db.get_all('apps', limit=10000)),
+            "featured": len(db.get_all('apps', where="featured = 1", limit=10000)),
+            "sponsored": len(db.get_all('apps', where="sponsored = 1", limit=10000))
+        },
+        "articles": len(db.get_all('articles', limit=10000)),
+        "categories": len(db.get_all('categories', limit=1000)),
+        "sponsors": {
+            "active": len(db.get_all('sponsors', where="active = 1", limit=1000)),
+            "total": len(db.get_all('sponsors', limit=10000))
+        },
+        "total_views": sum(app.get('views', 0) for app in db.get_all('apps', limit=10000))
+    }
+    return stats
+
+# Apps CRUD
+@router.post("/admin/apps", dependencies=[Depends(verify_token)])
+async def create_app(app_data: Dict[str, Any]):
+    """Create new app"""
+    try:
+        # Handle JSON fields
+        for field in ['screenshots', 'tags']:
+            if field in app_data and isinstance(app_data[field], list):
+                app_data[field] = json.dumps(app_data[field])
+
+        cursor = db.conn.cursor()
+        columns = ', '.join(app_data.keys())
+        placeholders = ', '.join(['?' for _ in app_data])
+        cursor.execute(f"INSERT INTO apps ({columns}) VALUES ({placeholders})",
+                      list(app_data.values()))
+        db.conn.commit()
+        return {"id": cursor.lastrowid, "message": "App created"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.put("/admin/apps/{app_id}", dependencies=[Depends(verify_token)])
+async def update_app(app_id: int, app_data: Dict[str, Any]):
+    """Update app"""
+    try:
+        # Handle JSON fields
+        for field in ['screenshots', 'tags']:
+            if field in app_data and isinstance(app_data[field], list):
+                app_data[field] = json.dumps(app_data[field])
+
+        set_clause = ', '.join([f"{k} = ?" for k in app_data.keys()])
+        cursor = db.conn.cursor()
+        cursor.execute(f"UPDATE apps SET {set_clause} WHERE id = ?",
+                      list(app_data.values()) + [app_id])
+        db.conn.commit()
+        return {"message": "App updated"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.delete("/admin/apps/{app_id}", dependencies=[Depends(verify_token)])
+async def delete_app(app_id: int):
+    """Delete app"""
+    cursor = db.conn.cursor()
+    cursor.execute("DELETE FROM apps WHERE id = ?", (app_id,))
+    db.conn.commit()
+    return {"message": "App deleted"}
+
+# Articles CRUD
+@router.post("/admin/articles", dependencies=[Depends(verify_token)])
+async def create_article(article_data: Dict[str, Any]):
+    """Create new article"""
+    try:
+        for field in ['related_apps', 'tags']:
+            if field in article_data and isinstance(article_data[field], list):
+                article_data[field] = json.dumps(article_data[field])
+
+        cursor = db.conn.cursor()
+        columns = ', '.join(article_data.keys())
+        placeholders = ', '.join(['?' for _ in article_data])
+        cursor.execute(f"INSERT INTO articles ({columns}) VALUES ({placeholders})",
+                      list(article_data.values()))
+        db.conn.commit()
+        return {"id": cursor.lastrowid, "message": "Article created"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.put("/admin/articles/{article_id}", dependencies=[Depends(verify_token)])
+async def update_article(article_id: int, article_data: Dict[str, Any]):
+    """Update article"""
+    try:
+        for field in ['related_apps', 'tags']:
+            if field in article_data and isinstance(article_data[field], list):
+                article_data[field] = json.dumps(article_data[field])
+
+        set_clause = ', '.join([f"{k} = ?" for k in article_data.keys()])
+        cursor = db.conn.cursor()
+        cursor.execute(f"UPDATE articles SET {set_clause} WHERE id = ?",
+                      list(article_data.values()) + [article_id])
+        db.conn.commit()
+        return {"message": "Article updated"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.delete("/admin/articles/{article_id}", dependencies=[Depends(verify_token)])
+async def delete_article(article_id: int):
+    """Delete article"""
+    cursor = db.conn.cursor()
+    cursor.execute("DELETE FROM articles WHERE id = ?", (article_id,))
+    db.conn.commit()
+    return {"message": "Article deleted"}
+
+# Categories CRUD
+@router.post("/admin/categories", dependencies=[Depends(verify_token)])
+async def create_category(category_data: Dict[str, Any]):
+    """Create new category"""
+    try:
+        category_data = dict(category_data)
+        category_data['order_index'] = to_int(category_data.get('order_index'), 0)
+
+        cursor = db.conn.cursor()
+        columns = ', '.join(category_data.keys())
+        placeholders = ', '.join(['?' for _ in category_data])
+        cursor.execute(f"INSERT INTO categories ({columns}) VALUES ({placeholders})",
+                      list(category_data.values()))
+        db.conn.commit()
+        return {"id": cursor.lastrowid, "message": "Category created"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.put("/admin/categories/{cat_id}", dependencies=[Depends(verify_token)])
+async def update_category(cat_id: int, category_data: Dict[str, Any]):
+    """Update category"""
+    try:
+        category_data = dict(category_data)
+        if 'order_index' in category_data:
+            category_data['order_index'] = to_int(category_data.get('order_index'), 0)
+
+        set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()])
+        cursor = db.conn.cursor()
+        cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?",
+                      list(category_data.values()) + [cat_id])
+        db.conn.commit()
+        return {"message": "Category updated"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@router.delete("/admin/categories/{cat_id}", dependencies=[Depends(verify_token)])
+async def delete_category(cat_id: int):
+    """Delete category"""
+    try:
+        cursor = db.conn.cursor()
+        cursor.execute("DELETE FROM categories WHERE id = ?", (cat_id,))
+        db.conn.commit()
+        return {"message": "Category deleted"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+# Sponsors CRUD
+@router.post("/admin/sponsors", dependencies=[Depends(verify_token)])
+async def create_sponsor(sponsor_data: Dict[str, Any]):
+    """Create new sponsor"""
+    try:
+        cursor = db.conn.cursor()
+        columns = ', '.join(sponsor_data.keys())
+        placeholders = ', '.join(['?' for _ in sponsor_data])
+        cursor.execute(f"INSERT INTO sponsors ({columns}) VALUES ({placeholders})",
+                      list(sponsor_data.values()))
+        db.conn.commit()
+        return {"id": cursor.lastrowid, "message": "Sponsor created"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.put("/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)])
+async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]):
+    """Update sponsor"""
+    try:
+        set_clause = ', '.join([f"{k} = ?" for k in sponsor_data.keys()])
+        cursor = db.conn.cursor()
+        cursor.execute(f"UPDATE sponsors SET {set_clause} WHERE id = ?",
+                      list(sponsor_data.values()) + [sponsor_id])
+        db.conn.commit()
+        return {"message": "Sponsor updated"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@router.delete("/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)])
+async def delete_sponsor(sponsor_id: int):
+    """Delete sponsor"""
+    try:
+        cursor = db.conn.cursor()
+        cursor.execute("DELETE FROM sponsors WHERE id = ?", (sponsor_id,))
+        db.conn.commit()
+        return {"message": "Sponsor deleted"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+app.include_router(router)
+
+# Version info
+VERSION = "1.1.0"
+BUILD_DATE = "2025-10-26"
+
+@app.get("/")
+async def root():
+    """API info"""
+    return {
+        "name": "Crawl4AI Marketplace API",
+        "version": VERSION,
+        "build_date": BUILD_DATE,
+        "endpoints": [
+            "/marketplace/api/apps",
+            "/marketplace/api/articles",
+            "/marketplace/api/categories",
+            "/marketplace/api/sponsors",
+            "/marketplace/api/search?q=query",
+            "/marketplace/api/stats"
+        ]
+    }
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="127.0.0.1", port=8100)
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/backend/uploads/.gitignore b/docs/md_v2/marketplace/backend/uploads/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/docs/md_v2/marketplace/backend/uploads/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/docs/md_v2/marketplace/frontend/app-detail.css b/docs/md_v2/marketplace/frontend/app-detail.css
new file mode 100644
index 0000000..9f04c13
--- /dev/null
+++ b/docs/md_v2/marketplace/frontend/app-detail.css
@@ -0,0 +1,462 @@
+/* App Detail Page Styles */
+
+.app-detail-container {
+    min-height: 100vh;
+    background: var(--bg-dark);
+}
+
+/* Back Button */
+.header-nav {
+    display: flex;
+    align-items: center;
+}
+
+.back-btn {
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+    font-size: 0.875rem;
+}
+
+.back-btn:hover {
+    border-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.1);
+}
+
+/* App Hero Section */
+.app-hero {
+    max-width: 1800px;
+    margin: 2rem auto;
+    padding: 0 2rem;
+}
+
+.app-hero-content {
+    display: grid;
+    grid-template-columns: 1fr 2fr;
+    gap: 3rem;
+    background: linear-gradient(135deg, #1a1a2e, #0f0f1e);
+    border: 2px solid var(--primary-cyan);
+    padding: 2rem;
+    box-shadow: 0 0 30px rgba(80, 255, 255, 0.15),
+                inset 0 0 20px rgba(80, 255, 255, 0.05);
+}
+
+.app-hero-image {
+    width: 100%;
+    height: 300px;
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05));
+    background-size: cover;
+    background-position: center;
+    border: 1px solid var(--border-color);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 4rem;
+    color: var(--primary-cyan);
+}
+
+.app-badges {
+    display: flex;
+    gap: 0.5rem;
+    margin-bottom: 1rem;
+}
+
+.app-badge {
+    padding: 0.3rem 0.6rem;
+    background: var(--bg-tertiary);
+    color: var(--text-secondary);
+    font-size: 0.75rem;
+    text-transform: uppercase;
+    font-weight: 600;
+}
+
+.app-badge.featured {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3);
+}
+
+.app-badge.sponsored {
+    background: linear-gradient(135deg, var(--warning), #ff8c00);
+    color: var(--bg-dark);
+    box-shadow: 0 2px 10px rgba(245, 158, 11, 0.3);
+}
+
+.app-hero-info h1 {
+    font-size: 2.5rem;
+    color: var(--primary-cyan);
+    margin: 0.5rem 0;
+    text-shadow: 0 0 20px rgba(80, 255, 255, 0.5);
+}
+
+.app-tagline {
+    font-size: 1.1rem;
+    color: var(--text-secondary);
+    margin-bottom: 2rem;
+}
+
+/* Stats */
+.app-stats {
+    display: flex;
+    gap: 2rem;
+    margin: 2rem 0;
+    padding: 1rem 0;
+    border-top: 1px solid var(--border-color);
+    border-bottom: 1px solid var(--border-color);
+}
+
+.stat {
+    display: flex;
+    flex-direction: column;
+    gap: 0.25rem;
+}
+
+.stat-value {
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.stat-label {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+}
+
+/* Action Buttons */
+.app-actions {
+    display: flex;
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.action-btn {
+    padding: 0.75rem 1.5rem;
+    border: 1px solid var(--border-color);
+    background: transparent;
+    color: var(--text-primary);
+    text-decoration: none;
+    display: inline-flex;
+    align-items: center;
+    gap: 0.5rem;
+    transition: all 0.2s;
+    cursor: pointer;
+    font-family: inherit;
+    font-size: 0.9rem;
+}
+
+.action-btn.primary {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    border-color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.action-btn.primary:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+    transform: translateY(-2px);
+}
+
+.action-btn.secondary {
+    border-color: var(--accent-pink);
+    color: var(--accent-pink);
+}
+
+.action-btn.secondary:hover {
+    background: rgba(243, 128, 245, 0.1);
+    box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2);
+}
+
+.action-btn.ghost {
+    border-color: var(--border-color);
+    color: var(--text-secondary);
+}
+
+.action-btn.ghost:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Pricing */
+.pricing-info {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+    font-size: 1.1rem;
+}
+
+.pricing-label {
+    color: var(--text-tertiary);
+}
+
+.pricing-value {
+    color: var(--warning);
+    font-weight: 600;
+}
+
+/* Navigation Tabs */
+.app-nav {
+    max-width: 1800px;
+    margin: 2rem auto 0;
+    padding: 0 2rem;
+    display: flex;
+    gap: 1rem;
+    border-bottom: 2px solid var(--border-color);
+}
+
+.nav-tab {
+    padding: 1rem 1.5rem;
+    background: transparent;
+    border: none;
+    border-bottom: 2px solid transparent;
+    color: var(--text-secondary);
+    cursor: pointer;
+    transition: all 0.2s;
+    font-family: inherit;
+    font-size: 0.9rem;
+    margin-bottom: -2px;
+}
+
+.nav-tab:hover {
+    color: var(--primary-cyan);
+}
+
+.nav-tab.active {
+    color: var(--primary-cyan);
+    border-bottom-color: var(--primary-cyan);
+}
+
+/* Content Sections */
+.app-content {
+    max-width: 1800px;
+    margin: 2rem auto;
+    padding: 0 2rem;
+}
+
+.tab-content {
+    display: none;
+}
+
+.tab-content.active {
+    display: block;
+}
+
+.docs-content {
+    max-width: 1200px;
+    padding: 2rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+}
+
+.docs-content h2 {
+    font-size: 1.8rem;
+    color: var(--primary-cyan);
+    margin-bottom: 1rem;
+    padding-bottom: 0.5rem;
+    border-bottom: 1px solid var(--border-color);
+}
+
+.docs-content h3 {
+    font-size: 1.3rem;
+    color: var(--text-primary);
+    margin: 2rem 0 1rem;
+}
+
+.docs-content h4 {
+    font-size: 1.1rem;
+    color: var(--accent-pink);
+    margin: 1.5rem 0 0.5rem;
+}
+
+.docs-content p {
+    color: var(--text-secondary);
+    line-height: 1.6;
+    margin-bottom: 1rem;
+}
+
+.docs-content code {
+    background: var(--bg-tertiary);
+    padding: 0.2rem 0.4rem;
+    color: var(--primary-cyan);
+    font-family: 'Dank Mono', Monaco, monospace;
+    font-size: 0.9em;
+}
+
+/* Code Blocks */
+.code-block {
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    margin: 1rem 0;
+    overflow: hidden;
+}
+
+.code-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 0.5rem 1rem;
+    background: var(--bg-tertiary);
+    border-bottom: 1px solid var(--border-color);
+}
+
+.code-lang {
+    color: var(--primary-cyan);
+    font-size: 0.875rem;
+    text-transform: uppercase;
+}
+
+.copy-btn {
+    padding: 0.25rem 0.5rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+    font-size: 0.75rem;
+    transition: all 0.2s;
+}
+
+.copy-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+.code-block pre {
+    margin: 0;
+    padding: 1rem;
+    overflow-x: auto;
+}
+
+.code-block code {
+    background: transparent;
+    padding: 0;
+    color: var(--text-secondary);
+    font-size: 0.875rem;
+    line-height: 1.5;
+}
+
+/* Feature Grid */
+.feature-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.feature-card {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+    transition: all 0.2s;
+}
+
+.feature-card:hover {
+    border-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.05);
+}
+
+.feature-card h4 {
+    margin-top: 0;
+}
+
+/* Info Box */
+.info-box {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.05), rgba(243, 128, 245, 0.03));
+    border: 1px solid var(--primary-cyan);
+    border-left: 4px solid var(--primary-cyan);
+    padding: 1.5rem;
+    margin: 2rem 0;
+}
+
+.info-box h4 {
+    margin-top: 0;
+    color: var(--primary-cyan);
+}
+
+/* Support Grid */
+.support-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.support-card {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+    text-align: center;
+}
+
+.support-card h3 {
+    color: var(--primary-cyan);
+    margin-bottom: 0.5rem;
+}
+
+/* Related Apps */
+.related-apps {
+    max-width: 1800px;
+    margin: 4rem auto;
+    padding: 0 2rem;
+}
+
+.related-apps h2 {
+    font-size: 1.5rem;
+    color: var(--text-primary);
+    margin-bottom: 1.5rem;
+}
+
+.related-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+    gap: 1rem;
+}
+
+.related-app-card {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 1rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.related-app-card:hover {
+    border-color: var(--primary-cyan);
+    transform: translateY(-2px);
+}
+
+/* Responsive */
+@media (max-width: 1024px) {
+    .app-hero-content {
+        grid-template-columns: 1fr;
+    }
+
+    .app-stats {
+        justify-content: space-around;
+    }
+}
+
+@media (max-width: 768px) {
+    .app-hero-info h1 {
+        font-size: 2rem;
+    }
+
+    .app-actions {
+        flex-direction: column;
+    }
+
+    .app-nav {
+        overflow-x: auto;
+        gap: 0;
+    }
+
+    .nav-tab {
+        white-space: nowrap;
+    }
+
+    .feature-grid,
+    .support-grid {
+        grid-template-columns: 1fr;
+    }
+}
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/frontend/app-detail.html b/docs/md_v2/marketplace/frontend/app-detail.html
new file mode 100644
index 0000000..4a83d36
--- /dev/null
+++ b/docs/md_v2/marketplace/frontend/app-detail.html
@@ -0,0 +1,239 @@
+
+
+
+    
+    
+    App Details - Crawl4AI Marketplace
+    
+    
+
+
+    
    
+        
+        
    
+            
    
+                
    
+                    
    
+                        
+                        
    
+                            [
+                            Marketplace
+                            ]
+                        
    
+                    
    
+                
    
+                
    
+                    ← Back to Marketplace
+                
    
+            
    
+        
    
+
+        
+        
    
+            
    
+                
    
+                    
+                
    
+                
    
+                    
    
+                        Open Source
+                        
+                        
+                    
    
+                    
    App Name
    
+                    
    App description goes here
    
+
+                    
    
+                        
    
+                            ★★★★★
+                            Rating
+                        
    
+                        
    
+                            0
+                            Downloads
+                        
    
+                        
    
+                            Category
+                            Category
+                        
    
+                    
    
+
+                    
    
+                        
+                            → Visit Website
+                        
+                        
+                            ⚡ View on GitHub
+                        
+                        
+                    
    
+
+                    
    
+                        Pricing:
+                        Free
+                    
    
+                
    
+            
    
+        
    
+
+        
+        
+
+        
+        
    
+            
+            
    
+                
    
+                    
    Quick Start
    
+                    
    Get started with this integration in just a few steps.
    
+
+                    
    Installation
    
+                    
    
+                        
    
+                            bash
+                            
+                        
    
+                        
    pip install crawl4ai
    
+                    
    
+
+                    
    Basic Usage
    
+                    
    
+                        
    
+                            python
+                            
+                        
    
+                        
    from crawl4ai import AsyncWebCrawler
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            # Your configuration here
+        )
+        print(result.markdown)
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
    
+                    
    
+
+                    
    Advanced Configuration
    
+                    
    Customize the crawler with these advanced options:
    
+
+                    
    
+                        
    
+                            
    🚀 Performance
    
+                            
    Optimize crawling speed with parallel processing and caching strategies.
    
+                        
    
+                        
    
+                            
    🔒 Authentication
    
+                            
    Handle login forms, cookies, and session management automatically.
    
+                        
    
+                        
    
+                            
    🎯 Extraction
    
+                            
    Use CSS selectors, XPath, or AI-powered content extraction.
    
+                        
    
+                        
    
+                            
    🔄 Proxy Support
    
+                            
    Rotate proxies and bypass rate limiting with built-in proxy management.
    
+                        
    
+                    
    
+
+                    
    Integration Example
    
+                    
    
+                        
    
+                            python
+                            
+                        
    
+                        
    from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai import LLMConfig
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+async def extract_with_llm():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=CrawlerRunConfig(
+                extraction_strategy=LLMExtractionStrategy(
+                    llm_config=LLMConfig(
+                        provider="openai/gpt-4o",
+                        api_token="your-api-key",
+                    ),
+                    instruction="Extract product information"
+                ),
+                cache_mode=CacheMode.BYPASS
+            )
+        )
+        return result.extracted_content
+
+# Run the extraction
+data = await extract_with_llm()
+print(data)
    
+                    
    
+
+                    
    
+                        
    💡 Pro Tip
    
+                        
    Use cache_mode=CacheMode.BYPASS for a fresh crawl, or CacheMode.WRITE_ONLY to avoid refetching.
    
+                    
    
+                
    
+            
    
+
+            
+            
    
+                
    
+                    
    Documentation
    
+                    
    Complete documentation and API reference.
    
+                    
+                
    
+            
    
+
+            
+            
    
+                
    
+                    
    Examples
    
+                    
    Real-world examples and use cases.
    
+                    
+                
    
+            
    
+
+            
+            
    
+                
    
+                    
    Support
    
+                    
    
+                        
    
+                            
    📧 Contact
    
+                            
    contact@example.com
    
+                        
    
+                        
    
+                            
    🐛 Report Issues
    
+                            
    Found a bug? Report it on GitHub Issues.
    
+                        
    
+                        
    
+                            
    💬 Community
    
+                            
    Join our Discord for help and discussions.
    
+                        
    
+                    
    
+                
    
+            
    
+        
    
+
+        
+        
    
+            
    Related Apps
    
+            
+        
    
+    
    
+
+    
+
+
diff --git a/docs/md_v2/marketplace/frontend/app-detail.js b/docs/md_v2/marketplace/frontend/app-detail.js
new file mode 100644
index 0000000..2f4549d
--- /dev/null
+++ b/docs/md_v2/marketplace/frontend/app-detail.js
@@ -0,0 +1,335 @@
+// App Detail Page JavaScript
+const { API_BASE, API_ORIGIN } = (() => {
+    const { hostname, port, protocol } = window.location;
+    const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname);
+
+    if (isLocalHost && port && port !== '8100') {
+        const origin = `${protocol}//127.0.0.1:8100`;
+        return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin };
+    }
+
+    return { API_BASE: '/marketplace/api', API_ORIGIN: '' };
+})();
+
+class AppDetailPage {
+    constructor() {
+        this.appSlug = this.getAppSlugFromURL();
+        this.appData = null;
+        this.init();
+    }
+
+    getAppSlugFromURL() {
+        const params = new URLSearchParams(window.location.search);
+        return params.get('app') || '';
+    }
+
+    async init() {
+        if (!this.appSlug) {
+            window.location.href = 'index.html';
+            return;
+        }
+
+        await this.loadAppDetails();
+        this.setupEventListeners();
+        await this.loadRelatedApps();
+    }
+
+    async loadAppDetails() {
+        try {
+            const response = await fetch(`${API_BASE}/apps/${this.appSlug}`);
+            if (!response.ok) throw new Error('App not found');
+
+            this.appData = await response.json();
+            this.renderAppDetails();
+        } catch (error) {
+            console.error('Error loading app details:', error);
+            // Fallback to loading all apps and finding the right one
+            try {
+                const response = await fetch(`${API_BASE}/apps`);
+                const apps = await response.json();
+                this.appData = apps.find(app => app.slug === this.appSlug || app.name.toLowerCase().replace(/\s+/g, '-') === this.appSlug);
+                if (this.appData) {
+                    this.renderAppDetails();
+                } else {
+                    window.location.href = 'index.html';
+                }
+            } catch (err) {
+                console.error('Error loading apps:', err);
+                window.location.href = 'index.html';
+            }
+        }
+    }
+
+    renderAppDetails() {
+        if (!this.appData) return;
+
+        // Update title
+        document.title = `${this.appData.name} - Crawl4AI Marketplace`;
+
+        // Hero image
+        const appImage = document.getElementById('app-image');
+        if (this.appData.image) {
+            appImage.style.backgroundImage = `url('${this.appData.image}')`;
+            appImage.innerHTML = '';
+        } else {
+            appImage.innerHTML = `[${this.appData.category || 'APP'}]`;
+        }
+
+        // Basic info
+        document.getElementById('app-name').textContent = this.appData.name;
+        document.getElementById('app-description').textContent = this.appData.description;
+        document.getElementById('app-type').textContent = this.appData.type || 'Open Source';
+        document.getElementById('app-category').textContent = this.appData.category;
+        document.getElementById('app-pricing').textContent = this.appData.pricing || 'Free';
+
+        // Badges
+        if (this.appData.featured) {
+            document.getElementById('app-featured').style.display = 'inline-block';
+        }
+        if (this.appData.sponsored) {
+            document.getElementById('app-sponsored').style.display = 'inline-block';
+        }
+
+        // Stats
+        const rating = this.appData.rating || 0;
+        const stars = '★'.repeat(Math.floor(rating)) + '☆'.repeat(5 - Math.floor(rating));
+        document.getElementById('app-rating').textContent = stars + ` ${rating}/5`;
+        document.getElementById('app-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
+
+        // Action buttons
+        const websiteBtn = document.getElementById('app-website');
+        const githubBtn = document.getElementById('app-github');
+
+        if (this.appData.website_url) {
+            websiteBtn.href = this.appData.website_url;
+        } else {
+            websiteBtn.style.display = 'none';
+        }
+
+        if (this.appData.github_url) {
+            githubBtn.href = this.appData.github_url;
+        } else {
+            githubBtn.style.display = 'none';
+        }
+
+        // Contact
+        document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available';
+
+        // Integration guide
+        this.renderIntegrationGuide();
+    }
+
+    renderIntegrationGuide() {
+        // Installation code
+        const installCode = document.getElementById('install-code');
+        if (this.appData.type === 'Open Source' && this.appData.github_url) {
+            installCode.textContent = `# Clone from GitHub
+git clone ${this.appData.github_url}
+
+# Install dependencies
+pip install -r requirements.txt`;
+        } else if (this.appData.name.toLowerCase().includes('api')) {
+            installCode.textContent = `# Install via pip
+pip install ${this.appData.slug}
+
+# Or install from source
+pip install git+${this.appData.github_url || 'https://github.com/example/repo'}`;
+        }
+
+        // Usage code - customize based on category
+        const usageCode = document.getElementById('usage-code');
+        if (this.appData.category === 'Browser Automation') {
+            usageCode.textContent = `from crawl4ai import AsyncWebCrawler
+from ${this.appData.slug.replace(/-/g, '_')} import ${this.appData.name.replace(/\s+/g, '')}
+
+async def main():
+    # Initialize ${this.appData.name}
+    automation = ${this.appData.name.replace(/\s+/g, '')}()
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            browser_config=automation.config,
+            wait_for="css:body"
+        )
+        print(result.markdown)`;
+        } else if (this.appData.category === 'Proxy Services') {
+            usageCode.textContent = `from crawl4ai import AsyncWebCrawler
+import ${this.appData.slug.replace(/-/g, '_')}
+
+# Configure proxy
+proxy_config = {
+    "server": "${this.appData.website_url || 'https://proxy.example.com'}",
+    "username": "your_username",
+    "password": "your_password"
+}
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://example.com",
+        cache_mode=CacheMode.BYPASS
+    )
+    print(result.status_code)`;
+        } else if (this.appData.category === 'LLM Integration') {
+            usageCode.textContent = `from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.extraction_strategy import LLMExtractionStrategy, LLMConfig
+
+# Configure LLM extraction
+strategy = LLMExtractionStrategy(
+    llm_config=LLMConfig(
+        provider="${this.appData.name.toLowerCase().includes('gpt') ? 'openai/gpt-4o' : 'anthropic/claude-3-5-sonnet-20240620'}",
+        api_token="your-api-key",
+    ),
+    instruction="Extract structured data"
+)
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://example.com",
+        extraction_strategy=strategy
+    )
+    print(result.extracted_content)`;
+        }
+
+        // Integration example
+        const integrationCode = document.getElementById('integration-code');
+        integrationCode.textContent = this.appData.integration_guide ||
+`# Complete ${this.appData.name} Integration Example
+
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+import json
+
+async def crawl_with_${this.appData.slug.replace(/-/g, '_')}():
+    """
+    Complete example showing how to use ${this.appData.name}
+    with Crawl4AI for production web scraping
+    """
+
+    # Define extraction schema
+    schema = {
+        "name": "ProductList",
+        "baseSelector": "div.product",
+        "fields": [
+            {"name": "title", "selector": "h2", "type": "text"},
+            {"name": "price", "selector": ".price", "type": "text"},
+            {"name": "image", "selector": "img", "type": "attribute", "attribute": "src"},
+            {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"}
+        ]
+    }
+
+    # Initialize crawler with ${this.appData.name}
+    async with AsyncWebCrawler(
+        browser_type="chromium",
+        headless=True,
+        verbose=True
+    ) as crawler:
+
+        # Crawl with extraction
+        result = await crawler.arun(
+            url="https://example.com/products",
+            extraction_strategy=JsonCssExtractionStrategy(schema),
+            cache_mode=CacheMode.BYPASS,
+            wait_for="css:.product",
+            screenshot=True
+        )
+
+        # Process results
+        if result.success:
+            products = json.loads(result.extracted_content)
+            print(f"Found {len(products)} products")
+
+            for product in products[:5]:
+                print(f"- {product['title']}: {product['price']}")
+
+        return products
+
+# Run the crawler
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(crawl_with_${this.appData.slug.replace(/-/g, '_')}())`;
+    }
+
+    formatNumber(num) {
+        if (num >= 1000000) {
+            return (num / 1000000).toFixed(1) + 'M';
+        } else if (num >= 1000) {
+            return (num / 1000).toFixed(1) + 'K';
+        }
+        return num.toString();
+    }
+
+    setupEventListeners() {
+        // Tab switching
+        const tabs = document.querySelectorAll('.nav-tab');
+        tabs.forEach(tab => {
+            tab.addEventListener('click', () => {
+                // Update active tab
+                tabs.forEach(t => t.classList.remove('active'));
+                tab.classList.add('active');
+
+                // Show corresponding content
+                const tabName = tab.dataset.tab;
+                document.querySelectorAll('.tab-content').forEach(content => {
+                    content.classList.remove('active');
+                });
+                document.getElementById(`${tabName}-tab`).classList.add('active');
+            });
+        });
+
+        // Copy integration code
+        document.getElementById('copy-integration').addEventListener('click', () => {
+            const code = document.getElementById('integration-code').textContent;
+            navigator.clipboard.writeText(code).then(() => {
+                const btn = document.getElementById('copy-integration');
+                const originalText = btn.innerHTML;
+                btn.innerHTML = '✓ Copied!';
+                setTimeout(() => {
+                    btn.innerHTML = originalText;
+                }, 2000);
+            });
+        });
+
+        // Copy code buttons
+        document.querySelectorAll('.copy-btn').forEach(btn => {
+            btn.addEventListener('click', (e) => {
+                const codeBlock = e.target.closest('.code-block');
+                const code = codeBlock.querySelector('code').textContent;
+                navigator.clipboard.writeText(code).then(() => {
+                    btn.textContent = 'Copied!';
+                    setTimeout(() => {
+                        btn.textContent = 'Copy';
+                    }, 2000);
+                });
+            });
+        });
+    }
+
+    async loadRelatedApps() {
+        try {
+            const response = await fetch(`${API_BASE}/apps?category=${encodeURIComponent(this.appData.category)}&limit=4`);
+            const apps = await response.json();
+
+            const relatedApps = apps.filter(app => app.slug !== this.appSlug).slice(0, 3);
+            const grid = document.getElementById('related-apps-grid');
+
+            grid.innerHTML = relatedApps.map(app => `
+                
    
+                    
    ${app.name}
    
+                    
    ${app.description.substring(0, 100)}...
    
+                    
    
+                        ${app.type}
+                        ★ ${app.rating}/5
+                    
    
+                
    
+            `).join('');
+        } catch (error) {
+            console.error('Error loading related apps:', error);
+        }
+    }
+}
+
+// Initialize when DOM is loaded
+document.addEventListener('DOMContentLoaded', () => {
+    new AppDetailPage();
+});
diff --git a/docs/md_v2/marketplace/frontend/index.html b/docs/md_v2/marketplace/frontend/index.html
new file mode 100644
index 0000000..d034638
--- /dev/null
+++ b/docs/md_v2/marketplace/frontend/index.html
@@ -0,0 +1,147 @@
+
+
+
+    
+    
+    Marketplace - Crawl4AI
+    
+
+
+    
    
+        
+        
    
+            
    
+                
    
+                    
    
+                        
+                        
    
+                            [
+                            Marketplace
+                            ]
+                        
    
+                    
    
+                    
    Tools, Integrations & Resources for Web Crawling
    
+                
    
+                
    
+                    Apps: --
+                    Articles: --
+                    Downloads: --
+                
    
+            
    
+        
    
+
+        
+        
    
+            
    
+                >
+                
+                /
+            
    
+            
    
+                
+                
+            
    
+        
    
+
+        
+        
    
+            
+            
    
+                
+            
    
+
+            
+            
    
+                
+            
    
+
+            
+            
+
+            
+            
    
+                
+                
    
+                    
    
+                        
    > Latest Apps
    
+                        
+                    
    
+                    
    
+                        
+                    
    
+                
    
+
+                
+                
    
+                    
    
+                        
    > Latest Articles
    
+                    
    
+                    
    
+                        
+                    
    
+                
    
+
+                
+                
    
+                    
    
+                        
    # Trending
    
+                    
    
+                    
+
+                    
    
+                        
    + Submit Your Tool
    
+                        
    Share your integration
    
+                        Submit →
+                    
    
+                
    
+            
    
+
+            
+            
    
+                
    
+                    
    > More Apps
    
+                    
+                
    
+                
    
+                    
+                
    
+            
    
+        
    
+
+        
+        
    
+            
    
+                
    
+                    
    About Marketplace
    
+                    
    Discover tools and integrations built by the Crawl4AI community.
    
+                
    
+                
    
+                    
    Become a Sponsor
    
+                    
    Reach developers building with Crawl4AI
    
+                    
+                
    
+            
    
+            
    
+                
    [ Crawl4AI Marketplace · Updated -- ]
    
+            
    
+        
    
+    
    
+
+    
+
+
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/frontend/marketplace.css b/docs/md_v2/marketplace/frontend/marketplace.css
new file mode 100644
index 0000000..ad26c34
--- /dev/null
+++ b/docs/md_v2/marketplace/frontend/marketplace.css
@@ -0,0 +1,957 @@
+/* Marketplace CSS - Magazine Style Terminal Theme */
+@import url('../../assets/styles.css');
+
+:root {
+    --primary-cyan: #50ffff;
+    --primary-teal: #09b5a5;
+    --accent-pink: #f380f5;
+    --bg-dark: #070708;
+    --bg-secondary: #1a1a1a;
+    --bg-tertiary: #3f3f44;
+    --text-primary: #e8e9ed;
+    --text-secondary: #d5cec0;
+    --text-tertiary: #a3abba;
+    --border-color: #3f3f44;
+    --success: #50ff50;
+    --error: #ff3c74;
+    --warning: #f59e0b;
+}
+
+* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    font-family: 'Dank Mono', Monaco, monospace;
+    background: var(--bg-dark);
+    color: var(--text-primary);
+    line-height: 1.6;
+}
+
+/* Global link styles */
+a {
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: color 0.2s;
+}
+
+a:hover {
+    color: var(--accent-pink);
+}
+
+.marketplace-container {
+    min-height: 100vh;
+}
+
+/* Header */
+.marketplace-header {
+    background: var(--bg-secondary);
+    border-bottom: 1px solid var(--border-color);
+    padding: 1.5rem 0;
+}
+
+.header-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.logo-title {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.header-logo {
+    height: 40px;
+    width: auto;
+    filter: brightness(1.2);
+}
+
+.marketplace-header h1 {
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    margin: 0;
+}
+
+.ascii-border {
+    color: var(--border-color);
+}
+
+.tagline {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+    margin-top: 0.25rem;
+}
+
+.header-stats {
+    display: flex;
+    gap: 2rem;
+}
+
+.stat-item {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+}
+
+.stat-item span {
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+/* Search and Filter Bar */
+.search-filter-bar {
+    max-width: 1800px;
+    margin: 1.5rem auto;
+    padding: 0 2rem;
+    display: flex;
+    gap: 1rem;
+    align-items: center;
+}
+
+.search-box {
+    flex: 1;
+    max-width: 500px;
+    display: flex;
+    align-items: center;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 0.75rem 1rem;
+    transition: border-color 0.2s;
+}
+
+.search-box:focus-within {
+    border-color: var(--primary-cyan);
+}
+
+.search-icon {
+    color: var(--text-tertiary);
+    margin-right: 1rem;
+}
+
+#search-input {
+    flex: 1;
+    background: transparent;
+    border: none;
+    color: var(--text-primary);
+    font-family: inherit;
+    font-size: 0.9rem;
+    outline: none;
+}
+
+.search-box kbd {
+    font-size: 0.75rem;
+    padding: 0.2rem 0.5rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-tertiary);
+}
+
+.category-filter {
+    display: flex;
+    gap: 0.5rem;
+    flex-wrap: wrap;
+}
+
+.filter-btn {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    padding: 0.5rem 1rem;
+    font-family: inherit;
+    font-size: 0.875rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.filter-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+.filter-btn.active {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+    border-color: var(--primary-cyan);
+}
+
+/* Magazine Layout */
+.magazine-layout {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem 4rem;
+    display: grid;
+    grid-template-columns: 1fr;
+    gap: 2rem;
+}
+
+/* Hero Featured Section */
+.hero-featured {
+    grid-column: 1 / -1;
+    position: relative;
+}
+
+.hero-featured::before {
+    content: '';
+    position: absolute;
+    top: -20px;
+    left: -20px;
+    right: -20px;
+    bottom: -20px;
+    background: radial-gradient(ellipse at center, rgba(80, 255, 255, 0.05), transparent 70%);
+    pointer-events: none;
+    z-index: -1;
+}
+
+.featured-hero-card {
+    background: linear-gradient(135deg, #1a1a2e, #0f0f1e);
+    border: 2px solid var(--primary-cyan);
+    box-shadow: 0 0 30px rgba(80, 255, 255, 0.15),
+                inset 0 0 20px rgba(80, 255, 255, 0.05);
+    height: 380px;
+    position: relative;
+    overflow: hidden;
+    cursor: pointer;
+    transition: all 0.3s ease;
+    display: flex;
+    flex-direction: column;
+}
+
+.featured-hero-card:hover {
+    border-color: var(--accent-pink);
+    box-shadow: 0 0 40px rgba(243, 128, 245, 0.2),
+                inset 0 0 30px rgba(243, 128, 245, 0.05);
+    transform: translateY(-2px);
+}
+
+.hero-image {
+    width: 100%;
+    height: 240px;
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05));
+    background-size: cover;
+    background-position: center;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 3rem;
+    color: var(--primary-cyan);
+    flex-shrink: 0;
+    position: relative;
+    filter: brightness(1.1) contrast(1.1);
+}
+
+.hero-image::after {
+    content: '';
+    position: absolute;
+    bottom: 0;
+    left: 0;
+    right: 0;
+    height: 60%;
+    background: linear-gradient(to top, rgba(10, 10, 20, 0.95), transparent);
+}
+
+.hero-content {
+    padding: 1.5rem;
+}
+
+.hero-badge {
+    display: inline-block;
+    padding: 0.3rem 0.6rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    font-size: 0.7rem;
+    text-transform: uppercase;
+    margin-bottom: 0.5rem;
+    font-weight: 600;
+    box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3);
+}
+
+.hero-title {
+    font-size: 1.6rem;
+    color: var(--primary-cyan);
+    margin: 0.5rem 0;
+    text-shadow: 0 0 20px rgba(80, 255, 255, 0.5);
+}
+
+.hero-description {
+    color: var(--text-secondary);
+    line-height: 1.5;
+}
+
+.hero-meta {
+    display: flex;
+    gap: 1.5rem;
+    margin-top: 1rem;
+    font-size: 0.875rem;
+}
+
+.hero-meta span {
+    color: var(--text-tertiary);
+}
+
+.hero-meta span:first-child {
+    color: var(--warning);
+}
+
+/* Secondary Featured */
+.secondary-featured {
+    grid-column: 1 / -1;
+    height: 380px;
+    display: flex;
+    align-items: stretch;
+}
+
+.featured-secondary-cards {
+    width: 100%;
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+    justify-content: space-between;
+}
+
+.secondary-card {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02));
+    border: 1px solid rgba(80, 255, 255, 0.3);
+    cursor: pointer;
+    transition: all 0.3s ease;
+    display: flex;
+    overflow: hidden;
+    height: calc((380px - 1.5rem) / 3);
+    flex: 1;
+    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.3);
+}
+
+.secondary-card:hover {
+    border-color: var(--accent-pink);
+    background: linear-gradient(135deg, rgba(243, 128, 245, 0.05), rgba(80, 255, 255, 0.03));
+    box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2);
+    transform: translateX(-3px);
+}
+
+.secondary-image {
+    width: 120px;
+    background: linear-gradient(135deg, var(--bg-tertiary), var(--bg-secondary));
+    background-size: cover;
+    background-position: center;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    flex-shrink: 0;
+}
+
+.secondary-content {
+    flex: 1;
+    padding: 1rem;
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
+.secondary-title {
+    font-size: 1rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.secondary-desc {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+.secondary-meta {
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+}
+
+.secondary-meta span:last-child {
+    color: var(--warning);
+}
+
+/* Sponsored Section */
+.sponsored-section {
+    grid-column: 1 / -1;
+    background: var(--bg-secondary);
+    border: 1px solid var(--warning);
+    padding: 1rem;
+    position: relative;
+}
+
+.section-label {
+    position: absolute;
+    top: -0.5rem;
+    left: 1rem;
+    background: var(--bg-secondary);
+    padding: 0 0.5rem;
+    color: var(--warning);
+    font-size: 0.65rem;
+    letter-spacing: 0.1em;
+}
+
+.sponsored-cards {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+}
+
+.sponsor-card {
+    padding: 1rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+}
+
+.sponsor-card h4 {
+    color: var(--accent-pink);
+    margin-bottom: 0.5rem;
+}
+
+.sponsor-card p {
+    color: var(--text-secondary);
+    font-size: 0.85rem;
+    margin-bottom: 0.75rem;
+}
+
+.sponsor-card a {
+    color: var(--primary-cyan);
+    text-decoration: none;
+    font-size: 0.85rem;
+}
+
+.sponsor-card a:hover {
+    color: var(--accent-pink);
+}
+
+/* Main Content Grid */
+.main-content {
+    grid-column: 1 / -1;
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+    gap: 2rem;
+}
+
+/* Column Headers */
+.column-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+    border-bottom: 1px solid var(--border-color);
+    padding-bottom: 0.5rem;
+}
+
+.column-header h2 {
+    font-size: 1.1rem;
+    color: var(--text-primary);
+}
+
+.mini-filter {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    padding: 0.25rem 0.5rem;
+    font-family: inherit;
+    font-size: 0.75rem;
+}
+
+.ascii-icon {
+    color: var(--primary-cyan);
+}
+
+/* Apps Column */
+.apps-compact-grid {
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+}
+
+.app-compact {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    border-left: 3px solid var(--border-color);
+    padding: 0.75rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.app-compact:hover {
+    border-color: var(--primary-cyan);
+    border-left-color: var(--accent-pink);
+    transform: translateX(2px);
+}
+
+.app-compact-header {
+    display: flex;
+    justify-content: space-between;
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+    margin-bottom: 0.25rem;
+}
+
+.app-compact-header span:first-child {
+    color: var(--primary-cyan);
+}
+
+.app-compact-header span:last-child {
+    color: var(--warning);
+}
+
+.app-compact-title {
+    font-size: 0.9rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.app-compact-desc {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+/* Articles Column */
+.articles-compact-list {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+}
+
+.article-compact {
+    border-left: 2px solid var(--border-color);
+    padding-left: 1rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.article-compact:hover {
+    border-left-color: var(--primary-cyan);
+}
+
+.article-meta {
+    font-size: 0.7rem;
+    color: var(--text-tertiary);
+    margin-bottom: 0.25rem;
+}
+
+.article-meta span:first-child {
+    color: var(--accent-pink);
+}
+
+.article-title {
+    font-size: 0.9rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.article-author {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+}
+
+/* Trending Column */
+.trending-items {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.trending-item {
+    display: flex;
+    align-items: center;
+    gap: 0.75rem;
+    padding: 0.5rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.trending-item:hover {
+    border-color: var(--primary-cyan);
+}
+
+.trending-rank {
+    font-size: 1.2rem;
+    color: var(--primary-cyan);
+    width: 2rem;
+    text-align: center;
+}
+
+.trending-info {
+    flex: 1;
+}
+
+.trending-name {
+    font-size: 0.85rem;
+    color: var(--text-primary);
+}
+
+.trending-stats {
+    font-size: 0.7rem;
+    color: var(--text-tertiary);
+}
+
+/* Submit Box */
+.submit-box {
+    margin-top: 1.5rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--primary-cyan);
+    padding: 1rem;
+    text-align: center;
+}
+
+.submit-box h3 {
+    font-size: 1rem;
+    color: var(--primary-cyan);
+    margin-bottom: 0.5rem;
+}
+
+.submit-box p {
+    font-size: 0.8rem;
+    color: var(--text-secondary);
+    margin-bottom: 0.75rem;
+}
+
+.submit-btn {
+    display: inline-block;
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+}
+
+.submit-btn:hover {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+/* More Apps Section */
+.more-apps {
+    grid-column: 1 / -1;
+    margin-top: 2rem;
+}
+
+.section-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+}
+
+.more-apps-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
+    gap: 1rem;
+}
+
+.load-more-btn {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    padding: 0.5rem 1.5rem;
+    font-family: inherit;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.load-more-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Footer */
+.marketplace-footer {
+    background: var(--bg-secondary);
+    border-top: 1px solid var(--border-color);
+    margin-top: 4rem;
+    padding: 2rem 0;
+}
+
+.footer-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 2rem;
+}
+
+.footer-section h3 {
+    font-size: 1rem;
+    margin-bottom: 0.5rem;
+    color: var(--primary-cyan);
+}
+
+.footer-section p {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+    margin-bottom: 1rem;
+}
+
+.sponsor-btn {
+    display: inline-block;
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+}
+
+.sponsor-btn:hover {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+.footer-bottom {
+    max-width: 1800px;
+    margin: 2rem auto 0;
+    padding: 1rem 2rem 0;
+    border-top: 1px solid var(--border-color);
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+}
+
+/* Modal */
+.modal {
+    position: fixed;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background: rgba(0, 0, 0, 0.8);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    z-index: 1000;
+}
+
+.modal.hidden {
+    display: none;
+}
+
+.modal-content {
+    background: var(--bg-secondary);
+    border: 1px solid var(--primary-cyan);
+    max-width: 800px;
+    width: 90%;
+    max-height: 80vh;
+    overflow-y: auto;
+    position: relative;
+}
+
+.modal-close {
+    position: absolute;
+    top: 1rem;
+    right: 1rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    padding: 0.25rem 0.5rem;
+    cursor: pointer;
+    font-size: 1.2rem;
+}
+
+.modal-close:hover {
+    border-color: var(--error);
+    color: var(--error);
+}
+
+.app-detail {
+    padding: 2rem;
+}
+
+.app-detail h2 {
+    font-size: 1.5rem;
+    margin-bottom: 1rem;
+    color: var(--primary-cyan);
+}
+
+/* Loading */
+.loading {
+    text-align: center;
+    padding: 2rem;
+    color: var(--text-tertiary);
+}
+
+.no-results {
+    text-align: center;
+    padding: 2rem;
+    color: var(--text-tertiary);
+}
+
+/* Responsive - Tablet */
+@media (min-width: 768px) {
+    .magazine-layout {
+        grid-template-columns: repeat(2, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / -1;
+    }
+
+    .secondary-featured {
+        grid-column: 1 / -1;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .main-content {
+        grid-column: 1 / -1;
+        grid-template-columns: repeat(2, 1fr);
+    }
+}
+
+/* Responsive - Desktop */
+@media (min-width: 1024px) {
+    .magazine-layout {
+        grid-template-columns: repeat(3, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+        grid-row: 1;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 4;
+        grid-row: 1;
+    }
+
+    .featured-secondary-cards {
+        flex-direction: column;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .main-content {
+        grid-column: 1 / -1;
+        grid-template-columns: repeat(3, 1fr);
+    }
+}
+
+/* Responsive - Wide Desktop */
+@media (min-width: 1400px) {
+    .magazine-layout {
+        grid-template-columns: repeat(4, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 5;
+        grid-row: 1;
+    }
+
+    .featured-secondary-cards {
+        grid-template-columns: repeat(2, 1fr);
+    }
+
+    .main-content {
+        grid-template-columns: repeat(4, 1fr);
+    }
+
+    .apps-column {
+        grid-column: span 2;
+    }
+
+    .more-apps-grid {
+        grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+    }
+}
+
+/* Responsive - Ultra Wide Desktop (for coders with wide monitors) */
+@media (min-width: 1800px) {
+    .magazine-layout {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 6;
+    }
+
+    .featured-secondary-cards {
+        grid-template-columns: repeat(3, 1fr);
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .sponsored-cards {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .main-content {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .apps-column {
+        grid-column: span 2;
+    }
+
+    .articles-column {
+        grid-column: span 2;
+    }
+
+    .more-apps-grid {
+        grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
+    }
+}
+
+/* Responsive - Mobile */
+@media (max-width: 767px) {
+    .header-content {
+        flex-direction: column;
+        gap: 1rem;
+    }
+
+    .search-filter-bar {
+        flex-direction: column;
+        align-items: stretch;
+    }
+
+    .search-box {
+        max-width: none;
+    }
+
+    .magazine-layout {
+        padding: 0 1rem 2rem;
+    }
+
+    .footer-content {
+        grid-template-columns: 1fr;
+    }
+
+    .secondary-card {
+        flex-direction: column;
+    }
+
+    .secondary-image {
+        width: 100%;
+        height: 150px;
+    }
+}
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/frontend/marketplace.js b/docs/md_v2/marketplace/frontend/marketplace.js
new file mode 100644
index 0000000..df07257
--- /dev/null
+++ b/docs/md_v2/marketplace/frontend/marketplace.js
@@ -0,0 +1,395 @@
+// Marketplace JS - Magazine Layout
+const API_BASE = '/marketplace/api';
+const CACHE_TTL = 3600000; // 1 hour in ms
+
+class MarketplaceCache {
+    constructor() {
+        this.prefix = 'c4ai_market_';
+    }
+
+    get(key) {
+        const item = localStorage.getItem(this.prefix + key);
+        if (!item) return null;
+
+        const data = JSON.parse(item);
+        if (Date.now() > data.expires) {
+            localStorage.removeItem(this.prefix + key);
+            return null;
+        }
+        return data.value;
+    }
+
+    set(key, value, ttl = CACHE_TTL) {
+        const data = {
+            value: value,
+            expires: Date.now() + ttl
+        };
+        localStorage.setItem(this.prefix + key, JSON.stringify(data));
+    }
+
+    clear() {
+        Object.keys(localStorage)
+            .filter(k => k.startsWith(this.prefix))
+            .forEach(k => localStorage.removeItem(k));
+    }
+}
+
+class MarketplaceAPI {
+    constructor() {
+        this.cache = new MarketplaceCache();
+        this.searchTimeout = null;
+    }
+
+    async fetch(endpoint, useCache = true) {
+        const cacheKey = endpoint.replace(/[^\w]/g, '_');
+
+        if (useCache) {
+            const cached = this.cache.get(cacheKey);
+            if (cached) return cached;
+        }
+
+        try {
+            const response = await fetch(`${API_BASE}${endpoint}`);
+            if (!response.ok) throw new Error(`HTTP ${response.status}`);
+
+            const data = await response.json();
+            this.cache.set(cacheKey, data);
+            return data;
+        } catch (error) {
+            console.error('API Error:', error);
+            return null;
+        }
+    }
+
+    async getStats() {
+        return this.fetch('/stats');
+    }
+
+    async getCategories() {
+        return this.fetch('/categories');
+    }
+
+    async getApps(params = {}) {
+        const query = new URLSearchParams(params).toString();
+        return this.fetch(`/apps${query ? '?' + query : ''}`);
+    }
+
+    async getArticles(params = {}) {
+        const query = new URLSearchParams(params).toString();
+        return this.fetch(`/articles${query ? '?' + query : ''}`);
+    }
+
+    async getSponsors() {
+        return this.fetch('/sponsors');
+    }
+
+    async search(query) {
+        if (query.length < 2) return {};
+        return this.fetch(`/search?q=${encodeURIComponent(query)}`, false);
+    }
+}
+
+class MarketplaceUI {
+    constructor() {
+        this.api = new MarketplaceAPI();
+        this.currentCategory = 'all';
+        this.currentType = '';
+        this.searchTimeout = null;
+        this.loadedApps = 10;
+        this.init();
+    }
+
+    async init() {
+        await this.loadStats();
+        await this.loadCategories();
+        await this.loadFeaturedContent();
+        await this.loadSponsors();
+        await this.loadMainContent();
+        this.setupEventListeners();
+    }
+
+    async loadStats() {
+        const stats = await this.api.getStats();
+        if (stats) {
+            document.getElementById('total-apps').textContent = stats.total_apps || '0';
+            document.getElementById('total-articles').textContent = stats.total_articles || '0';
+            document.getElementById('total-downloads').textContent = stats.total_downloads || '0';
+            document.getElementById('last-update').textContent = new Date().toLocaleDateString();
+        }
+    }
+
+    async loadCategories() {
+        const categories = await this.api.getCategories();
+        if (!categories) return;
+
+        const filter = document.getElementById('category-filter');
+        categories.forEach(cat => {
+            const btn = document.createElement('button');
+            btn.className = 'filter-btn';
+            btn.dataset.category = cat.slug;
+            btn.textContent = cat.name;
+            btn.onclick = () => this.filterByCategory(cat.slug);
+            filter.appendChild(btn);
+        });
+    }
+
+    async loadFeaturedContent() {
+        // Load hero featured
+        const featured = await this.api.getApps({ featured: true, limit: 4 });
+        if (!featured || !featured.length) return;
+
+        // Hero card (first featured)
+        const hero = featured[0];
+        const heroCard = document.getElementById('featured-hero');
+        if (hero) {
+            const imageUrl = hero.image || '';
+            heroCard.innerHTML = `
+                
    
+                    ${!imageUrl ? `[${hero.category || 'APP'}]` : ''}
+                
    
+                
    
+                    ${hero.type || 'PAID'}
+                    
    ${hero.name}
    
+                    
    ${hero.description}
    
+                    
    
+                        ★ ${hero.rating || 0}/5
+                        ${hero.downloads || 0} downloads
+                    
    
+                
    
+            `;
+            heroCard.onclick = () => this.showAppDetail(hero);
+        }
+
+        // Secondary featured cards
+        const secondary = document.getElementById('featured-secondary');
+        secondary.innerHTML = '';
+        if (featured.length > 1) {
+            featured.slice(1, 4).forEach(app => {
+                const card = document.createElement('div');
+                card.className = 'secondary-card';
+                const imageUrl = app.image || '';
+                card.innerHTML = `
+                    
    
+                        ${!imageUrl ? `[${app.category || 'APP'}]` : ''}
+                    
    
+                    
    
+                        
    ${app.name}
    
+                        
    ${(app.description || '').substring(0, 100)}...
    
+                        
    
+                            ${app.type || 'Open Source'} · ★ ${app.rating || 0}/5
+                        
    
+                    
    
+                `;
+                card.onclick = () => this.showAppDetail(app);
+                secondary.appendChild(card);
+            });
+        }
+    }
+
+    async loadSponsors() {
+        const sponsors = await this.api.getSponsors();
+        if (!sponsors || !sponsors.length) {
+            // Show placeholder if no sponsors
+            const container = document.getElementById('sponsored-content');
+            container.innerHTML = `
+                
+            `;
+            return;
+        }
+
+        const container = document.getElementById('sponsored-content');
+        container.innerHTML = sponsors.slice(0, 5).map(sponsor => `
+            
+        `).join('');
+    }
+
+    async loadMainContent() {
+        // Load apps column
+        const apps = await this.api.getApps({ limit: 8 });
+        if (apps && apps.length) {
+            const appsGrid = document.getElementById('apps-grid');
+            appsGrid.innerHTML = apps.map(app => `
+                
    
+                    
    
+                        ${app.category}
+                        ★ ${app.rating}/5
+                    
    
+                    
    ${app.name}
    
+                    
    ${app.description}
    
+                
    
+            `).join('');
+        }
+
+        // Load articles column
+        const articles = await this.api.getArticles({ limit: 6 });
+        if (articles && articles.length) {
+            const articlesList = document.getElementById('articles-list');
+            articlesList.innerHTML = articles.map(article => `
+                
    
+                    
    
+                        ${article.category} · ${new Date(article.published_at).toLocaleDateString()}
+                    
    
+                    
    ${article.title}
    
+                    
    by ${article.author}
    
+                
    
+            `).join('');
+        }
+
+        // Load trending
+        if (apps && apps.length) {
+            const trending = apps.slice(0, 5);
+            const trendingList = document.getElementById('trending-list');
+            trendingList.innerHTML = trending.map((app, i) => `
+                
    
+                    
    ${i + 1}
    
+                    
    
+                        
    ${app.name}
    
+                        
    ${app.downloads} downloads
    
+                    
    
+                
    
+            `).join('');
+        }
+
+        // Load more apps grid
+        const moreApps = await this.api.getApps({ offset: 8, limit: 12 });
+        if (moreApps && moreApps.length) {
+            const moreGrid = document.getElementById('more-apps-grid');
+            moreGrid.innerHTML = moreApps.map(app => `
+                
    
+                    
    
+                        ${app.category}
+                        ${app.type}
+                    
    
+                    
    ${app.name}
    
+                
    
+            `).join('');
+        }
+    }
+
+    setupEventListeners() {
+        // Search
+        const searchInput = document.getElementById('search-input');
+        searchInput.addEventListener('input', (e) => {
+            clearTimeout(this.searchTimeout);
+            this.searchTimeout = setTimeout(() => this.search(e.target.value), 300);
+        });
+
+        // Keyboard shortcut
+        document.addEventListener('keydown', (e) => {
+            if (e.key === '/' && !searchInput.contains(document.activeElement)) {
+                e.preventDefault();
+                searchInput.focus();
+            }
+            if (e.key === 'Escape' && searchInput.contains(document.activeElement)) {
+                searchInput.blur();
+                searchInput.value = '';
+            }
+        });
+
+        // Type filter
+        const typeFilter = document.getElementById('type-filter');
+        typeFilter.addEventListener('change', (e) => {
+            this.currentType = e.target.value;
+            this.loadMainContent();
+        });
+
+        // Load more
+        const loadMore = document.getElementById('load-more');
+        loadMore.addEventListener('click', () => this.loadMoreApps());
+    }
+
+    async filterByCategory(category) {
+        // Update active state
+        document.querySelectorAll('.filter-btn').forEach(btn => {
+            btn.classList.toggle('active', btn.dataset.category === category);
+        });
+
+        this.currentCategory = category;
+        await this.loadMainContent();
+    }
+
+    async search(query) {
+        if (!query) {
+            await this.loadMainContent();
+            return;
+        }
+
+        const results = await this.api.search(query);
+        if (!results) return;
+
+        // Update apps grid with search results
+        if (results.apps && results.apps.length) {
+            const appsGrid = document.getElementById('apps-grid');
+            appsGrid.innerHTML = results.apps.map(app => `
+                
    
+                    
    
+                        ${app.category}
+                        ★ ${app.rating}/5
+                    
    
+                    
    ${app.name}
    
+                    
    ${app.description}
    
+                
    
+            `).join('');
+        }
+
+        // Update articles with search results
+        if (results.articles && results.articles.length) {
+            const articlesList = document.getElementById('articles-list');
+            articlesList.innerHTML = results.articles.map(article => `
+                
    
+                    
    
+                        ${article.category} · ${new Date(article.published_at).toLocaleDateString()}
+                    
    
+                    
    ${article.title}
    
+                    
    by ${article.author}
    
+                
    
+            `).join('');
+        }
+    }
+
+    async loadMoreApps() {
+        this.loadedApps += 12;
+        const moreApps = await this.api.getApps({ offset: this.loadedApps, limit: 12 });
+        if (moreApps && moreApps.length) {
+            const moreGrid = document.getElementById('more-apps-grid');
+            moreApps.forEach(app => {
+                const card = document.createElement('div');
+                card.className = 'app-compact';
+                card.innerHTML = `
+                    
    
+                        ${app.category}
+                        ${app.type}
+                    
    
+                    
    ${app.name}
    
+                `;
+                card.onclick = () => this.showAppDetail(app);
+                moreGrid.appendChild(card);
+            });
+        }
+    }
+
+    showAppDetail(app) {
+        // Navigate to detail page instead of showing modal
+        const slug = app.slug || app.name.toLowerCase().replace(/\s+/g, '-');
+        window.location.href = `app-detail.html?app=${slug}`;
+    }
+
+    showArticle(articleId) {
+        // Could create article detail page similarly
+        console.log('Show article:', articleId);
+    }
+}
+
+// Initialize marketplace
+let marketplace;
+document.addEventListener('DOMContentLoaded', () => {
+    marketplace = new MarketplaceUI();
+});
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/index.html b/docs/md_v2/marketplace/index.html
new file mode 100644
index 0000000..c425420
--- /dev/null
+++ b/docs/md_v2/marketplace/index.html
@@ -0,0 +1,147 @@
+
+
+
+    
+    
+    Marketplace - Crawl4AI
+    
+
+
+    
    
+        
+        
    
+            
    
+                
    
+                    
    
+                        
+                        
    
+                            [
+                            Marketplace
+                            ]
+                        
    
+                    
    
+                    
    Tools, Integrations & Resources for Web Crawling
    
+                
    
+                
    
+                    Apps: --
+                    Articles: --
+                    Downloads: --
+                
    
+            
    
+        
    
+
+        
+        
    
+            
    
+                >
+                
+                /
+            
    
+            
    
+                
+                
+            
    
+        
    
+
+        
+        
    
+            
+            
    
+                
+            
    
+
+            
+            
    
+                
+            
    
+
+            
+            
+
+            
+            
    
+                
+                
    
+                    
    
+                        
    > Latest Apps
    
+                        
+                    
    
+                    
    
+                        
+                    
    
+                
    
+
+                
+                
    
+                    
    
+                        
    > Latest Articles
    
+                    
    
+                    
    
+                        
+                    
    
+                
    
+
+                
+                
    
+                    
    
+                        
    # Trending
    
+                    
    
+                    
+
+                    
    
+                        
    + Submit Your Tool
    
+                        
    Share your integration
    
+                        Submit →
+                    
    
+                
    
+            
    
+
+            
+            
    
+                
    
+                    
    > More Apps
    
+                    
+                
    
+                
    
+                    
+                
    
+            
    
+        
    
+
+        
+        
    
+            
    
+                
    
+                    
    About Marketplace
    
+                    
    Discover tools and integrations built by the Crawl4AI community.
    
+                
    
+                
    
+                    
    Become a Sponsor
    
+                    
    Reach developers building with Crawl4AI
    
+                    
+                
    
+            
    
+            
    
+                
    [ Crawl4AI Marketplace · Updated -- ]
    
+            
    
+        
    
+    
    
+
+    
+
+
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/marketplace.css b/docs/md_v2/marketplace/marketplace.css
new file mode 100644
index 0000000..57f54d2
--- /dev/null
+++ b/docs/md_v2/marketplace/marketplace.css
@@ -0,0 +1,994 @@
+/* Marketplace CSS - Magazine Style Terminal Theme */
+@import url('../../assets/styles.css');
+
+:root {
+    --primary-cyan: #50ffff;
+    --primary-teal: #09b5a5;
+    --accent-pink: #f380f5;
+    --bg-dark: #070708;
+    --bg-secondary: #1a1a1a;
+    --bg-tertiary: #3f3f44;
+    --text-primary: #e8e9ed;
+    --text-secondary: #d5cec0;
+    --text-tertiary: #a3abba;
+    --border-color: #3f3f44;
+    --success: #50ff50;
+    --error: #ff3c74;
+    --warning: #f59e0b;
+}
+
+* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    font-family: 'Dank Mono', Monaco, monospace;
+    background: var(--bg-dark);
+    color: var(--text-primary);
+    line-height: 1.6;
+}
+
+/* Global link styles */
+a {
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: color 0.2s;
+}
+
+a:hover {
+    color: var(--accent-pink);
+}
+
+.marketplace-container {
+    min-height: 100vh;
+}
+
+/* Header */
+.marketplace-header {
+    background: var(--bg-secondary);
+    border-bottom: 1px solid var(--border-color);
+    padding: 1.5rem 0;
+}
+
+.header-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.logo-title {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.header-logo {
+    height: 40px;
+    width: auto;
+    filter: brightness(1.2);
+}
+
+.marketplace-header h1 {
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    margin: 0;
+}
+
+.ascii-border {
+    color: var(--border-color);
+}
+
+.tagline {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+    margin-top: 0.25rem;
+}
+
+.header-stats {
+    display: flex;
+    gap: 2rem;
+}
+
+.stat-item {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+}
+
+.stat-item span {
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+/* Search and Filter Bar */
+.search-filter-bar {
+    max-width: 1800px;
+    margin: 1.5rem auto;
+    padding: 0 2rem;
+    display: flex;
+    gap: 1rem;
+    align-items: center;
+}
+
+.search-box {
+    flex: 1;
+    max-width: 500px;
+    display: flex;
+    align-items: center;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 0.75rem 1rem;
+    transition: border-color 0.2s;
+}
+
+.search-box:focus-within {
+    border-color: var(--primary-cyan);
+}
+
+.search-icon {
+    color: var(--text-tertiary);
+    margin-right: 1rem;
+}
+
+#search-input {
+    flex: 1;
+    background: transparent;
+    border: none;
+    color: var(--text-primary);
+    font-family: inherit;
+    font-size: 0.9rem;
+    outline: none;
+}
+
+.search-box kbd {
+    font-size: 0.75rem;
+    padding: 0.2rem 0.5rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-tertiary);
+}
+
+.category-filter {
+    display: flex;
+    gap: 0.5rem;
+    flex-wrap: wrap;
+}
+
+.filter-btn {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    padding: 0.5rem 1rem;
+    font-family: inherit;
+    font-size: 0.875rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.filter-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+.filter-btn.active {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+    border-color: var(--primary-cyan);
+}
+
+/* Magazine Layout */
+.magazine-layout {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem 4rem;
+    display: grid;
+    grid-template-columns: 1fr;
+    gap: 2rem;
+}
+
+/* Hero Featured Section */
+.hero-featured {
+    grid-column: 1 / -1;
+    position: relative;
+}
+
+.hero-featured::before {
+    content: '';
+    position: absolute;
+    top: -20px;
+    left: -20px;
+    right: -20px;
+    bottom: -20px;
+    background: radial-gradient(ellipse at center, rgba(80, 255, 255, 0.05), transparent 70%);
+    pointer-events: none;
+    z-index: -1;
+}
+
+.featured-hero-card {
+    background: linear-gradient(135deg, #1a1a2e, #0f0f1e);
+    border: 2px solid var(--primary-cyan);
+    box-shadow: 0 0 30px rgba(80, 255, 255, 0.15),
+                inset 0 0 20px rgba(80, 255, 255, 0.05);
+    height: 380px;
+    position: relative;
+    overflow: hidden;
+    cursor: pointer;
+    transition: all 0.3s ease;
+    display: flex;
+    flex-direction: column;
+}
+
+.featured-hero-card:hover {
+    border-color: var(--accent-pink);
+    box-shadow: 0 0 40px rgba(243, 128, 245, 0.2),
+                inset 0 0 30px rgba(243, 128, 245, 0.05);
+    transform: translateY(-2px);
+}
+
+.hero-image {
+    width: 100%;
+    height: 200px;
+    min-height: 200px;
+    max-height: 200px;
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05));
+    background-size: cover;
+    background-position: center;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 3rem;
+    color: var(--primary-cyan);
+    flex-shrink: 0;
+    position: relative;
+    filter: brightness(1.1) contrast(1.1);
+    overflow: hidden;
+}
+
+.hero-image img {
+    width: 100%;
+    height: 100%;
+    object-fit: cover;
+    object-position: center;
+}
+
+.hero-image::after {
+    content: '';
+    position: absolute;
+    bottom: 0;
+    left: 0;
+    right: 0;
+    height: 60%;
+    background: linear-gradient(to top, rgba(10, 10, 20, 0.95), transparent);
+}
+
+.hero-content {
+    padding: 1.5rem;
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
+.hero-badge {
+    display: inline-block;
+    padding: 0.3rem 0.6rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    font-size: 0.7rem;
+    text-transform: uppercase;
+    margin-bottom: 0.5rem;
+    font-weight: 600;
+    box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3);
+}
+
+.hero-title {
+    font-size: 1.6rem;
+    color: var(--primary-cyan);
+    margin: 0.5rem 0;
+    text-shadow: 0 0 20px rgba(80, 255, 255, 0.5);
+}
+
+.hero-description {
+    color: var(--text-secondary);
+    line-height: 1.5;
+}
+
+.hero-meta {
+    display: flex;
+    gap: 1.5rem;
+    margin-top: 1rem;
+    font-size: 0.875rem;
+}
+
+.hero-meta span {
+    color: var(--text-tertiary);
+}
+
+.hero-meta span:first-child {
+    color: var(--warning);
+}
+
+/* Secondary Featured */
+.secondary-featured {
+    grid-column: 1 / -1;
+    min-height: 380px;
+    display: flex;
+    align-items: flex-start;
+}
+
+.featured-secondary-cards {
+    width: 100%;
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+    align-items: stretch;
+}
+
+.secondary-card {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02));
+    border: 1px solid rgba(80, 255, 255, 0.3);
+    cursor: pointer;
+    transition: all 0.3s ease;
+    display: flex;
+    overflow: hidden;
+    height: 118px;
+    min-height: 118px;
+    max-height: 118px;
+    flex-shrink: 0;
+    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.3);
+}
+
+.secondary-card:hover {
+    border-color: var(--accent-pink);
+    background: linear-gradient(135deg, rgba(243, 128, 245, 0.05), rgba(80, 255, 255, 0.03));
+    box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2);
+    transform: translateX(-3px);
+}
+
+.secondary-image {
+    width: 120px;
+    background: linear-gradient(135deg, var(--bg-tertiary), var(--bg-secondary));
+    background-size: cover;
+    background-position: center;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    flex-shrink: 0;
+}
+
+.secondary-content {
+    flex: 1;
+    padding: 1rem;
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
+.secondary-title {
+    font-size: 1rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.secondary-desc {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+.secondary-meta {
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+}
+
+.secondary-meta span:last-child {
+    color: var(--warning);
+}
+
+/* Sponsored Section */
+.sponsored-section {
+    grid-column: 1 / -1;
+    background: var(--bg-secondary);
+    border: 1px solid var(--warning);
+    padding: 1rem;
+    position: relative;
+}
+
+.section-label {
+    position: absolute;
+    top: -0.5rem;
+    left: 1rem;
+    background: var(--bg-secondary);
+    padding: 0 0.5rem;
+    color: var(--warning);
+    font-size: 0.65rem;
+    letter-spacing: 0.1em;
+}
+
+.sponsored-cards {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+}
+
+.sponsor-card {
+    padding: 1rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+}
+
+.sponsor-logo {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 60px;
+    margin-bottom: 0.75rem;
+}
+
+.sponsor-logo img {
+    max-height: 60px;
+    max-width: 100%;
+    width: auto;
+    object-fit: contain;
+}
+
+.sponsor-card h4 {
+    color: var(--accent-pink);
+    margin-bottom: 0.5rem;
+}
+
+.sponsor-card p {
+    color: var(--text-secondary);
+    font-size: 0.85rem;
+    margin-bottom: 0.75rem;
+}
+
+.sponsor-card a {
+    color: var(--primary-cyan);
+    text-decoration: none;
+    font-size: 0.85rem;
+}
+
+.sponsor-card a:hover {
+    color: var(--accent-pink);
+}
+
+/* Main Content Grid */
+.main-content {
+    grid-column: 1 / -1;
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+    gap: 2rem;
+}
+
+/* Column Headers */
+.column-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+    border-bottom: 1px solid var(--border-color);
+    padding-bottom: 0.5rem;
+}
+
+.column-header h2 {
+    font-size: 1.1rem;
+    color: var(--text-primary);
+}
+
+.mini-filter {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    padding: 0.25rem 0.5rem;
+    font-family: inherit;
+    font-size: 0.75rem;
+}
+
+.ascii-icon {
+    color: var(--primary-cyan);
+}
+
+/* Apps Column */
+.apps-compact-grid {
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+}
+
+.app-compact {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    border-left: 3px solid var(--border-color);
+    padding: 0.75rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.app-compact:hover {
+    border-color: var(--primary-cyan);
+    border-left-color: var(--accent-pink);
+    transform: translateX(2px);
+}
+
+.app-compact-header {
+    display: flex;
+    justify-content: space-between;
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+    margin-bottom: 0.25rem;
+}
+
+.app-compact-header span:first-child {
+    color: var(--primary-cyan);
+}
+
+.app-compact-header span:last-child {
+    color: var(--warning);
+}
+
+.app-compact-title {
+    font-size: 0.9rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.app-compact-desc {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+/* Articles Column */
+.articles-compact-list {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+}
+
+.article-compact {
+    border-left: 2px solid var(--border-color);
+    padding-left: 1rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.article-compact:hover {
+    border-left-color: var(--primary-cyan);
+}
+
+.article-meta {
+    font-size: 0.7rem;
+    color: var(--text-tertiary);
+    margin-bottom: 0.25rem;
+}
+
+.article-meta span:first-child {
+    color: var(--accent-pink);
+}
+
+.article-title {
+    font-size: 0.9rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.article-author {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+}
+
+/* Trending Column */
+.trending-items {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.trending-item {
+    display: flex;
+    align-items: center;
+    gap: 0.75rem;
+    padding: 0.5rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.trending-item:hover {
+    border-color: var(--primary-cyan);
+}
+
+.trending-rank {
+    font-size: 1.2rem;
+    color: var(--primary-cyan);
+    width: 2rem;
+    text-align: center;
+}
+
+.trending-info {
+    flex: 1;
+}
+
+.trending-name {
+    font-size: 0.85rem;
+    color: var(--text-primary);
+}
+
+.trending-stats {
+    font-size: 0.7rem;
+    color: var(--text-tertiary);
+}
+
+/* Submit Box */
+.submit-box {
+    margin-top: 1.5rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--primary-cyan);
+    padding: 1rem;
+    text-align: center;
+}
+
+.submit-box h3 {
+    font-size: 1rem;
+    color: var(--primary-cyan);
+    margin-bottom: 0.5rem;
+}
+
+.submit-box p {
+    font-size: 0.8rem;
+    color: var(--text-secondary);
+    margin-bottom: 0.75rem;
+}
+
+.submit-btn {
+    display: inline-block;
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+}
+
+.submit-btn:hover {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+/* More Apps Section */
+.more-apps {
+    grid-column: 1 / -1;
+    margin-top: 2rem;
+}
+
+.section-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+}
+
+.more-apps-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
+    gap: 1rem;
+}
+
+.load-more-btn {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    padding: 0.5rem 1.5rem;
+    font-family: inherit;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.load-more-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Footer */
+.marketplace-footer {
+    background: var(--bg-secondary);
+    border-top: 1px solid var(--border-color);
+    margin-top: 4rem;
+    padding: 2rem 0;
+}
+
+.footer-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 2rem;
+}
+
+.footer-section h3 {
+    font-size: 1rem;
+    margin-bottom: 0.5rem;
+    color: var(--primary-cyan);
+}
+
+.footer-section p {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+    margin-bottom: 1rem;
+}
+
+.sponsor-btn {
+    display: inline-block;
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+}
+
+.sponsor-btn:hover {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+.footer-bottom {
+    max-width: 1800px;
+    margin: 2rem auto 0;
+    padding: 1rem 2rem 0;
+    border-top: 1px solid var(--border-color);
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+}
+
+/* Modal */
+.modal {
+    position: fixed;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background: rgba(0, 0, 0, 0.8);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    z-index: 1000;
+}
+
+.modal.hidden {
+    display: none;
+}
+
+.modal-content {
+    background: var(--bg-secondary);
+    border: 1px solid var(--primary-cyan);
+    max-width: 800px;
+    width: 90%;
+    max-height: 80vh;
+    overflow-y: auto;
+    position: relative;
+}
+
+.modal-close {
+    position: absolute;
+    top: 1rem;
+    right: 1rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    padding: 0.25rem 0.5rem;
+    cursor: pointer;
+    font-size: 1.2rem;
+}
+
+.modal-close:hover {
+    border-color: var(--error);
+    color: var(--error);
+}
+
+.app-detail {
+    padding: 2rem;
+}
+
+.app-detail h2 {
+    font-size: 1.5rem;
+    margin-bottom: 1rem;
+    color: var(--primary-cyan);
+}
+
+/* Loading */
+.loading {
+    text-align: center;
+    padding: 2rem;
+    color: var(--text-tertiary);
+}
+
+.no-results {
+    text-align: center;
+    padding: 2rem;
+    color: var(--text-tertiary);
+}
+
+/* Responsive - Tablet */
+@media (min-width: 768px) {
+    .magazine-layout {
+        grid-template-columns: repeat(2, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / -1;
+    }
+
+    .secondary-featured {
+        grid-column: 1 / -1;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .main-content {
+        grid-column: 1 / -1;
+        grid-template-columns: repeat(2, 1fr);
+    }
+}
+
+/* Responsive - Desktop */
+@media (min-width: 1024px) {
+    .magazine-layout {
+        grid-template-columns: repeat(3, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+        grid-row: 1;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 4;
+        grid-row: 1;
+    }
+
+    .featured-secondary-cards {
+        flex-direction: column;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .main-content {
+        grid-column: 1 / -1;
+        grid-template-columns: repeat(3, 1fr);
+    }
+}
+
+/* Responsive - Wide Desktop */
+@media (min-width: 1400px) {
+    .magazine-layout {
+        grid-template-columns: repeat(4, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 5;
+        grid-row: 1;
+        min-height: auto;
+    }
+
+    .featured-secondary-cards {
+        display: grid;
+        grid-template-columns: repeat(2, 1fr);
+        flex-direction: unset;
+    }
+
+    .main-content {
+        grid-template-columns: repeat(4, 1fr);
+    }
+
+    .apps-column {
+        grid-column: span 2;
+    }
+
+    .more-apps-grid {
+        grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+    }
+}
+
+/* Responsive - Ultra Wide Desktop (for coders with wide monitors) */
+@media (min-width: 1800px) {
+    .magazine-layout {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 6;
+        min-height: auto;
+    }
+
+    .featured-secondary-cards {
+        display: grid;
+        grid-template-columns: repeat(3, 1fr);
+        flex-direction: unset;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .sponsored-cards {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .main-content {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .apps-column {
+        grid-column: span 2;
+    }
+
+    .articles-column {
+        grid-column: span 2;
+    }
+
+    .more-apps-grid {
+        grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
+    }
+}
+
+/* Responsive - Mobile */
+@media (max-width: 767px) {
+    .header-content {
+        flex-direction: column;
+        gap: 1rem;
+    }
+
+    .search-filter-bar {
+        flex-direction: column;
+        align-items: stretch;
+    }
+
+    .search-box {
+        max-width: none;
+    }
+
+    .magazine-layout {
+        padding: 0 1rem 2rem;
+    }
+
+    .footer-content {
+        grid-template-columns: 1fr;
+    }
+
+    .secondary-card {
+        flex-direction: column;
+    }
+
+    .secondary-image {
+        width: 100%;
+        height: 150px;
+    }
+}
\ No newline at end of file
diff --git a/docs/md_v2/marketplace/marketplace.js b/docs/md_v2/marketplace/marketplace.js
new file mode 100644
index 0000000..7813e3b
--- /dev/null
+++ b/docs/md_v2/marketplace/marketplace.js
@@ -0,0 +1,412 @@
+// Marketplace JS - Magazine Layout
+const { API_BASE, API_ORIGIN } = (() => {
+    const { hostname, port } = window.location;
+    if ((hostname === 'localhost' || hostname === '127.0.0.1') && port === '8000') {
+        const origin = 'http://127.0.0.1:8100';
+        return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin };
+    }
+    return { API_BASE: '/marketplace/api', API_ORIGIN: '' };
+})();
+
+const resolveAssetUrl = (path) => {
+    if (!path) return '';
+    if (/^https?:\/\//i.test(path)) return path;
+    if (path.startsWith('/') && API_ORIGIN) {
+        return `${API_ORIGIN}${path}`;
+    }
+    return path;
+};
+const CACHE_TTL = 3600000; // 1 hour in ms
+
+class MarketplaceCache {
+    constructor() {
+        this.prefix = 'c4ai_market_';
+    }
+
+    get(key) {
+        const item = localStorage.getItem(this.prefix + key);
+        if (!item) return null;
+
+        const data = JSON.parse(item);
+        if (Date.now() > data.expires) {
+            localStorage.removeItem(this.prefix + key);
+            return null;
+        }
+        return data.value;
+    }
+
+    set(key, value, ttl = CACHE_TTL) {
+        const data = {
+            value: value,
+            expires: Date.now() + ttl
+        };
+        localStorage.setItem(this.prefix + key, JSON.stringify(data));
+    }
+
+    clear() {
+        Object.keys(localStorage)
+            .filter(k => k.startsWith(this.prefix))
+            .forEach(k => localStorage.removeItem(k));
+    }
+}
+
+class MarketplaceAPI {
+    constructor() {
+        this.cache = new MarketplaceCache();
+        this.searchTimeout = null;
+    }
+
+    async fetch(endpoint, useCache = true) {
+        const cacheKey = endpoint.replace(/[^\w]/g, '_');
+
+        if (useCache) {
+            const cached = this.cache.get(cacheKey);
+            if (cached) return cached;
+        }
+
+        try {
+            const response = await fetch(`${API_BASE}${endpoint}`);
+            if (!response.ok) throw new Error(`HTTP ${response.status}`);
+
+            const data = await response.json();
+            this.cache.set(cacheKey, data);
+            return data;
+        } catch (error) {
+            console.error('API Error:', error);
+            return null;
+        }
+    }
+
+    async getStats() {
+        return this.fetch('/stats');
+    }
+
+    async getCategories() {
+        return this.fetch('/categories');
+    }
+
+    async getApps(params = {}) {
+        const query = new URLSearchParams(params).toString();
+        return this.fetch(`/apps${query ? '?' + query : ''}`);
+    }
+
+    async getArticles(params = {}) {
+        const query = new URLSearchParams(params).toString();
+        return this.fetch(`/articles${query ? '?' + query : ''}`);
+    }
+
+    async getSponsors() {
+        return this.fetch('/sponsors');
+    }
+
+    async search(query) {
+        if (query.length < 2) return {};
+        return this.fetch(`/search?q=${encodeURIComponent(query)}`, false);
+    }
+}
+
+class MarketplaceUI {
+    constructor() {
+        this.api = new MarketplaceAPI();
+        this.currentCategory = 'all';
+        this.currentType = '';
+        this.searchTimeout = null;
+        this.loadedApps = 10;
+        this.init();
+    }
+
+    async init() {
+        await this.loadStats();
+        await this.loadCategories();
+        await this.loadFeaturedContent();
+        await this.loadSponsors();
+        await this.loadMainContent();
+        this.setupEventListeners();
+    }
+
+    async loadStats() {
+        const stats = await this.api.getStats();
+        if (stats) {
+            document.getElementById('total-apps').textContent = stats.total_apps || '0';
+            document.getElementById('total-articles').textContent = stats.total_articles || '0';
+            document.getElementById('total-downloads').textContent = stats.total_downloads || '0';
+            document.getElementById('last-update').textContent = new Date().toLocaleDateString();
+        }
+    }
+
+    async loadCategories() {
+        const categories = await this.api.getCategories();
+        if (!categories) return;
+
+        const filter = document.getElementById('category-filter');
+        categories.forEach(cat => {
+            const btn = document.createElement('button');
+            btn.className = 'filter-btn';
+            btn.dataset.category = cat.slug;
+            btn.textContent = cat.name;
+            btn.onclick = () => this.filterByCategory(cat.slug);
+            filter.appendChild(btn);
+        });
+    }
+
+    async loadFeaturedContent() {
+        // Load hero featured
+        const featured = await this.api.getApps({ featured: true, limit: 4 });
+        if (!featured || !featured.length) return;
+
+        // Hero card (first featured)
+        const hero = featured[0];
+        const heroCard = document.getElementById('featured-hero');
+        if (hero) {
+            const imageUrl = hero.image || '';
+            heroCard.innerHTML = `
+                
    
+                    ${!imageUrl ? `[${hero.category || 'APP'}]` : ''}
+                
    
+                
    
+                    ${hero.type || 'PAID'}
+                    
    ${hero.name}
    
+                    
    ${hero.description}
    
+                    
    
+                        ★ ${hero.rating || 0}/5
+                        ${hero.downloads || 0} downloads
+                    
    
+                
    
+            `;
+            heroCard.onclick = () => this.showAppDetail(hero);
+        }
+
+        // Secondary featured cards
+        const secondary = document.getElementById('featured-secondary');
+        secondary.innerHTML = '';
+        if (featured.length > 1) {
+            featured.slice(1, 4).forEach(app => {
+                const card = document.createElement('div');
+                card.className = 'secondary-card';
+                const imageUrl = app.image || '';
+                card.innerHTML = `
+                    
    
+                        ${!imageUrl ? `[${app.category || 'APP'}]` : ''}
+                    
    
+                    
    
+                        
    ${app.name}
    
+                        
    ${(app.description || '').substring(0, 100)}...
    
+                        
    
+                            ${app.type || 'Open Source'} · ★ ${app.rating || 0}/5
+                        
    
+                    
    
+                `;
+                card.onclick = () => this.showAppDetail(app);
+                secondary.appendChild(card);
+            });
+        }
+    }
+
+    async loadSponsors() {
+        const sponsors = await this.api.getSponsors();
+        if (!sponsors || !sponsors.length) {
+            // Show placeholder if no sponsors
+            const container = document.getElementById('sponsored-content');
+            container.innerHTML = `
+                
+            `;
+            return;
+        }
+
+        const container = document.getElementById('sponsored-content');
+        container.innerHTML = sponsors.slice(0, 5).map(sponsor => `
+            
+        `).join('');
+    }
+
+    async loadMainContent() {
+        // Load apps column
+        const apps = await this.api.getApps({ limit: 8 });
+        if (apps && apps.length) {
+            const appsGrid = document.getElementById('apps-grid');
+            appsGrid.innerHTML = apps.map(app => `
+                
    
+                    
    
+                        ${app.category}
+                        ★ ${app.rating}/5
+                    
    
+                    
    ${app.name}
    
+                    
    ${app.description}
    
+                
    
+            `).join('');
+        }
+
+        // Load articles column
+        const articles = await this.api.getArticles({ limit: 6 });
+        if (articles && articles.length) {
+            const articlesList = document.getElementById('articles-list');
+            articlesList.innerHTML = articles.map(article => `
+                
    
+                    
    
+                        ${article.category} · ${new Date(article.published_at).toLocaleDateString()}
+                    
    
+                    
    ${article.title}
    
+                    
    by ${article.author}
    
+                
    
+            `).join('');
+        }
+
+        // Load trending
+        if (apps && apps.length) {
+            const trending = apps.slice(0, 5);
+            const trendingList = document.getElementById('trending-list');
+            trendingList.innerHTML = trending.map((app, i) => `
+                
    
+                    
    ${i + 1}
    
+                    
    
+                        
    ${app.name}
    
+                        
    ${app.downloads} downloads
    
+                    
    
+                
    
+            `).join('');
+        }
+
+        // Load more apps grid
+        const moreApps = await this.api.getApps({ offset: 8, limit: 12 });
+        if (moreApps && moreApps.length) {
+            const moreGrid = document.getElementById('more-apps-grid');
+            moreGrid.innerHTML = moreApps.map(app => `
+                
    
+                    
    
+                        ${app.category}
+                        ${app.type}
+                    
    
+                    
    ${app.name}
    
+                
    
+            `).join('');
+        }
+    }
+
+    setupEventListeners() {
+        // Search
+        const searchInput = document.getElementById('search-input');
+        searchInput.addEventListener('input', (e) => {
+            clearTimeout(this.searchTimeout);
+            this.searchTimeout = setTimeout(() => this.search(e.target.value), 300);
+        });
+
+        // Keyboard shortcut
+        document.addEventListener('keydown', (e) => {
+            if (e.key === '/' && !searchInput.contains(document.activeElement)) {
+                e.preventDefault();
+                searchInput.focus();
+            }
+            if (e.key === 'Escape' && searchInput.contains(document.activeElement)) {
+                searchInput.blur();
+                searchInput.value = '';
+            }
+        });
+
+        // Type filter
+        const typeFilter = document.getElementById('type-filter');
+        typeFilter.addEventListener('change', (e) => {
+            this.currentType = e.target.value;
+            this.loadMainContent();
+        });
+
+        // Load more
+        const loadMore = document.getElementById('load-more');
+        loadMore.addEventListener('click', () => this.loadMoreApps());
+    }
+
+    async filterByCategory(category) {
+        // Update active state
+        document.querySelectorAll('.filter-btn').forEach(btn => {
+            btn.classList.toggle('active', btn.dataset.category === category);
+        });
+
+        this.currentCategory = category;
+        await this.loadMainContent();
+    }
+
+    async search(query) {
+        if (!query) {
+            await this.loadMainContent();
+            return;
+        }
+
+        const results = await this.api.search(query);
+        if (!results) return;
+
+        // Update apps grid with search results
+        if (results.apps && results.apps.length) {
+            const appsGrid = document.getElementById('apps-grid');
+            appsGrid.innerHTML = results.apps.map(app => `
+                
    
+                    
    
+                        ${app.category}
+                        ★ ${app.rating}/5
+                    
    
+                    
    ${app.name}
    
+                    
    ${app.description}
    
+                
    
+            `).join('');
+        }
+
+        // Update articles with search results
+        if (results.articles && results.articles.length) {
+            const articlesList = document.getElementById('articles-list');
+            articlesList.innerHTML = results.articles.map(article => `
+                
    
+                    
    
+                        ${article.category} · ${new Date(article.published_at).toLocaleDateString()}
+                    
    
+                    
    ${article.title}
    
+                    
    by ${article.author}
    
+                
    
+            `).join('');
+        }
+    }
+
+    async loadMoreApps() {
+        this.loadedApps += 12;
+        const moreApps = await this.api.getApps({ offset: this.loadedApps, limit: 12 });
+        if (moreApps && moreApps.length) {
+            const moreGrid = document.getElementById('more-apps-grid');
+            moreApps.forEach(app => {
+                const card = document.createElement('div');
+                card.className = 'app-compact';
+                card.innerHTML = `
+                    
    
+                        ${app.category}
+                        ${app.type}
+                    
    
+                    
    ${app.name}
    
+                `;
+                card.onclick = () => this.showAppDetail(app);
+                moreGrid.appendChild(card);
+            });
+        }
+    }
+
+    showAppDetail(app) {
+        // Navigate to detail page instead of showing modal
+        const slug = app.slug || app.name.toLowerCase().replace(/\s+/g, '-');
+        window.location.href = `app-detail.html?app=${slug}`;
+    }
+
+    showArticle(articleId) {
+        // Could create article detail page similarly
+        console.log('Show article:', articleId);
+    }
+}
+
+// Initialize marketplace
+let marketplace;
+document.addEventListener('DOMContentLoaded', () => {
+    marketplace = new MarketplaceUI();
+});
\ No newline at end of file
diff --git a/docs/md_v2/migration/table_extraction_v073.md b/docs/md_v2/migration/table_extraction_v073.md
new file mode 100644
index 0000000..464ff8b
--- /dev/null
+++ b/docs/md_v2/migration/table_extraction_v073.md
@@ -0,0 +1,376 @@
+# Migration Guide: Table Extraction v0.7.3
+
+## Overview
+
+Version 0.7.3 introduces the **Table Extraction Strategy Pattern**, providing a more flexible and extensible approach to table extraction while maintaining full backward compatibility.
+
+## What's New
+
+### Strategy Pattern Implementation
+
+Table extraction now follows the same strategy pattern used throughout Crawl4AI:
+
+- **Consistent Architecture**: Aligns with extraction, chunking, and markdown strategies
+- **Extensibility**: Easy to create custom table extraction strategies
+- **Better Separation**: Table logic moved from content scraping to dedicated module
+- **Full Control**: Fine-grained control over table detection and extraction
+
+### New Classes
+
+```python
+from crawl4ai import (
+    TableExtractionStrategy,    # Abstract base class
+    DefaultTableExtraction,      # Current implementation (default)
+    NoTableExtraction           # Explicitly disable extraction
+)
+```
+
+## Backward Compatibility
+
+**✅ All existing code continues to work without changes.**
+
+### No Changes Required
+
+If your code looks like this, it will continue to work:
+
+```python
+# This still works exactly the same
+config = CrawlerRunConfig(
+    table_score_threshold=7
+)
+result = await crawler.arun(url, config)
+tables = result.tables  # Same structure, same data
+```
+
+### What Happens Behind the Scenes
+
+When you don't specify a `table_extraction` strategy:
+
+1. `CrawlerRunConfig` automatically creates `DefaultTableExtraction`
+2. It uses your `table_score_threshold` parameter
+3. Tables are extracted exactly as before
+4. Results appear in `result.tables` with the same structure
+
+## New Capabilities
+
+### 1. Explicit Strategy Configuration
+
+You can now explicitly configure table extraction:
+
+```python
+# New: Explicit control
+strategy = DefaultTableExtraction(
+    table_score_threshold=7,
+    min_rows=2,              # New: minimum row filter
+    min_cols=2,              # New: minimum column filter
+    verbose=True             # New: detailed logging
+)
+
+config = CrawlerRunConfig(
+    table_extraction=strategy
+)
+```
+
+### 2. Disable Table Extraction
+
+Improve performance when tables aren't needed:
+
+```python
+# New: Skip table extraction entirely
+config = CrawlerRunConfig(
+    table_extraction=NoTableExtraction()
+)
+# No CPU cycles spent on table detection/extraction
+```
+
+### 3. Custom Extraction Strategies
+
+Create specialized extractors:
+
+```python
+class MyTableExtractor(TableExtractionStrategy):
+    def extract_tables(self, element, **kwargs):
+        # Custom extraction logic
+        return custom_tables
+
+config = CrawlerRunConfig(
+    table_extraction=MyTableExtractor()
+)
+```
+
+## Migration Scenarios
+
+### Scenario 1: Basic Usage (No Changes Needed)
+
+**Before (v0.7.2):**
+```python
+config = CrawlerRunConfig()
+result = await crawler.arun(url, config)
+for table in result.tables:
+    print(table['headers'])
+```
+
+**After (v0.7.3):**
+```python
+# Exactly the same - no changes required
+config = CrawlerRunConfig()
+result = await crawler.arun(url, config)
+for table in result.tables:
+    print(table['headers'])
+```
+
+### Scenario 2: Custom Threshold (No Changes Needed)
+
+**Before (v0.7.2):**
+```python
+config = CrawlerRunConfig(
+    table_score_threshold=5
+)
+```
+
+**After (v0.7.3):**
+```python
+# Still works the same
+config = CrawlerRunConfig(
+    table_score_threshold=5
+)
+
+# Or use new explicit approach for more control
+strategy = DefaultTableExtraction(
+    table_score_threshold=5,
+    min_rows=2  # Additional filtering
+)
+config = CrawlerRunConfig(
+    table_extraction=strategy
+)
+```
+
+### Scenario 3: Advanced Filtering (New Feature)
+
+**Before (v0.7.2):**
+```python
+# Had to filter after extraction
+config = CrawlerRunConfig(
+    table_score_threshold=5
+)
+result = await crawler.arun(url, config)
+
+# Manual filtering
+large_tables = [
+    t for t in result.tables 
+    if len(t['rows']) >= 5 and len(t['headers']) >= 3
+]
+```
+
+**After (v0.7.3):**
+```python
+# Filter during extraction (more efficient)
+strategy = DefaultTableExtraction(
+    table_score_threshold=5,
+    min_rows=5,
+    min_cols=3
+)
+config = CrawlerRunConfig(
+    table_extraction=strategy
+)
+result = await crawler.arun(url, config)
+# result.tables already filtered
+```
+
+## Code Organization Changes
+
+### Module Structure
+
+**Before (v0.7.2):**
+```
+crawl4ai/
+  content_scraping_strategy.py
+    - LXMLWebScrapingStrategy
+      - is_data_table()      # Table detection
+      - extract_table_data() # Table extraction
+```
+
+**After (v0.7.3):**
+```
+crawl4ai/
+  content_scraping_strategy.py
+    - LXMLWebScrapingStrategy
+      # Table methods removed, uses strategy
+  
+  table_extraction.py (NEW)
+    - TableExtractionStrategy    # Base class
+    - DefaultTableExtraction      # Moved logic here
+    - NoTableExtraction          # New option
+```
+
+### Import Changes
+
+**New imports available (optional):**
+```python
+# These are now available but not required for existing code
+from crawl4ai import (
+    TableExtractionStrategy,
+    DefaultTableExtraction,
+    NoTableExtraction
+)
+```
+
+## Performance Implications
+
+### No Performance Impact
+
+For existing code, performance remains identical:
+- Same extraction logic
+- Same scoring algorithm
+- Same processing time
+
+### Performance Improvements Available
+
+New options for better performance:
+
+```python
+# Skip tables entirely (faster)
+config = CrawlerRunConfig(
+    table_extraction=NoTableExtraction()
+)
+
+# Process only specific areas (faster)
+config = CrawlerRunConfig(
+    css_selector="main.content",
+    table_extraction=DefaultTableExtraction(
+        min_rows=5,  # Skip small tables
+        min_cols=3
+    )
+)
+```
+
+## Testing Your Migration
+
+### Verification Script
+
+Run this to verify your extraction still works:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def verify_extraction():
+    url = "your_url_here"
+    
+    async with AsyncWebCrawler() as crawler:
+        # Test 1: Old approach
+        config_old = CrawlerRunConfig(
+            table_score_threshold=7
+        )
+        result_old = await crawler.arun(url, config_old)
+        
+        # Test 2: New explicit approach
+        from crawl4ai import DefaultTableExtraction
+        config_new = CrawlerRunConfig(
+            table_extraction=DefaultTableExtraction(
+                table_score_threshold=7
+            )
+        )
+        result_new = await crawler.arun(url, config_new)
+        
+        # Compare results
+        assert len(result_old.tables) == len(result_new.tables)
+        print(f"✓ Both approaches extracted {len(result_old.tables)} tables")
+        
+        # Verify structure
+        for old, new in zip(result_old.tables, result_new.tables):
+            assert old['headers'] == new['headers']
+            assert old['rows'] == new['rows']
+        
+        print("✓ Table content identical")
+
+asyncio.run(verify_extraction())
+```
+
+## Deprecation Notes
+
+### No Deprecations
+
+- All existing parameters continue to work
+- `table_score_threshold` in `CrawlerRunConfig` is still supported
+- No breaking changes
+
+### Internal Changes (Transparent to Users)
+
+- `LXMLWebScrapingStrategy.is_data_table()` - Moved to `DefaultTableExtraction`
+- `LXMLWebScrapingStrategy.extract_table_data()` - Moved to `DefaultTableExtraction`
+
+These methods were internal and not part of the public API.
+
+## Benefits of Upgrading
+
+While not required, using the new pattern provides:
+
+1. **Better Control**: Filter tables during extraction, not after
+2. **Performance Options**: Skip extraction when not needed
+3. **Extensibility**: Create custom extractors for specific needs
+4. **Consistency**: Same pattern as other Crawl4AI strategies
+5. **Future-Proof**: Ready for upcoming advanced strategies
+
+## Troubleshooting
+
+### Issue: Different Number of Tables
+
+**Cause**: Threshold or filtering differences
+
+**Solution**: 
+```python
+# Ensure same threshold
+strategy = DefaultTableExtraction(
+    table_score_threshold=7,  # Match your old setting
+    min_rows=0,               # No filtering (default)
+    min_cols=0                # No filtering (default)
+)
+```
+
+### Issue: Import Errors
+
+**Cause**: Using new classes without importing
+
+**Solution**:
+```python
+# Add imports if using new features
+from crawl4ai import (
+    DefaultTableExtraction,
+    NoTableExtraction,
+    TableExtractionStrategy
+)
+```
+
+### Issue: Custom Strategy Not Working
+
+**Cause**: Incorrect method signature
+
+**Solution**:
+```python
+class CustomExtractor(TableExtractionStrategy):
+    def extract_tables(self, element, **kwargs):  # Correct signature
+        # Not: extract_tables(self, html)
+        # Not: extract(self, element)
+        return tables_list
+```
+
+## Getting Help
+
+If you encounter issues:
+
+1. Check your `table_score_threshold` matches previous settings
+2. Verify imports if using new classes
+3. Enable verbose logging: `DefaultTableExtraction(verbose=True)`
+4. Review the [Table Extraction Documentation](../core/table_extraction.md)
+5. Check [examples](../examples/table_extraction_example.py)
+
+## Summary
+
+- ✅ **Full backward compatibility** - No code changes required
+- ✅ **Same results** - Identical extraction behavior by default
+- ✅ **New options** - Additional control when needed
+- ✅ **Better architecture** - Consistent with Crawl4AI patterns
+- ✅ **Ready for future** - Foundation for advanced strategies
+
+The migration to v0.7.3 is seamless with no required changes while providing new capabilities for those who need them.
\ No newline at end of file
diff --git a/docs/md_v2/migration/webscraping-strategy-migration.md b/docs/md_v2/migration/webscraping-strategy-migration.md
new file mode 100644
index 0000000..687ec9b
--- /dev/null
+++ b/docs/md_v2/migration/webscraping-strategy-migration.md
@@ -0,0 +1,92 @@
+# WebScrapingStrategy Migration Guide
+
+## Overview
+
+Crawl4AI has simplified its content scraping architecture. The BeautifulSoup-based `WebScrapingStrategy` has been deprecated in favor of the faster LXML-based implementation. However, **no action is required** - your existing code will continue to work.
+
+## What Changed?
+
+1. **`WebScrapingStrategy` is now an alias** for `LXMLWebScrapingStrategy`
+2. **The BeautifulSoup implementation has been removed** (~1000 lines of redundant code)
+3. **`LXMLWebScrapingStrategy` inherits directly** from `ContentScrapingStrategy`
+4. **Performance remains optimal** with LXML as the sole implementation
+
+## Backward Compatibility
+
+**Your existing code continues to work without any changes:**
+
+```python
+# This still works perfectly
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, WebScrapingStrategy
+
+config = CrawlerRunConfig(
+    scraping_strategy=WebScrapingStrategy()  # Works as before
+)
+```
+
+## Migration Options
+
+You have three options:
+
+### Option 1: Do Nothing (Recommended)
+Your code will continue to work. `WebScrapingStrategy` is permanently aliased to `LXMLWebScrapingStrategy`.
+
+### Option 2: Update Imports (Optional)
+For clarity, you can update your imports:
+
+```python
+# Old (still works)
+from crawl4ai import WebScrapingStrategy
+strategy = WebScrapingStrategy()
+
+# New (more explicit)
+from crawl4ai import LXMLWebScrapingStrategy
+strategy = LXMLWebScrapingStrategy()
+```
+
+### Option 3: Use Default Configuration
+Since `LXMLWebScrapingStrategy` is the default, you can omit the strategy parameter:
+
+```python
+# Simplest approach - uses LXMLWebScrapingStrategy by default
+config = CrawlerRunConfig()
+```
+
+## Type Hints
+
+If you use type hints, both work:
+
+```python
+from crawl4ai import WebScrapingStrategy, LXMLWebScrapingStrategy
+
+def process_with_strategy(strategy: WebScrapingStrategy) -> None:
+    # Works with both WebScrapingStrategy and LXMLWebScrapingStrategy
+    pass
+
+# Both are valid
+process_with_strategy(WebScrapingStrategy())
+process_with_strategy(LXMLWebScrapingStrategy())
+```
+
+## Subclassing
+
+If you've subclassed `WebScrapingStrategy`, it continues to work:
+
+```python
+class MyCustomStrategy(WebScrapingStrategy):
+    def __init__(self):
+        super().__init__()
+        # Your custom code
+```
+
+## Performance Benefits
+
+By consolidating to LXML:
+- **10-20x faster** HTML parsing for large documents
+- **Lower memory usage**
+- **Consistent behavior** across all use cases
+- **Simplified maintenance** and bug fixes
+
+## Summary
+
+This change simplifies Crawl4AI's internals while maintaining 100% backward compatibility. Your existing code continues to work, and you get better performance automatically.
\ No newline at end of file
diff --git a/docs/md_v2/overrides/main.html b/docs/md_v2/overrides/main.html
new file mode 100644
index 0000000..a072d48
--- /dev/null
+++ b/docs/md_v2/overrides/main.html
@@ -0,0 +1,47 @@
+{% set extra_html_attrs = 'data-theme="dark"' %}
+{% extends "base.html" %}
+
+{% block extrahead %}
+{{ super() }}
+
+
+
+
+
+{% endblock %}
+
+{% block footer %}
+
+    >
+    Feedback
+
+{% endblock %}
\ No newline at end of file
diff --git a/docs/md_v2/privacy.md b/docs/md_v2/privacy.md
new file mode 100644
index 0000000..61ae3c3
--- /dev/null
+++ b/docs/md_v2/privacy.md
@@ -0,0 +1,102 @@
+# Privacy Policy
+
+**Last updated: April 20, 2026**
+
+Crawl4AI ("we", "us", "our") provides web crawling, scraping, and structured data extraction services through our open-source library, hosted API ("Crawl4AI Cloud"), dashboard, integrations, and Workspace add-ons. This Privacy Policy explains what we collect, how we use it, and the choices you have.
+
+By creating an account or using our services, you agree to this Policy.
+
+---
+
+## 1. Who we are
+
+Crawl4AI is operated by **CRAWL4AI**, located at 38 Beach Road, #26-12, South Beach Tower, Singapore 189767. Contact: [unclecode@crawl4ai.com](mailto:unclecode@crawl4ai.com).
+
+## 2. Information we collect
+
+We collect only what we need to operate the service.
+
+**Account information** — when you sign up via Google or GitHub OAuth, we receive your email address, display name, profile picture URL, and a stable account identifier from the provider. We do not receive or store your password.
+
+**Usage data** — for jobs you submit (crawls, extractions, enrichments, screenshots, scans), we record: the input URLs or keywords, the configuration you chose, timestamps, billable credits consumed, status, and links to results stored in our object storage. We retain results for the period defined by your plan (typically 30 days).
+
+**API keys** — when you generate an API key, we store a hashed version on our servers. The plaintext is shown to you once at creation and never stored in retrievable form afterwards.
+
+**Operational logs** — request method, path, status code, latency, IP address, user agent, and request ID. Used for debugging, abuse prevention, capacity planning, and security incident response.
+
+**Payment information** — handled by our payment processor (Stripe). We do not store full card numbers; we keep only the customer reference, plan, and the last four digits of the card for display.
+
+**Cookies** — minimal session cookies for authentication and CSRF protection. We do not use third-party advertising cookies.
+
+## 3. How we use your information
+
+- Operate, maintain, and secure the Crawl4AI services
+- Authenticate you and authorise API requests against your plan
+- Bill and meter usage
+- Notify you about service status, security issues, and material changes
+- Improve product quality through aggregate, de-identified usage analysis
+- Comply with legal obligations and respond to lawful requests
+
+We do **not** sell your personal information. We do not use the contents of pages you crawl, the keywords you submit, or the results we extract for advertising or to train third-party machine-learning models.
+
+## 4. Crawl4AI Workspace add-ons (Google Sheets, etc.)
+
+Our Google Workspace add-ons run within your Google account using Apps Script. Specifically:
+
+- **Sheet contents** — the add-on reads the active spreadsheet you have open and writes results back into the same spreadsheet. Sheet contents are sent to our API only as part of jobs you explicitly trigger (for example, by clicking "Generate Data"). Cell contents are not stored beyond the duration of the job.
+- **API key storage** — the add-on stores your Crawl4AI API key in Google's per-user `PropertiesService`, which is encrypted at rest by Google and scoped to your Google account.
+- **Email** — when granted, we read your Google account email address solely to display it in the sidebar so you know which account is in use.
+- **Limited use** — we follow Google's [Limited Use Requirements](https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes) for Workspace API data. We do not transfer Workspace data to third parties except as necessary to provide the user-facing feature, comply with applicable law, or as part of a merger / acquisition where successor entity is bound by an at-least-equally-protective policy. We do not use Workspace data for advertising, and we do not allow humans to read it except with your explicit permission, for security investigations, or to comply with applicable law.
+
+## 5. How we share information
+
+We share information only when needed to run the service:
+
+- **Sub-processors** — we use a small set of vendors for hosting (Hetzner, AWS), object storage, transactional email (Postmark), payment processing (Stripe), error tracking, LLM inference (OpenAI, Anthropic, Google), and search (Serper). Each is bound by data-processing terms.
+- **Legal compliance** — we may disclose information if required by law, subpoena, or to protect rights, property, or safety.
+- **Business transfers** — if the service is acquired or merged, your information may transfer to the successor under an at-least-equally-protective policy.
+
+## 6. Data retention
+
+- Account information — kept while your account is active
+- Job results and stored objects — retained per your plan, typically 30 days, then permanently deleted
+- Operational logs — typically 90 days
+- Billing records — kept for at least 7 years where required by tax law
+- Deletion requests — see Section 9
+
+## 7. Security
+
+We use TLS in transit, encryption at rest for stored objects and database backups, scoped service credentials, network isolation, and least-privilege access controls. No system is perfectly secure; if we become aware of a breach affecting your data, we will notify you without undue delay.
+
+## 8. International data transfers
+
+We host primarily in the EU (Hetzner) and the US (AWS). When we transfer personal data across regions, we rely on standard contractual clauses or other legally recognised transfer mechanisms.
+
+## 9. Your rights
+
+Depending on where you live, you may have the right to:
+
+- Access the personal data we hold about you
+- Correct inaccurate data
+- Delete your data ("right to be forgotten")
+- Export your data in a portable format
+- Object to or restrict certain processing
+- Withdraw consent
+
+To exercise any of these rights, email [unclecode@crawl4ai.com](mailto:unclecode@crawl4ai.com). We will respond within 30 days. We will not discriminate against you for exercising these rights.
+
+## 10. Children
+
+Our services are not directed to children under 16, and we do not knowingly collect personal information from them. If you believe a child has provided us personal information, contact us and we will delete it.
+
+## 11. Changes to this Policy
+
+We may update this Policy from time to time. Material changes will be announced via email and through an in-product notice at least 14 days before they take effect. The "Last updated" date at the top reflects the latest revision.
+
+## 12. Contact
+
+Questions, complaints, or requests:
+
+**Crawl4AI**
+38 Beach Road, #26-12, South Beach Tower, Singapore 189767
+[unclecode@crawl4ai.com](mailto:unclecode@crawl4ai.com)
diff --git a/docs/md_v2/stats.md b/docs/md_v2/stats.md
new file mode 100644
index 0000000..ccba194
--- /dev/null
+++ b/docs/md_v2/stats.md
@@ -0,0 +1,524 @@
+---
+hide:
+  - navigation
+  - toc
+---
+
+
+
+
+
+
+
+
    
+
+
    
+    Crawl4AI
+    
    Growth
    
+    
    Community growth & adoption metrics for Crawl4AI
    
+
    
+
+
+
    
+    
    
+        
    GitHub Stars
    
+        
    60,904
    
+        
    stargazers
    
+    
    
+    
    
+        
    Monthly Downloads
    
+        
    914.2K
    
+        
    PyPI · latest month
    
+    
    
+    
    
+        
    Total Downloads
    
+        
    9.72M
    
+        
    PyPI cumulative
    
+    
    
+    
    
+        
    Docker Pulls
    
+        
    1.41M
    
+        
    hub.docker.com
    
+    
    
+    
    
+        
    Forks / Contributors
    
+        
    6,217 / 57
    
+        
    GitHub
    
+    
    
+
    
+
+
+
    
+    
    PyPI Monthly Downloads
    
+    
    
+        
+    
    
+
    
+
+
+
    
+    
    
+        
    GitHub Star Growth
    
+        
    
+            
+        
    
+    
    
+    
    
+        
    Cumulative PyPI Downloads
    
+        
    
+            
+        
    
+    
    
+
    
+
+
+
    
+    
    
+        
    Daily Download Trend
    
+        
    
+            
+        
    
+    
    
+    
    
+        
    GitHub Traffic (14 days)
    
+        
    
+            
+        
    
+    
    
+
    
+
+
    
+    Updated February 24, 2026 · Data from GitHub API, PyPI Stats, Docker Hub
+
    
+
+
    
+
+
diff --git a/docs/md_v2/support.md b/docs/md_v2/support.md
new file mode 100644
index 0000000..c2bcaa4
--- /dev/null
+++ b/docs/md_v2/support.md
@@ -0,0 +1,63 @@
+# Support
+
+We're here to help. Pick the channel that fits.
+
+---
+
+## Email
+
+For account, billing, integration, or sensitive questions:
+
+[**unclecode@crawl4ai.com**](mailto:unclecode@crawl4ai.com)
+
+We aim to reply within one business day.
+
+## Documentation
+
+The fastest answer is usually in the docs.
+
+- [Quick Start](/core/quickstart/) — install, first crawl, first extract in 5 minutes
+- [API Reference](/api/async-webcrawler/) — every parameter, every endpoint
+- [Examples](/core/examples/) — copy-paste recipes for common patterns
+- [Self-Hosting Guide](/core/self-hosting/) — run Crawl4AI on your own infrastructure
+
+## GitHub
+
+For bugs, feature requests, and library issues:
+
+[**github.com/unclecode/crawl4ai/issues**](https://github.com/unclecode/crawl4ai/issues)
+
+Please search existing issues first, and include a minimal reproducible example when filing a bug.
+
+## Community
+
+- **Discord** — chat with other users and the team: [discord.gg/crawl4ai](https://discord.gg/crawl4ai)
+- **Twitter / X** — release news and tips: [@unclecode](https://twitter.com/unclecode)
+
+## Status
+
+Live status of the hosted service:
+
+- Dashboard: [crawl4ai.com/dashboard](https://crawl4ai.com/dashboard)
+- Incidents and maintenance windows are posted in the in-app notification bell
+
+## Security
+
+Found a vulnerability? Please report it privately so we can fix it before disclosure:
+
+[**unclecode@crawl4ai.com**](mailto:unclecode@crawl4ai.com)
+
+Include reproduction steps and the affected component. We will acknowledge within 48 hours and keep you informed through resolution.
+
+## Workspace add-on testers
+
+If you are a draft tester for one of our Google Workspace add-ons (e.g. Crawl4AI for Sheets) and want to opt out:
+
+Email [unclecode@crawl4ai.com](mailto:unclecode@crawl4ai.com) with the subject "Opt out of draft tester programme" and we will remove your address.
+
+## Contact
+
+**Crawl4AI**
+38 Beach Road, #26-12, South Beach Tower
+Singapore 189767
+[unclecode@crawl4ai.com](mailto:unclecode@crawl4ai.com)
diff --git a/docs/md_v2/terms.md b/docs/md_v2/terms.md
new file mode 100644
index 0000000..9f84fd8
--- /dev/null
+++ b/docs/md_v2/terms.md
@@ -0,0 +1,108 @@
+# Terms of Service
+
+**Last updated: April 20, 2026**
+
+These Terms of Service ("Terms") govern your use of the Crawl4AI hosted services, including the API, dashboard, integrations, Workspace add-ons, and any other software offered by **CRAWL4AI** ("we", "us", "our") at crawl4ai.com and its subdomains (collectively, the "Service").
+
+By creating an account or using the Service, you agree to these Terms. If you are using the Service on behalf of an organisation, you represent that you have authority to bind that organisation, and "you" refers to both you and the organisation.
+
+The open-source `crawl4ai` Python library is governed separately by its [open-source licence](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) and is not subject to these Terms.
+
+---
+
+## 1. Your account
+
+You must provide accurate information when creating an account and keep it current. You are responsible for activity under your account and for keeping your API keys confidential. Notify us immediately at [unclecode@crawl4ai.com](mailto:unclecode@crawl4ai.com) if you believe an unauthorised party has accessed your account.
+
+You must be at least 16 years old, or the age of digital consent in your jurisdiction, whichever is greater.
+
+## 2. Plans, credits, and billing
+
+The Service is offered as a freemium model with paid plans. Each plan defines a daily credit limit, rate limits, storage allowance, and feature set. Current plans are listed at [crawl4ai.com/pricing](https://crawl4ai.com/pricing).
+
+**Billing.** Paid plans are billed in advance on a recurring basis (monthly or annual). All fees are exclusive of taxes, which you are responsible for. Payment is processed through Stripe.
+
+**Credits.** Credits do not roll over between billing cycles unless your plan explicitly states otherwise. Credits are consumed by jobs you submit; the credit cost of each job type is documented in our API reference.
+
+**Refunds.** Fees are non-refundable except where required by law. If you cancel a paid plan, you keep access until the end of the current billing period.
+
+**Plan changes.** We may change plan pricing or features with at least 30 days' notice. Changes take effect at the start of your next billing cycle.
+
+## 3. Acceptable use
+
+You agree not to:
+
+- Crawl, scrape, or extract data in violation of applicable law (including copyright, trade-secret, computer-misuse, or data-protection law) or in violation of the target website's terms of service
+- Bypass authentication, paywalls, CAPTCHAs, or other access controls without authorisation from the target site owner
+- Submit excessive requests intended to harm a target website, our infrastructure, or other users
+- Use the Service to send spam, distribute malware, or facilitate abuse, harassment, or illegal activity
+- Reverse-engineer, resell, or sublicense the Service except as expressly permitted
+- Use the Service to build a competing service offering substantially similar features
+- Misrepresent your identity or impersonate another person or entity
+
+We may suspend or terminate accounts that violate these rules, with or without notice depending on severity.
+
+## 4. Content and ownership
+
+**Your input.** You retain ownership of the URLs, keywords, schemas, and configurations you submit. You grant us a limited licence to process them as needed to provide the Service.
+
+**Extracted content.** Pages you crawl belong to their respective owners. Your right to crawl, store, or republish that content depends on the source's terms and applicable law. You are solely responsible for ensuring your usage complies.
+
+**Our materials.** The Crawl4AI brand, logos, dashboard UI, documentation, and hosted infrastructure are our intellectual property. The open-source library is licensed under its own [open-source terms](https://github.com/unclecode/crawl4ai/blob/main/LICENSE).
+
+## 5. Workspace add-ons
+
+When you install a Crawl4AI Workspace add-on (e.g. Google Sheets):
+
+- The add-on operates on the active document you have open and writes results back into it
+- We do not access other documents in your Drive
+- We do not retain document contents beyond the duration of the job
+- See our [Privacy Policy](/privacy/) for full details on Workspace data handling
+
+## 6. Service availability and changes
+
+We aim for high availability but do not guarantee uninterrupted service. We perform periodic maintenance and may need to make emergency changes. We will notify you of significant changes through the dashboard, email, or our status page.
+
+We may add, remove, or change features. If a change materially reduces the functionality of a paid plan, we will offer you a pro-rated refund of unused fees.
+
+## 7. Third-party services
+
+The Service integrates with third-party providers (OpenAI, Anthropic, Google, Stripe, etc.). Your use of those features is also subject to the respective provider's terms. We are not responsible for the availability or behaviour of third-party services.
+
+## 8. Confidentiality and security
+
+We take reasonable measures to protect your data (see [Privacy Policy](/privacy/)). You agree not to publicly disclose non-public information about the Service obtained through your account (such as private API responses, internal error messages, or security findings) except as required by law or pursuant to our [security disclosure process](mailto:unclecode@crawl4ai.com).
+
+## 9. Disclaimers
+
+THE SERVICE IS PROVIDED "AS IS" AND "AS AVAILABLE", WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. WE DO NOT WARRANT THAT THE SERVICE WILL BE ERROR-FREE OR UNINTERRUPTED, OR THAT EXTRACTED DATA WILL BE COMPLETE OR ACCURATE.
+
+## 10. Limitation of liability
+
+TO THE MAXIMUM EXTENT PERMITTED BY LAW, OUR TOTAL LIABILITY FOR ANY CLAIMS ARISING OUT OF OR RELATED TO THESE TERMS OR THE SERVICE IS LIMITED TO THE GREATER OF (A) THE AMOUNT YOU PAID TO US IN THE 12 MONTHS PRECEDING THE CLAIM, OR (B) USD 100. IN NO EVENT WILL WE BE LIABLE FOR INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL, OR PUNITIVE DAMAGES, OR FOR LOSS OF PROFITS, REVENUE, GOODWILL, OR DATA, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+Some jurisdictions do not allow these limitations, in which case the above applies only to the extent permitted.
+
+## 11. Indemnification
+
+You agree to indemnify and hold us harmless from claims, damages, and expenses arising from (a) your use of the Service in violation of these Terms or applicable law, (b) the URLs or keywords you submit, or (c) your use or republication of extracted content.
+
+## 12. Termination
+
+You may terminate your account at any time from the dashboard. We may suspend or terminate your account if you violate these Terms, fail to pay, or engage in activity that places the Service or other users at risk.
+
+On termination, your right to access the Service ends. We will delete your data per the retention schedule in our Privacy Policy unless we are required to retain it for legal reasons.
+
+## 13. Governing law and disputes
+
+These Terms are governed by the laws of Singapore, without regard to conflict-of-laws principles. Disputes will be resolved exclusively in the courts of Singapore, except that either party may seek injunctive relief in any court of competent jurisdiction to protect intellectual property or confidential information.
+
+## 14. Changes to these Terms
+
+We may update these Terms from time to time. Material changes will be announced via email and an in-product notice at least 14 days before they take effect. Continued use of the Service after changes take effect constitutes acceptance.
+
+## 15. Contact
+
+**Crawl4AI**
+38 Beach Road, #26-12, South Beach Tower, Singapore 189767
+[unclecode@crawl4ai.com](mailto:unclecode@crawl4ai.com)
diff --git a/docs/migration/v0.8.0-upgrade-guide.md b/docs/migration/v0.8.0-upgrade-guide.md
new file mode 100644
index 0000000..a310475
--- /dev/null
+++ b/docs/migration/v0.8.0-upgrade-guide.md
@@ -0,0 +1,301 @@
+# Migration Guide: Upgrading to Crawl4AI v0.8.0
+
+This guide helps you upgrade from v0.7.x to v0.8.0, with special attention to breaking changes and security updates.
+
+## Quick Summary
+
+| Change | Impact | Action Required |
+|--------|--------|-----------------|
+| Hooks disabled by default | Docker API users with hooks | Set `CRAWL4AI_HOOKS_ENABLED=true` |
+| file:// URLs blocked | Docker API users reading local files | Use Python library directly |
+| Security fixes | All Docker API users | Update immediately |
+
+---
+
+## Step 1: Update the Package
+
+### PyPI Installation
+
+```bash
+pip install --upgrade crawl4ai
+```
+
+### Docker Installation
+
+```bash
+docker pull unclecode/crawl4ai:latest
+# or
+docker pull unclecode/crawl4ai:0.8.0
+```
+
+### From Source
+
+```bash
+git pull origin main
+pip install -e .
+```
+
+---
+
+## Step 2: Check for Breaking Changes
+
+### Are You Affected?
+
+**You ARE affected if you:**
+- Use the Docker API deployment
+- Use the `hooks` parameter in `/crawl` requests
+- Use `file://` URLs via API endpoints
+
+**You are NOT affected if you:**
+- Only use Crawl4AI as a Python library
+- Don't use hooks in your API calls
+- Don't use `file://` URLs via the API
+
+---
+
+## Step 3: Migrate Hooks Usage
+
+### Before v0.8.0
+
+Hooks worked by default:
+
+```bash
+# This worked without any configuration
+curl -X POST http://localhost:11235/crawl \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"],
+    "hooks": {
+      "code": {
+        "on_page_context_created": "async def hook(page, context, **kwargs):\n    await context.add_cookies([...])\n    return page"
+      }
+    }
+  }'
+```
+
+### After v0.8.0
+
+You must explicitly enable hooks:
+
+**Option A: Environment Variable (Recommended)**
+```bash
+# In your Docker run command or docker-compose.yml
+export CRAWL4AI_HOOKS_ENABLED=true
+```
+
+```yaml
+# docker-compose.yml
+services:
+  crawl4ai:
+    image: unclecode/crawl4ai:0.8.0
+    environment:
+      - CRAWL4AI_HOOKS_ENABLED=true
+```
+
+**Option B: For Kubernetes**
+```yaml
+env:
+  - name: CRAWL4AI_HOOKS_ENABLED
+    value: "true"
+```
+
+### Security Warning
+
+Only enable hooks if:
+- You trust all users who can access the API
+- The API is not exposed to the public internet
+- You have other authentication/authorization in place
+
+---
+
+## Step 4: Migrate file:// URL Usage
+
+### Before v0.8.0
+
+```bash
+# This worked via API
+curl -X POST http://localhost:11235/execute_js \
+  -d '{"url": "file:///var/data/page.html", "scripts": ["document.title"]}'
+```
+
+### After v0.8.0
+
+**Option A: Use the Python Library Directly**
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def process_local_file():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="file:///var/data/page.html",
+            config=CrawlerRunConfig(js_code=["document.title"])
+        )
+        return result
+```
+
+**Option B: Use raw: Protocol for HTML Content**
+
+If you have the HTML content, you can still use the API:
+
+```bash
+# Read file content and send as raw:
+HTML_CONTENT=$(cat /var/data/page.html)
+curl -X POST http://localhost:11235/html \
+  -H "Content-Type: application/json" \
+  -d "{\"url\": \"raw:$HTML_CONTENT\"}"
+```
+
+**Option C: Create a Preprocessing Service**
+
+```python
+# preprocessing_service.py
+from fastapi import FastAPI
+from crawl4ai import AsyncWebCrawler
+
+app = FastAPI()
+
+@app.post("/process-local")
+async def process_local(file_path: str):
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url=f"file://{file_path}")
+        return result.model_dump()
+```
+
+---
+
+## Step 5: Review Security Configuration
+
+### Recommended Production Settings
+
+```yaml
+# config.yml
+security:
+  enabled: true
+  jwt_enabled: true
+  https_redirect: true  # If behind HTTPS proxy
+  trusted_hosts:
+    - "your-domain.com"
+    - "api.your-domain.com"
+```
+
+### Environment Variables
+
+```bash
+# Required for JWT authentication
+export SECRET_KEY="your-secure-random-key-minimum-32-characters"
+
+# Only if you need hooks
+export CRAWL4AI_HOOKS_ENABLED=true
+```
+
+### Generate a Secure Secret Key
+
+```python
+import secrets
+print(secrets.token_urlsafe(32))
+```
+
+---
+
+## Step 6: Test Your Integration
+
+### Quick Validation Script
+
+```python
+import asyncio
+import aiohttp
+
+async def test_upgrade():
+    base_url = "http://localhost:11235"
+
+    # Test 1: Basic crawl should work
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            f"{base_url}/crawl",
+            json={"urls": ["https://example.com"]}
+        ) as resp:
+            assert resp.status == 200, "Basic crawl failed"
+            print("✓ Basic crawl works")
+
+    # Test 2: Hooks should be blocked (unless enabled)
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            f"{base_url}/crawl",
+            json={
+                "urls": ["https://example.com"],
+                "hooks": {"code": {"on_page_context_created": "async def hook(page, context, **kwargs): return page"}}
+            }
+        ) as resp:
+            if resp.status == 403:
+                print("✓ Hooks correctly blocked (default)")
+            elif resp.status == 200:
+                print("! Hooks enabled - ensure this is intentional")
+
+    # Test 3: file:// should be blocked
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            f"{base_url}/execute_js",
+            json={"url": "file:///etc/passwd", "scripts": ["1"]}
+        ) as resp:
+            assert resp.status == 400, "file:// should be blocked"
+            print("✓ file:// URLs correctly blocked")
+
+asyncio.run(test_upgrade())
+```
+
+---
+
+## Troubleshooting
+
+### "Hooks are disabled" Error
+
+**Symptom**: API returns 403 with "Hooks are disabled"
+
+**Solution**: Set `CRAWL4AI_HOOKS_ENABLED=true` if you need hooks
+
+### "URL must start with http://, https://" Error
+
+**Symptom**: API returns 400 when using `file://` URLs
+
+**Solution**: Use Python library directly or `raw:` protocol
+
+### Authentication Errors After Enabling JWT
+
+**Symptom**: API returns 401 Unauthorized
+
+**Solution**:
+1. Get a token: `POST /token` with your email
+2. Include token in requests: `Authorization: Bearer `
+
+---
+
+## Rollback Plan
+
+If you need to rollback:
+
+```bash
+# PyPI
+pip install crawl4ai==0.7.6
+
+# Docker
+docker pull unclecode/crawl4ai:0.7.6
+```
+
+**Warning**: Rolling back re-exposes the security vulnerabilities. Only do this temporarily while fixing integration issues.
+
+---
+
+## Getting Help
+
+- **GitHub Issues**: [github.com/unclecode/crawl4ai/issues](https://github.com/unclecode/crawl4ai/issues)
+- **Security Issues**: See [SECURITY.md](../../SECURITY.md)
+- **Documentation**: [docs.crawl4ai.com](https://docs.crawl4ai.com)
+
+---
+
+## Changelog Reference
+
+For complete list of changes, see:
+- [Release Notes v0.8.0](../RELEASE_NOTES_v0.8.0.md)
+- [CHANGELOG.md](../../CHANGELOG.md)
diff --git a/docs/releases_review/Crawl4AI_v0.3.72_Release_Announcement.ipynb b/docs/releases_review/Crawl4AI_v0.3.72_Release_Announcement.ipynb
new file mode 100644
index 0000000..c6ceab2
--- /dev/null
+++ b/docs/releases_review/Crawl4AI_v0.3.72_Release_Announcement.ipynb
@@ -0,0 +1,235 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 🚀 Crawl4AI v0.3.72 Release Announcement\n",
+    "\n",
+    "Welcome to the new release of **Crawl4AI v0.3.72**! This notebook highlights the latest features and demonstrates how they work in real-time. Follow along to see each feature in action!\n",
+    "\n",
+    "### What’s New?\n",
+    "- ✨ `Fit Markdown`: Extracts only the main content from articles and blogs\n",
+    "- 🛡️ **Magic Mode**: Comprehensive anti-bot detection bypass\n",
+    "- 🌐 **Multi-browser support**: Switch between Chromium, Firefox, WebKit\n",
+    "- 🔍 **Knowledge Graph Extraction**: Generate structured graphs of entities & relationships from any URL\n",
+    "- 🤖 **Crawl4AI GPT Assistant**: Chat directly with our AI assistant for help, code generation, and faster learning (available [here](https://tinyurl.com/your-gpt-assistant-link))\n",
+    "\n",
+    "---\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 📥 Setup\n",
+    "To start, we'll install `Crawl4AI` along with Playwright and `nest_asyncio` to ensure compatibility with Colab’s asynchronous environment."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install Crawl4AI and dependencies\n",
+    "!pip install crawl4ai\n",
+    "!playwright install\n",
+    "!pip install nest_asyncio"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import nest_asyncio and apply it to allow asyncio in Colab\n",
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "print('Setup complete!')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## ✨ Feature 1: `Fit Markdown`\n",
+    "Extracts only the main content from articles and blog pages, removing sidebars, ads, and other distractions.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import asyncio\n",
+    "from crawl4ai import AsyncWebCrawler\n",
+    "\n",
+    "async def fit_markdown_demo():\n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        result = await crawler.arun(url=\"https://janineintheworld.com/places-to-visit-in-central-mexico\")\n",
+    "        print(result.fit_markdown)  # Shows main content in Markdown format\n",
+    "\n",
+    "# Run the demo\n",
+    "await fit_markdown_demo()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🛡️ Feature 2: Magic Mode\n",
+    "Magic Mode bypasses anti-bot detection to make crawling more reliable on protected websites.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "async def magic_mode_demo():\n",
+    "    async with AsyncWebCrawler() as crawler:  # Enables anti-bot detection bypass\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://www.reuters.com/markets/us/global-markets-view-usa-pix-2024-08-29/\",\n",
+    "            magic=True  # Enables magic mode\n",
+    "        )\n",
+    "        print(result.markdown)  # Shows the full content in Markdown format\n",
+    "\n",
+    "# Run the demo\n",
+    "await magic_mode_demo()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🌐 Feature 3: Multi-Browser Support\n",
+    "Crawl4AI now supports Chromium, Firefox, and WebKit. Here’s how to specify Firefox for a crawl.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "async def multi_browser_demo():\n",
+    "    async with AsyncWebCrawler(browser_type=\"firefox\") as crawler:  # Using Firefox instead of default Chromium\n",
+    "        result = await crawler.arun(url=\"https://crawl4i.com\")\n",
+    "        print(result.markdown)  # Shows content extracted using Firefox\n",
+    "\n",
+    "# Run the demo\n",
+    "await multi_browser_demo()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## ✨ Put them all together\n",
+    "\n",
+    "Let's combine all the features to extract the main content from a blog post, bypass anti-bot detection, and generate a knowledge graph from the extracted content."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from crawl4ai import LLMExtractionStrategy\n",
+    "from pydantic import BaseModel\n",
+    "import json, os\n",
+    "from typing import List\n",
+    "\n",
+    "# Define classes for the knowledge graph structure\n",
+    "class Landmark(BaseModel):\n",
+    "    name: str\n",
+    "    description: str\n",
+    "    activities: list[str]  # E.g., visiting, sightseeing, relaxing\n",
+    "\n",
+    "class City(BaseModel):\n",
+    "    name: str\n",
+    "    description: str\n",
+    "    landmarks: list[Landmark]\n",
+    "    cultural_highlights: list[str]  # E.g., food, music, traditional crafts\n",
+    "\n",
+    "class TravelKnowledgeGraph(BaseModel):\n",
+    "    cities: list[City]  # Central Mexican cities to visit\n",
+    "\n",
+    "async def combined_demo():\n",
+    "    # Define the knowledge graph extraction strategy\n",
+    "    strategy = LLMExtractionStrategy(\n",
+    "        # provider=\"ollama/nemotron\",\n",
+    "        provider='openai/gpt-4o-mini', # Or any other provider, including Ollama and open source models\n",
+    "        pi_token=os.getenv('OPENAI_API_KEY'), # In case of Ollama just pass \"no-token\"\n",
+    "        schema=TravelKnowledgeGraph.schema(),\n",
+    "        instruction=(\n",
+    "            \"Extract cities, landmarks, and cultural highlights for places to visit in Central Mexico. \"\n",
+    "            \"For each city, list main landmarks with descriptions and activities, as well as cultural highlights.\"\n",
+    "        )\n",
+    "    )\n",
+    "\n",
+    "    # Set up the AsyncWebCrawler with multi-browser support, Magic Mode, and Fit Markdown\n",
+    "    async with AsyncWebCrawler(browser_type=\"firefox\") as crawler:\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://janineintheworld.com/places-to-visit-in-central-mexico\",\n",
+    "            extraction_strategy=strategy,\n",
+    "            bypass_cache=True,\n",
+    "            magic=True\n",
+    "        )\n",
+    "        \n",
+    "        # Display main article content in Fit Markdown format\n",
+    "        print(\"Extracted Main Content:\\n\", result.fit_markdown)\n",
+    "        \n",
+    "        # Display extracted knowledge graph of cities, landmarks, and cultural highlights\n",
+    "        if result.extracted_content:\n",
+    "            travel_graph = json.loads(result.extracted_content)\n",
+    "            print(\"\\nExtracted Knowledge Graph:\\n\", json.dumps(travel_graph, indent=2))\n",
+    "\n",
+    "# Run the combined demo\n",
+    "await combined_demo()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🤖 Crawl4AI GPT Assistant\n",
+    "Chat with the Crawl4AI GPT Assistant for code generation, support, and learning Crawl4AI faster. Try it out [here](https://tinyurl.com/crawl4ai-gpt)!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/docs/releases_review/crawl4ai_v0_7_0_showcase.py b/docs/releases_review/crawl4ai_v0_7_0_showcase.py
new file mode 100644
index 0000000..d78af7f
--- /dev/null
+++ b/docs/releases_review/crawl4ai_v0_7_0_showcase.py
@@ -0,0 +1,1584 @@
+"""
+🚀 Crawl4AI v0.7.0 Feature Showcase
+=====================================
+This demo showcases the major features introduced in v0.7.0:
+1. Link Preview/Peek - Advanced link analysis with 3-layer scoring
+2. Adaptive Crawling - Intelligent crawling with confidence tracking
+3. Virtual Scroll - Capture content from modern infinite scroll pages
+4. C4A Script - Domain-specific language for web automation
+5. URL Seeder - Smart URL discovery and filtering
+6. LLM Context Builder - 3D context for AI assistants
+
+Let's explore each feature with practical examples!
+"""
+
+import asyncio
+import json
+import time
+import re
+from typing import List, Dict
+from rich.console import Console
+from rich.table import Table
+from rich.progress import Progress, SpinnerColumn, TextColumn
+from rich.panel import Panel
+from rich.syntax import Syntax
+from rich.layout import Layout
+from rich.live import Live
+from rich import box
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, AdaptiveCrawler, AdaptiveConfig, BrowserConfig, CacheMode
+from crawl4ai import AsyncUrlSeeder, SeedingConfig
+from crawl4ai import LinkPreviewConfig, VirtualScrollConfig
+from crawl4ai import c4a_compile, CompilationResult
+
+# Initialize Rich console for beautiful output
+console = Console()
+
+
+def print_banner(title: str, subtitle: str = ""):
+    """Print a beautiful banner for each section"""
+    console.print(f"\n[bold cyan]{'=' * 80}[/bold cyan]")
+    console.print(f"[bold yellow]{title.center(80)}[/bold yellow]")
+    if subtitle:
+        console.print(f"[dim white]{subtitle.center(80)}[/dim white]")
+    console.print(f"[bold cyan]{'=' * 80}[/bold cyan]\n")
+
+
+def create_score_bar(score: float, max_score: float = 10.0) -> str:
+    """Create a visual progress bar for scores"""
+    percentage = (score / max_score)
+    filled = int(percentage * 20)
+    bar = "█" * filled + "░" * (20 - filled)
+    return f"[{'green' if score >= 7 else 'yellow' if score >= 4 else 'red'}]{bar}[/] {score:.2f}/{max_score}"
+
+
+async def link_preview_demo(auto_mode=False):
+    """
+    🔗 Link Preview/Peek Demo
+    Showcases the 3-layer scoring system for intelligent link analysis
+    """
+    print_banner(
+        "🔗 LINK PREVIEW & INTELLIGENT SCORING",
+        "Advanced link analysis with intrinsic, contextual, and total scoring"
+    )
+    
+    # Explain the feature
+    console.print(Panel(
+        "[bold]What is Link Preview?[/bold]\n\n"
+        "Link Preview analyzes links on a page with a sophisticated 3-layer scoring system:\n\n"
+        "• [cyan]Intrinsic Score[/cyan]: Quality based on link text, position, and attributes (0-10)\n"
+        "• [magenta]Contextual Score[/magenta]: Relevance to your query using semantic analysis (0-1)\n"
+        "• [green]Total Score[/green]: Combined score for intelligent prioritization\n\n"
+        "This helps you find the most relevant and high-quality links automatically!",
+        title="Feature Overview",
+        border_style="blue"
+    ))
+    
+    await asyncio.sleep(2)
+    
+    # Demo 1: Basic link analysis with visual scoring
+    console.print("\n[bold yellow]Demo 1: Analyzing Python Documentation Links[/bold yellow]\n")
+    
+    query = "async await coroutines tutorial"
+    console.print(f"[cyan]🔍 Query:[/cyan] [bold]{query}[/bold]")
+    console.print("[dim]Looking for links related to asynchronous programming...[/dim]\n")
+    
+    config = CrawlerRunConfig(
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,
+            include_external=False,
+            max_links=10,
+            concurrency=5,
+            query=query,  # Our search context
+            verbose=False  # We'll handle the display
+        ),
+        score_links=True,
+        only_text=True
+    )
+    
+    # Create a progress display
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        console=console
+    ) as progress:
+        task = progress.add_task("[cyan]Crawling and analyzing links...", total=None)
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://docs.python.org/3/library/asyncio.html", config=config)
+        
+        progress.remove_task(task)
+    
+    if result.success:
+        # Extract links with scores
+        links = result.links.get("internal", [])
+        scored_links = [l for l in links if l.get("head_data") and l.get("total_score")]
+        
+        # Sort by total score
+        scored_links.sort(key=lambda x: x.get("total_score", 0), reverse=True)
+        
+        # Create a beautiful table for results
+        table = Table(
+            title="🎯 Top Scored Links",
+            box=box.ROUNDED,
+            show_lines=True,
+            title_style="bold magenta"
+        )
+        
+        table.add_column("Rank", style="cyan", width=6)
+        table.add_column("Link Text", style="white", width=40)
+        table.add_column("Intrinsic Score", width=25)
+        table.add_column("Contextual Score", width=25)
+        table.add_column("Total Score", style="bold", width=15)
+        
+        for i, link in enumerate(scored_links[:5], 1):
+            intrinsic = link.get('intrinsic_score', 0)
+            contextual = link.get('contextual_score', 0)
+            total = link.get('total_score', 0)
+            
+            # Get link text and title
+            text = link.get('text', '')[:35] + "..." if len(link.get('text', '')) > 35 else link.get('text', '')
+            title = link.get('head_data', {}).get('title', 'No title')[:40]
+            
+            table.add_row(
+                f"#{i}",
+                text or title,
+                create_score_bar(intrinsic, 10.0),
+                create_score_bar(contextual, 1.0),
+                f"[bold green]{total:.3f}[/bold green]"
+            )
+        
+        console.print(table)
+        
+        # Show what makes a high-scoring link
+        if scored_links:
+            best_link = scored_links[0]
+            console.print(f"\n[bold green]🏆 Best Match:[/bold green]")
+            console.print(f"URL: [link]{best_link['href']}[/link]")
+            console.print(f"Title: {best_link.get('head_data', {}).get('title', 'N/A')}")
+            
+            desc = best_link.get('head_data', {}).get('meta', {}).get('description', '')
+            if desc:
+                console.print(f"Description: [dim]{desc[:100]}...[/dim]")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 2...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 2: Research Assistant Mode
+    console.print("\n[bold yellow]Demo 2: Research Assistant - Finding Machine Learning Resources[/bold yellow]\n")
+    
+    # First query - will find no results
+    query1 = "deep learning neural networks beginners tutorial"
+    console.print(f"[cyan]🔍 Query 1:[/cyan] [bold]{query1}[/bold]")
+    console.print("[dim]Note: scikit-learn focuses on traditional ML, not deep learning[/dim]\n")
+    
+    # Configure for research mode
+    research_config = CrawlerRunConfig(
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,
+            include_external=True,
+            query=query1,
+            max_links=20,
+            score_threshold=0.3,  # Only high-relevance links
+            concurrency=10
+        ),
+        score_links=True
+    )
+    
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        console=console
+    ) as progress:
+        task = progress.add_task("[cyan]Discovering learning resources...", total=None)
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://scikit-learn.org/stable/", config=research_config)
+        
+        progress.remove_task(task)
+    
+    if result.success:
+        all_links = result.links.get("internal", []) + result.links.get("external", [])
+        # Filter for links with actual scores
+        relevant_links = [l for l in all_links if l.get("total_score") is not None and l.get("total_score") > 0.3]
+        relevant_links.sort(key=lambda x: x.get("total_score", 0), reverse=True)
+        
+        console.print(f"[bold green]📚 Found {len(relevant_links)} highly relevant resources![/bold green]\n")
+        
+        # Group by score ranges
+        excellent = [l for l in relevant_links if l.get("total_score", 0) > 0.7]
+        good = [l for l in relevant_links if 0.5 <= l.get("total_score", 0) <= 0.7]
+        fair = [l for l in relevant_links if 0.3 <= l.get("total_score", 0) < 0.5]
+        
+        if excellent:
+            console.print("[bold green]⭐⭐⭐ Excellent Matches:[/bold green]")
+            for link in excellent[:3]:
+                title = link.get('head_data', {}).get('title', link.get('text', 'No title'))
+                console.print(f"  • {title[:60]}... [dim]({link.get('total_score', 0):.2f})[/dim]")
+        
+        if good:
+            console.print("\n[yellow]⭐⭐ Good Matches:[/yellow]")
+            for link in good[:3]:
+                title = link.get('head_data', {}).get('title', link.get('text', 'No title'))
+                console.print(f"  • {title[:60]}... [dim]({link.get('total_score', 0):.2f})[/dim]")
+    
+    # Second query - will find results
+    console.print("\n[bold cyan]Let's try a more relevant query for scikit-learn:[/bold cyan]\n")
+    
+    query2 = "machine learning classification tutorial examples"
+    console.print(f"[cyan]🔍 Query 2:[/cyan] [bold]{query2}[/bold]")
+    console.print("[dim]This should find relevant content about traditional ML[/dim]\n")
+    
+    research_config2 = CrawlerRunConfig(
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,
+            include_external=True,
+            query=query2,
+            max_links=15,
+            score_threshold=0.2,  # Slightly lower threshold
+            concurrency=10
+        ),
+        score_links=True
+    )
+    
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        console=console
+    ) as progress:
+        task = progress.add_task("[cyan]Finding ML tutorials...", total=None)
+        
+        async with AsyncWebCrawler() as crawler:
+            result2 = await crawler.arun("https://scikit-learn.org/stable/", config=research_config2)
+        
+        progress.remove_task(task)
+    
+    if result2.success:
+        all_links2 = result2.links.get("internal", []) + result2.links.get("external", [])
+        relevant_links2 = [l for l in all_links2 if l.get("total_score") is not None and l.get("total_score") > 0.2]
+        relevant_links2.sort(key=lambda x: x.get("total_score", 0), reverse=True)
+        
+        console.print(f"[bold green]📚 Now found {len(relevant_links2)} relevant resources![/bold green]\n")
+        
+        if relevant_links2:
+            console.print("[bold]Top relevant links for ML tutorials:[/bold]")
+            for i, link in enumerate(relevant_links2[:5], 1):
+                title = link.get('head_data', {}).get('title', link.get('text', 'No title'))
+                score = link.get('total_score', 0)
+                console.print(f"{i}. [{score:.3f}] {title[:70]}...")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 3...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 3: Live scoring visualization
+    console.print("\n[bold yellow]Demo 3: Understanding the 3-Layer Scoring System[/bold yellow]\n")
+    
+    demo_query = "async programming tutorial"
+    console.print(f"[cyan]🔍 Query:[/cyan] [bold]{demo_query}[/bold]")
+    console.print("[dim]Let's see how different link types score against this query[/dim]\n")
+    
+    # Create a sample link analysis
+    sample_links = [
+        {
+            "text": "Complete Guide to Async Programming",
+            "intrinsic": 9.2,
+            "contextual": 0.95,
+            "factors": ["Strong keywords", "Title position", "Descriptive text"]
+        },
+        {
+            "text": "API Reference",
+            "intrinsic": 6.5,
+            "contextual": 0.15,
+            "factors": ["Common link text", "Navigation menu", "Low relevance"]
+        },
+        {
+            "text": "Click here",
+            "intrinsic": 2.1,
+            "contextual": 0.05,
+            "factors": ["Poor link text", "No context", "Generic anchor"]
+        }
+    ]
+    
+    for link in sample_links:
+        total = (link["intrinsic"] / 10 * 0.4) + (link["contextual"] * 0.6)
+        
+        panel_content = (
+            f"[bold]Link Text:[/bold] {link['text']}\n\n"
+            f"[cyan]Intrinsic Score:[/cyan] {create_score_bar(link['intrinsic'], 10.0)}\n"
+            f"[magenta]Contextual Score:[/magenta] {create_score_bar(link['contextual'], 1.0)}\n"
+            f"[green]Total Score:[/green] {total:.3f}\n\n"
+            f"[dim]Factors: {', '.join(link['factors'])}[/dim]"
+        )
+        
+        console.print(Panel(
+            panel_content,
+            title=f"Link Analysis",
+            border_style="blue" if total > 0.7 else "yellow" if total > 0.3 else "red"
+        ))
+        await asyncio.sleep(1)
+    
+    # Summary
+    console.print("\n[bold green]✨ Link Preview Benefits:[/bold green]")
+    console.print("• Automatically finds the most relevant links for your research")
+    console.print("• Saves time by prioritizing high-quality content")
+    console.print("• Provides semantic understanding beyond simple keyword matching")
+    console.print("• Enables intelligent crawling decisions\n")
+
+
+async def adaptive_crawling_demo(auto_mode=False):
+    """
+    🎯 Adaptive Crawling Demo
+    Shows intelligent crawling that knows when to stop
+    """
+    print_banner(
+        "🎯 ADAPTIVE CRAWLING",
+        "Intelligent crawling that knows when it has enough information"
+    )
+    
+    # Explain the feature
+    console.print(Panel(
+        "[bold]What is Adaptive Crawling?[/bold]\n\n"
+        "Adaptive Crawling intelligently determines when sufficient information has been gathered:\n\n"
+        "• [cyan]Confidence Tracking[/cyan]: Monitors how well we understand the topic (0-100%)\n"
+        "• [magenta]Smart Exploration[/magenta]: Follows most promising links based on relevance\n"
+        "• [green]Early Stopping[/green]: Stops when confidence threshold is reached\n"
+        "• [yellow]Two Strategies[/yellow]: Statistical (fast) vs Embedding (semantic)\n\n"
+        "Perfect for research tasks where you need 'just enough' information!",
+        title="Feature Overview",
+        border_style="blue"
+    ))
+    
+    await asyncio.sleep(2)
+    
+    # Demo 1: Basic adaptive crawling with confidence visualization
+    console.print("\n[bold yellow]Demo 1: Statistical Strategy - Fast Topic Understanding[/bold yellow]\n")
+    
+    query = "Python async web scraping best practices"
+    console.print(f"[cyan]🔍 Research Query:[/cyan] [bold]{query}[/bold]")
+    console.print(f"[cyan]🎯 Goal:[/cyan] Gather enough information to understand the topic")
+    console.print(f"[cyan]📊 Strategy:[/cyan] Statistical (keyword-based, fast)\n")
+    
+    # Configure adaptive crawler
+    config = AdaptiveConfig(
+        strategy="statistical",
+        max_pages=3,  # Limit to 3 pages for demo
+        confidence_threshold=0.7,  # Stop at 70% confidence
+        top_k_links=2,  # Follow top 2 links per page
+        min_gain_threshold=0.05  # Need 5% information gain to continue
+    )
+    
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        adaptive = AdaptiveCrawler(crawler, config)
+        
+        # Create progress tracking
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            console=console
+        ) as progress:
+            
+            # Track crawling progress
+            crawl_task = progress.add_task("[cyan]Starting adaptive crawl...", total=None)
+            
+            # Start crawling
+            start_time = time.time()
+            result = await adaptive.digest(
+                start_url="https://docs.python.org/3/library/asyncio.html",
+                query=query
+            )
+            elapsed = time.time() - start_time
+            
+            progress.remove_task(crawl_task)
+        
+        # Display results with visual confidence meter
+        console.print(f"\n[bold green]✅ Crawling Complete in {elapsed:.1f} seconds![/bold green]\n")
+        
+        # Create confidence visualization
+        confidence = adaptive.confidence
+        conf_percentage = int(confidence * 100)
+        conf_bar = "█" * (conf_percentage // 5) + "░" * (20 - conf_percentage // 5)
+        
+        console.print(f"[bold]Confidence Level:[/bold] [{('green' if confidence >= 0.7 else 'yellow' if confidence >= 0.5 else 'red')}]{conf_bar}[/] {conf_percentage}%")
+        
+        # Show crawl statistics
+        stats_table = Table(
+            title="📊 Crawl Statistics",
+            box=box.ROUNDED,
+            show_lines=True
+        )
+        
+        stats_table.add_column("Metric", style="cyan", width=25)
+        stats_table.add_column("Value", style="white", width=20)
+        
+        stats_table.add_row("Pages Crawled", str(len(result.crawled_urls)))
+        stats_table.add_row("Knowledge Base Size", f"{len(adaptive.state.knowledge_base)} documents")
+        # Calculate total content from CrawlResult objects
+        total_content = 0
+        for doc in adaptive.state.knowledge_base:
+            if hasattr(doc, 'markdown') and doc.markdown and hasattr(doc.markdown, 'raw_markdown'):
+                total_content += len(doc.markdown.raw_markdown)
+        stats_table.add_row("Total Content", f"{total_content:,} chars")
+        stats_table.add_row("Time per Page", f"{elapsed / len(result.crawled_urls):.2f}s")
+        
+        console.print(stats_table)
+        
+        # Show top relevant pages
+        console.print("\n[bold]🏆 Most Relevant Pages Found:[/bold]")
+        relevant_pages = adaptive.get_relevant_content(top_k=3)
+        for i, page in enumerate(relevant_pages, 1):
+            console.print(f"\n{i}. [bold]{page['url']}[/bold]")
+            console.print(f"   Relevance: {page['score']:.2%}")
+            
+            # Show key information extracted
+            content = page['content'] or ""
+            if content:
+                # Find most relevant sentence
+                sentences = [s.strip() for s in content.split('.') if s.strip()]
+                if sentences:
+                    console.print(f"   [dim]Key insight: {sentences[0]}...[/dim]")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 2...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 2: Early Stopping Demonstration
+    console.print("\n[bold yellow]Demo 2: Early Stopping - Stop When We Know Enough[/bold yellow]\n")
+    
+    query2 = "Python requests library tutorial"
+    console.print(f"[cyan]🔍 Research Query:[/cyan] [bold]{query2}[/bold]")
+    console.print(f"[cyan]🎯 Goal:[/cyan] Stop as soon as we reach 60% confidence")
+    console.print("[dim]Watch how adaptive crawling stops early when it has enough info[/dim]\n")
+    
+    # Configure for early stopping
+    early_stop_config = AdaptiveConfig(
+        strategy="statistical",
+        max_pages=10,  # Allow up to 10, but will stop early
+        confidence_threshold=0.6,  # Lower threshold for demo
+        top_k_links=2
+    )
+    
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        adaptive_early = AdaptiveCrawler(crawler, early_stop_config)
+        
+        # Track progress
+        console.print("[cyan]Starting crawl with early stopping enabled...[/cyan]")
+        start_time = time.time()
+        
+        result = await adaptive_early.digest(
+            start_url="https://docs.python-requests.org/en/latest/",
+            query=query2
+        )
+        
+        elapsed = time.time() - start_time
+        
+        # Show results
+        console.print(f"\n[bold green]✅ Stopped early at {int(adaptive_early.confidence * 100)}% confidence![/bold green]")
+        console.print(f"• Crawled only {len(result.crawled_urls)} pages (max was 10)")
+        console.print(f"• Saved time: ~{elapsed:.1f}s total")
+        console.print(f"• Efficiency: {elapsed / len(result.crawled_urls):.1f}s per page\n")
+        
+        # Show why it stopped
+        if adaptive_early.confidence >= 0.6:
+            console.print("[green]✓ Reached confidence threshold - no need to crawl more![/green]")
+        else:
+            console.print("[yellow]⚠ Hit max pages limit before reaching threshold[/yellow]")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 3...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 3: Knowledge Base Export/Import
+    console.print("\n[bold yellow]Demo 3: Knowledge Base Export & Import[/bold yellow]\n")
+    
+    query3 = "Python decorators tutorial"
+    console.print(f"[cyan]🔍 Research Query:[/cyan] [bold]{query3}[/bold]")
+    console.print("[dim]Build knowledge base, export it, then import for continued research[/dim]\n")
+    
+    # First crawl - build knowledge base
+    export_config = AdaptiveConfig(
+        strategy="statistical",
+        max_pages=2,  # Small for demo
+        confidence_threshold=0.5
+    )
+    
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # Phase 1: Initial research
+        console.print("[bold]Phase 1: Initial Research[/bold]")
+        adaptive1 = AdaptiveCrawler(crawler, export_config)
+        
+        result1 = await adaptive1.digest(
+            start_url="https://realpython.com/",
+            query=query3
+        )
+        
+        console.print(f"✓ Built knowledge base with {len(adaptive1.state.knowledge_base)} documents")
+        console.print(f"✓ Confidence: {int(adaptive1.confidence * 100)}%\n")
+        
+        # Export knowledge base
+        console.print("[bold]💾 Exporting Knowledge Base:[/bold]")
+        kb_export = adaptive1.export_knowledge_base()
+        
+        export_stats = {
+            "documents": len(kb_export['documents']),
+            "urls": len(kb_export['visited_urls']),
+            "size": len(json.dumps(kb_export)),
+            "confidence": kb_export['confidence']
+        }
+        
+        for key, value in export_stats.items():
+            console.print(f"• {key.capitalize()}: {value:,}" if isinstance(value, int) else f"• {key.capitalize()}: {value:.2%}")
+        
+        # Phase 2: Import and continue
+        console.print("\n[bold]Phase 2: Import & Continue Research[/bold]")
+        adaptive2 = AdaptiveCrawler(crawler, export_config)
+        
+        # Import the knowledge base
+        await adaptive2.import_knowledge_base(kb_export)
+        console.print(f"✓ Imported {len(adaptive2.state.knowledge_base)} documents")
+        console.print(f"✓ Starting confidence: {int(adaptive2.confidence * 100)}%")
+        
+        # Continue research from a different starting point
+        console.print("\n[cyan]Continuing research from a different angle...[/cyan]")
+        result2 = await adaptive2.digest(
+            start_url="https://docs.python.org/3/glossary.html#term-decorator",
+            query=query3
+        )
+        
+        console.print(f"\n[bold green]✅ Research Complete![/bold green]")
+        console.print(f"• Total documents: {len(adaptive2.state.knowledge_base)}")
+        console.print(f"• Final confidence: {int(adaptive2.confidence * 100)}%")
+        console.print(f"• Knowledge preserved across sessions!")
+    
+    # Summary
+    console.print("\n[bold green]✨ Adaptive Crawling Benefits:[/bold green]")
+    console.print("• Automatically stops when enough information is gathered")
+    console.print("• Follows most promising links based on relevance")
+    console.print("• Saves time and resources with intelligent exploration")
+    console.print("• Export/import knowledge bases for continued research")
+    console.print("• Choose strategy based on needs: speed vs semantic understanding\n")
+
+
+async def virtual_scroll_demo(auto_mode=False):
+    """
+    📜 Virtual Scroll Demo
+    Shows how to capture content from modern infinite scroll pages
+    """
+    import os
+    import http.server
+    import socketserver
+    import threading
+    from pathlib import Path
+    
+    print_banner(
+        "📜 VIRTUAL SCROLL SUPPORT",
+        "Capture all content from pages with DOM recycling"
+    )
+    
+    # Explain the feature
+    console.print(Panel(
+        "[bold]What is Virtual Scroll?[/bold]\n\n"
+        "Virtual Scroll handles modern web pages that use DOM recycling techniques:\n\n"
+        "• [cyan]Twitter/X-like feeds[/cyan]: Content replaced as you scroll\n"
+        "• [magenta]Instagram grids[/magenta]: Visual content with virtualization\n"
+        "• [green]News feeds[/green]: Mixed content with different behaviors\n"
+        "• [yellow]Infinite scroll[/yellow]: Captures everything, not just visible\n\n"
+        "Without this, you'd only get the initially visible content!",
+        title="Feature Overview",
+        border_style="blue"
+    ))
+    
+    await asyncio.sleep(2)
+    
+    # Start test server with HTML examples
+    ASSETS_DIR = Path(__file__).parent / "assets"
+    
+    class TestServer:
+        """Simple HTTP server to serve our test HTML files"""
+        
+        def __init__(self, port=8080):
+            self.port = port
+            self.httpd = None
+            self.server_thread = None
+            
+        async def start(self):
+            """Start the test server"""
+            Handler = http.server.SimpleHTTPRequestHandler
+            
+            # Save current directory and change to assets directory
+            self.original_cwd = os.getcwd()
+            os.chdir(ASSETS_DIR)
+            
+            # Try to find an available port
+            for _ in range(10):
+                try:
+                    self.httpd = socketserver.TCPServer(("", self.port), Handler)
+                    break
+                except OSError:
+                    self.port += 1
+                    
+            if self.httpd is None:
+                raise RuntimeError("Could not find available port")
+                
+            self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+            self.server_thread.daemon = True
+            self.server_thread.start()
+            
+            # Give server time to start
+            await asyncio.sleep(0.5)
+            
+            console.print(f"[green]Test server started on http://localhost:{self.port}[/green]")
+            return self.port
+            
+        def stop(self):
+            """Stop the test server"""
+            if self.httpd:
+                self.httpd.shutdown()
+            # Restore original directory
+            if hasattr(self, 'original_cwd'):
+                os.chdir(self.original_cwd)
+    
+    server = TestServer()
+    port = await server.start()
+    
+    try:
+        # Demo 1: Twitter-like virtual scroll (content REPLACED)
+        console.print("\n[bold yellow]Demo 1: Twitter-like Virtual Scroll - Content Replaced[/bold yellow]\n")
+        console.print("[cyan]This simulates Twitter/X where only visible tweets exist in DOM[/cyan]\n")
+        
+        url = f"http://localhost:{port}/virtual_scroll_twitter_like.html"
+        
+        # First, crawl WITHOUT virtual scroll
+        console.print("[red]WITHOUT Virtual Scroll:[/red]")
+        
+        config_normal = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+        browser_config = BrowserConfig(
+            headless=False if not auto_mode else True,
+            viewport={"width": 1280, "height": 800}
+        )
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result_normal = await crawler.arun(url=url, config=config_normal)
+            
+            # Count tweets
+            tweets_normal = len(set(re.findall(r'data-tweet-id="(\d+)"', result_normal.html)))
+            console.print(f"• Captured only {tweets_normal} tweets (initial visible)")
+            console.print(f"• HTML size: {len(result_normal.html):,} bytes\n")
+        
+        # Then, crawl WITH virtual scroll  
+        console.print("[green]WITH Virtual Scroll:[/green]")
+        
+        virtual_config = VirtualScrollConfig(
+            container_selector="#timeline",
+            scroll_count=50,
+            scroll_by="container_height",
+            wait_after_scroll=0.2
+        )
+        
+        config_virtual = CrawlerRunConfig(
+            virtual_scroll_config=virtual_config,
+            cache_mode=CacheMode.BYPASS
+        )
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result_virtual = await crawler.arun(url=url, config=config_virtual)
+            
+            tweets_virtual = len(set(re.findall(r'data-tweet-id="(\d+)"', result_virtual.html)))
+            console.print(f"• Captured {tweets_virtual} tweets (all content)")
+            console.print(f"• HTML size: {len(result_virtual.html):,} bytes")
+            console.print(f"• [bold]{tweets_virtual / tweets_normal if tweets_normal > 0 else 'N/A':.1f}x more content![/bold]\n")
+        
+        if not auto_mode:
+            console.print("\n[dim]Press Enter to continue to Demo 2...[/dim]")
+            input()
+        else:
+            await asyncio.sleep(1)
+    
+        # Demo 2: Instagram Grid Example
+        console.print("\n[bold yellow]Demo 2: Instagram Grid - Visual Grid Layout[/bold yellow]\n")
+        console.print("[cyan]This shows how virtual scroll works with grid layouts[/cyan]\n")
+        
+        url2 = f"http://localhost:{port}/virtual_scroll_instagram_grid.html"
+        
+        # Configure for grid layout
+        grid_config = VirtualScrollConfig(
+            container_selector=".feed-container",
+            scroll_count=100,  # Many scrolls for 999 posts
+            scroll_by="container_height",
+            wait_after_scroll=0.1 if auto_mode else 0.3
+        )
+        
+        config = CrawlerRunConfig(
+            virtual_scroll_config=grid_config,
+            cache_mode=CacheMode.BYPASS,
+            screenshot=True  # Take a screenshot
+        )
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=url2, config=config)
+            
+            # Count posts in grid
+            posts = re.findall(r'data-post-id="(\d+)"', result.html)
+            unique_posts = sorted(set(int(id) for id in posts))
+            
+            console.print(f"[green]✅ Results:[/green]")
+            console.print(f"• Posts captured: {len(unique_posts)} unique posts")
+            if unique_posts:
+                console.print(f"• Post IDs range: {min(unique_posts)} to {max(unique_posts)}")
+                console.print(f"• Expected: 0 to 998 (999 posts total)")
+                
+                if len(unique_posts) >= 900:
+                    console.print(f"• [bold green]SUCCESS! Captured {len(unique_posts)/999*100:.1f}% of all posts[/bold green]")
+        
+        if not auto_mode:
+            console.print("\n[dim]Press Enter to continue to Demo 3...[/dim]")
+            input()
+        else:
+            await asyncio.sleep(1)
+    
+        # Demo 3: Show the actual code
+        console.print("\n[bold yellow]Demo 3: The Code - How It Works[/bold yellow]\n")
+        
+        # Show the actual implementation
+        code = '''# Example: Crawling Twitter-like feed with virtual scroll
+url = "http://localhost:8080/virtual_scroll_twitter_like.html"
+
+# Configure virtual scroll
+virtual_config = VirtualScrollConfig(
+    container_selector="#timeline",      # The scrollable container
+    scroll_count=50,                    # Max number of scrolls
+    scroll_by="container_height",       # Scroll by container height
+    wait_after_scroll=0.3              # Wait 300ms after each scroll
+)
+
+config = CrawlerRunConfig(
+    virtual_scroll_config=virtual_config,
+    cache_mode=CacheMode.BYPASS
+)
+
+# Use headless=False to watch it work!
+browser_config = BrowserConfig(
+    headless=False,
+    viewport={"width": 1280, "height": 800}
+)
+
+async with AsyncWebCrawler(config=browser_config) as crawler:
+    result = await crawler.arun(url=url, config=config)
+    
+    # Extract all tweets
+    tweets = re.findall(r\'data-tweet-id="(\\d+)"\', result.html)
+    unique_tweets = set(tweets)
+    
+    print(f"Captured {len(unique_tweets)} unique tweets!")
+    print(f"Without virtual scroll: only ~10 tweets")
+    print(f"With virtual scroll: all 500 tweets!")'''
+        
+        syntax = Syntax(code, "python", theme="monokai", line_numbers=True)
+        console.print(Panel(syntax, title="Implementation", border_style="green"))
+    
+        # Summary
+        console.print("\n[bold green]✨ Virtual Scroll Benefits:[/bold green]")
+        console.print("• Captures ALL content, not just initially visible")
+        console.print("• Handles Twitter, Instagram, LinkedIn, and more")
+        console.print("• Smart scrolling with configurable parameters")
+        console.print("• Essential for modern web scraping")
+        console.print("• Works with any virtualized content\n")
+        
+    finally:
+        # Stop the test server
+        server.stop()
+        console.print("[dim]Test server stopped[/dim]")
+
+
+async def url_seeder_demo(auto_mode=False):
+    """
+    🌱 URL Seeder Demo
+    Shows intelligent URL discovery and filtering
+    """
+    print_banner(
+        "🌱 URL SEEDER - INTELLIGENT URL DISCOVERY",
+        "Pre-discover and filter URLs before crawling"
+    )
+    
+    # Explain the feature
+    console.print(Panel(
+        "[bold]What is URL Seeder?[/bold]\n\n"
+        "URL Seeder enables intelligent crawling at scale by pre-discovering URLs:\n\n"
+        "• [cyan]Discovery[/cyan]: Find all URLs from sitemaps or by crawling\n"
+        "• [magenta]Filtering[/magenta]: Filter by patterns, dates, or content\n"
+        "• [green]Ranking[/green]: Score URLs by relevance (BM25 or semantic)\n"
+        "• [yellow]Metadata[/yellow]: Extract head data without full crawl\n\n"
+        "Perfect for targeted crawling of large websites!",
+        title="Feature Overview",
+        border_style="blue"
+    ))
+    
+    await asyncio.sleep(2)
+    
+    # Demo 1: Basic URL discovery
+    console.print("\n[bold yellow]Demo 1: Discover URLs from Sitemap[/bold yellow]\n")
+    
+    target_site = "realpython.com"
+    console.print(f"[cyan]🔍 Target:[/cyan] [bold]{target_site}[/bold]")
+    console.print("[dim]Let's discover what content is available[/dim]\n")
+    
+    async with AsyncUrlSeeder() as seeder:
+        # First, see total URLs available
+        console.print("[cyan]Discovering ALL URLs from sitemap...[/cyan]")
+        
+        all_urls = await seeder.urls(
+            target_site, 
+            SeedingConfig(source="sitemap")
+        )
+        
+        console.print(f"[green]✅ Found {len(all_urls)} total URLs![/green]\n")
+        
+        # Show URL categories
+        categories = {}
+        for url_info in all_urls[:100]:  # Sample first 100
+            url = url_info['url']
+            if '/tutorials/' in url:
+                categories['tutorials'] = categories.get('tutorials', 0) + 1
+            elif '/python-' in url:
+                categories['python-topics'] = categories.get('python-topics', 0) + 1
+            elif '/courses/' in url:
+                categories['courses'] = categories.get('courses', 0) + 1
+            else:
+                categories['other'] = categories.get('other', 0) + 1
+        
+        console.print("[bold]URL Categories (sample of first 100):[/bold]")
+        for cat, count in sorted(categories.items(), key=lambda x: x[1], reverse=True):
+            console.print(f"• {cat}: {count} URLs")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 2...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 2: Pattern filtering
+    console.print("\n[bold yellow]Demo 2: Filter URLs by Pattern[/bold yellow]\n")
+    
+    pattern = "*python-basics*"
+    console.print(f"[cyan]🎯 Pattern:[/cyan] [bold]{pattern}[/bold]")
+    console.print("[dim]Finding Python basics tutorials[/dim]\n")
+    
+    async with AsyncUrlSeeder() as seeder:
+        filtered_urls = await seeder.urls(
+            target_site,
+            SeedingConfig(
+                source="sitemap",
+                pattern=pattern,
+                max_urls=10
+            )
+        )
+        
+        console.print(f"[green]✅ Found {len(filtered_urls)} Python basics URLs:[/green]\n")
+        
+        for i, url_info in enumerate(filtered_urls[:5], 1):
+            console.print(f"{i}. {url_info['url']}")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 3...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 3: Smart search with BM25 ranking
+    console.print("\n[bold yellow]Demo 3: Smart Search with BM25 Ranking[/bold yellow]\n")
+    
+    query = "web scraping beautifulsoup tutorial"
+    console.print(f"[cyan]🔍 Query:[/cyan] [bold]{query}[/bold]")
+    console.print("[dim]Using BM25 to find most relevant content[/dim]\n")
+    
+    async with AsyncUrlSeeder() as seeder:
+        # Search with relevance scoring
+        results = await seeder.urls(
+            target_site,
+            SeedingConfig(
+                source="sitemap",
+                pattern="*beautiful-soup*",  # Find Beautiful Soup pages
+                extract_head=True,  # Get metadata
+                query=query,
+                scoring_method="bm25",
+                # No threshold - show all results ranked by BM25
+                max_urls=10
+            )
+        )
+        
+        console.print(f"[green]✅ Top {len(results)} most relevant results:[/green]\n")
+        
+        # Create a table for results
+        table = Table(
+            title="🎯 Relevance-Ranked Results",
+            box=box.ROUNDED,
+            show_lines=True
+        )
+        
+        table.add_column("Rank", style="cyan", width=6)
+        table.add_column("Score", style="yellow", width=8)
+        table.add_column("Title", style="white", width=50)
+        table.add_column("URL", style="dim", width=40)
+        
+        for i, result in enumerate(results[:5], 1):
+            score = result.get('relevance_score', 0)
+            title = result.get('head_data', {}).get('title', 'No title')[:50]
+            url = result['url'].split('/')[-2]  # Just the slug
+            
+            table.add_row(
+                f"#{i}",
+                f"{score:.3f}",
+                title,
+                f".../{url}/"
+            )
+        
+        console.print(table)
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 4...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 4: Complete pipeline - Discover → Filter → Crawl
+    console.print("\n[bold yellow]Demo 4: Complete Pipeline - Discover → Filter → Crawl[/bold yellow]\n")
+    
+    console.print("[cyan]Let's build a complete crawling pipeline:[/cyan]")
+    console.print("1. Discover URLs about Python decorators")
+    console.print("2. Filter and rank by relevance")
+    console.print("3. Crawl top results\n")
+    
+    async with AsyncUrlSeeder() as seeder:
+        # Step 1: Discover and filter
+        console.print("[bold]Step 1: Discovering decorator tutorials...[/bold]")
+        
+        decorator_urls = await seeder.urls(
+            target_site,
+            SeedingConfig(
+                source="sitemap",
+                pattern="*decorator*",
+                extract_head=True,
+                query="python decorators tutorial examples",
+                scoring_method="bm25",
+                max_urls=5
+            )
+        )
+        
+        console.print(f"Found {len(decorator_urls)} relevant URLs\n")
+        
+        # Step 2: Show what we'll crawl
+        console.print("[bold]Step 2: URLs to crawl (ranked by relevance):[/bold]")
+        urls_to_crawl = []
+        for i, url_info in enumerate(decorator_urls[:3], 1):
+            urls_to_crawl.append(url_info['url'])
+            title = url_info.get('head_data', {}).get('title', 'No title')
+            console.print(f"{i}. {title[:60]}...")
+            console.print(f"   [dim]{url_info['url']}[/dim]")
+        
+        # Step 3: Crawl them
+        console.print("\n[bold]Step 3: Crawling selected URLs...[/bold]")
+        
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(
+                only_text=True,
+                cache_mode=CacheMode.BYPASS
+            )
+            
+            # Crawl just the first URL for demo
+            if urls_to_crawl:
+                console.print(f"\n[dim]Crawling first URL: {urls_to_crawl[0]}[/dim]")
+                result = await crawler.arun(urls_to_crawl[0], config=config)
+                
+                if result.success:
+                    console.print(f"\n[green]✅ Successfully crawled the page![/green]")
+                    console.print("\n[bold]Sample content:[/bold]")
+                    content = result.markdown.raw_markdown[:300].replace('\n', ' ')
+                    console.print(f"[dim]{content}...[/dim]")
+                else:
+                    console.print(f"[red]Failed to crawl: {result.error_message}[/red]")
+    
+    # Show code example
+    console.print("\n[bold yellow]Code Example:[/bold yellow]\n")
+    
+    code = '''# Complete URL Seeder pipeline
+async with AsyncUrlSeeder() as seeder:
+    # 1. Discover and filter URLs
+    urls = await seeder.urls(
+        "example.com",
+        SeedingConfig(
+            source="sitemap",              # or "crawl" 
+            pattern="*tutorial*",          # URL pattern
+            extract_head=True,             # Get metadata
+            query="python web scraping",   # Search query
+            scoring_method="bm25",         # Ranking method
+            score_threshold=0.2,           # Quality filter
+            max_urls=10                    # Max URLs
+        )
+    )
+    
+    # 2. Extract just the URLs
+    urls_to_crawl = [u["url"] for u in urls[:5]]
+    
+    # 3. Crawl them efficiently
+    async with AsyncWebCrawler() as crawler:
+        results = await crawler.arun_many(urls_to_crawl)
+        
+        async for result in results:
+            if result.success:
+                print(f"Crawled: {result.url}")
+                # Process content...'''
+    
+    syntax = Syntax(code, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, title="Implementation", border_style="green"))
+    
+    # Summary
+    console.print("\n[bold green]✨ URL Seeder Benefits:[/bold green]")
+    console.print("• Pre-discover URLs before crawling - save time!")
+    console.print("• Filter by patterns, dates, or content relevance")
+    console.print("• Rank URLs by BM25 or semantic similarity")
+    console.print("• Extract metadata without full crawl")
+    console.print("• Perfect for large-scale targeted crawling\n")
+
+
+async def c4a_script_demo(auto_mode=False):
+    """
+    🎭 C4A Script Demo
+    Shows the power of our domain-specific language for web automation
+    """
+    print_banner(
+        "🎭 C4A SCRIPT - AUTOMATION MADE SIMPLE",
+        "Domain-specific language for complex web interactions"
+    )
+    
+    # Explain the feature
+    console.print(Panel(
+        "[bold]What is C4A Script?[/bold]\n\n"
+        "C4A Script is a simple yet powerful language for web automation:\n\n"
+        "• [cyan]English-like syntax[/cyan]: IF, CLICK, TYPE, WAIT - intuitive commands\n"
+        "• [magenta]Smart transpiler[/magenta]: Converts to optimized JavaScript\n"
+        "• [green]Error handling[/green]: Helpful error messages with suggestions\n"
+        "• [yellow]Reusable procedures[/yellow]: Build complex workflows easily\n\n"
+        "Perfect for automating logins, handling popups, pagination, and more!",
+        title="Feature Overview",
+        border_style="blue"
+    ))
+    
+    await asyncio.sleep(2)
+    
+    # Demo 1: Basic transpilation demonstration
+    console.print("\n[bold yellow]Demo 1: Understanding C4A Script Transpilation[/bold yellow]\n")
+    
+    simple_script = """# Handle cookie banner and scroll
+WAIT `body` 2
+IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
+SCROLL DOWN 500
+WAIT 1"""
+    
+    console.print("[cyan]C4A Script:[/cyan]")
+    syntax = Syntax(simple_script, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, border_style="cyan"))
+    
+    # Compile it
+    from crawl4ai import c4a_compile
+    
+    console.print("\n[cyan]Transpiling to JavaScript...[/cyan]")
+    result = c4a_compile(simple_script)
+    
+    if result.success:
+        console.print("[green]✅ Compilation successful![/green]\n")
+        console.print("[cyan]Generated JavaScript:[/cyan]")
+        
+        js_display = "\n".join(result.js_code)
+        js_syntax = Syntax(js_display, "javascript", theme="monokai", line_numbers=True)
+        console.print(Panel(js_syntax, border_style="green"))
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 2...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 2: Error handling showcase
+    console.print("\n[bold yellow]Demo 2: Smart Error Detection & Suggestions[/bold yellow]\n")
+    
+    # Script with intentional errors
+    error_script = """WAIT body 2
+CLICK button.submit
+IF (EXISTS .modal) CLICK .close"""
+    
+    console.print("[cyan]C4A Script with errors:[/cyan]")
+    syntax = Syntax(error_script, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, border_style="red"))
+    
+    console.print("\n[cyan]Compiling...[/cyan]")
+    result = c4a_compile(error_script)
+    
+    if not result.success:
+        console.print("[red]❌ Compilation failed (as expected)[/red]\n")
+        
+        # Show the first error
+        error = result.first_error
+        console.print(f"[bold red]Error at line {error.line}, column {error.column}:[/bold red]")
+        console.print(f"[yellow]{error.message}[/yellow]")
+        console.print(f"\nProblematic code: [red]{error.source_line}[/red]")
+        console.print(" " * (16 + error.column) + "[red]^[/red]")
+        
+        if error.suggestions:
+            console.print("\n[green]💡 Suggestions:[/green]")
+            for suggestion in error.suggestions:
+                console.print(f"   • {suggestion.message}")
+    
+    # Show the fixed version
+    fixed_script = """WAIT `body` 2
+CLICK `button.submit`
+IF (EXISTS `.modal`) THEN CLICK `.close`"""
+    
+    console.print("\n[cyan]Fixed C4A Script:[/cyan]")
+    syntax = Syntax(fixed_script, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, border_style="green"))
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 3...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 3: Real-world example - E-commerce automation
+    console.print("\n[bold yellow]Demo 3: Real-World E-commerce Automation[/bold yellow]\n")
+    
+    console.print("[cyan]Scenario:[/cyan] Automate product search with smart handling\n")
+    
+    ecommerce_script = """# E-commerce Product Search Automation
+# Define reusable procedures
+PROC handle_popups
+  # Close cookie banner if present
+  IF (EXISTS `.cookie-notice`) THEN CLICK `.cookie-accept`
+  
+  # Close newsletter popup if it appears
+  IF (EXISTS `#newsletter-modal`) THEN CLICK `.modal-close`
+ENDPROC
+
+PROC search_product
+  # Click search box and type query
+  CLICK `.search-input`
+  TYPE "wireless headphones"
+  PRESS Enter
+  
+  # Wait for results
+  WAIT `.product-grid` 10
+ENDPROC
+
+# Main automation flow
+SET max_products = 50
+
+# Step 1: Navigate and handle popups
+GO https://shop.example.com
+WAIT `body` 3
+handle_popups
+
+# Step 2: Perform search
+search_product
+
+# Step 3: Load more products (infinite scroll)
+REPEAT (SCROLL DOWN 1000, `document.querySelectorAll('.product-card').length < 50`)
+
+# Step 4: Apply filters
+IF (EXISTS `.filter-price`) THEN CLICK `input[data-filter="under-100"]`
+WAIT 2
+
+# Step 5: Extract product count
+EVAL `console.log('Found ' + document.querySelectorAll('.product-card').length + ' products')`"""
+    
+    syntax = Syntax(ecommerce_script, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, title="E-commerce Automation Script", border_style="cyan"))
+    
+    # Compile and show results
+    console.print("\n[cyan]Compiling automation script...[/cyan]")
+    result = c4a_compile(ecommerce_script)
+    
+    if result.success:
+        console.print(f"[green]✅ Successfully compiled to {len(result.js_code)} JavaScript statements![/green]")
+        console.print("\n[bold]Script Analysis:[/bold]")
+        console.print(f"• Procedures defined: {len(result.metadata.get('procedures', []))}")
+        console.print(f"• Variables used: {len(result.metadata.get('variables', []))}")
+        console.print(f"• Total commands: {result.metadata.get('total_commands', 0)}")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to continue to Demo 4...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 4: Integration with Crawl4AI - LIVE DEMO
+    console.print("\n[bold yellow]Demo 4: Live Integration with Crawl4AI[/bold yellow]\n")
+    
+    console.print("[cyan]Let's see C4A Script in action with real web crawling![/cyan]\n")
+    
+    # Create a simple C4A script for demo
+    live_script = """# Handle common website patterns
+WAIT `body` 2
+# Close cookie banner if exists
+IF (EXISTS `.cookie-banner, .cookie-notice, #cookie-consent`) THEN CLICK `.accept, .agree, button[aria-label*="accept"]`
+# Scroll to load content
+SCROLL DOWN 500
+WAIT 1"""
+    
+    console.print("[bold]Our C4A Script:[/bold]")
+    syntax = Syntax(live_script, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, border_style="cyan"))
+    
+    # Method 1: Direct C4A Script usage
+    console.print("\n[bold cyan]Method 1: Direct C4A Script Integration[/bold cyan]\n")
+    
+    try:
+        # Import necessary components
+        from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+        
+        # Define extraction schema
+        schema = {
+            "name": "page_content",
+            "selector": "body",
+            "fields": {
+                "title": {"selector": "h1, title", "type": "text"},
+                "paragraphs": {"selector": "p", "type": "list", "fields": {"text": {"type": "text"}}},
+                "links": {"selector": "a[href]", "type": "list", "fields": {"text": {"type": "text"}, "href": {"type": "attribute", "attribute": "href"}}}
+            }
+        }
+        
+        # Create config with C4A script
+        config = CrawlerRunConfig(
+            c4a_script=live_script,
+            extraction_strategy=JsonCssExtractionStrategy(schema),
+            only_text=True,
+            cache_mode=CacheMode.BYPASS
+        )
+        
+        console.print("[green]✅ Config created with C4A script![/green]")
+        console.print(f"[dim]The C4A script will be automatically transpiled when crawling[/dim]\n")
+        
+        # Show the actual code
+        code_example1 = f'''# Live code that's actually running:
+config = CrawlerRunConfig(
+    c4a_script="""{live_script}""",
+    extraction_strategy=JsonCssExtractionStrategy(schema),
+    only_text=True,
+    cache_mode=CacheMode.BYPASS
+)
+
+# This would run the crawler:
+# async with AsyncWebCrawler() as crawler:
+#     result = await crawler.arun("https://example.com", config=config)
+#     print(f"Extracted {{len(result.extracted_content)}} items")'''
+        
+        syntax = Syntax(code_example1, "python", theme="monokai", line_numbers=True)
+        console.print(Panel(syntax, title="Method 1: Direct Integration (Live Code)", border_style="green"))
+        
+    except Exception as e:
+        console.print(f"[red]Error in demo: {e}[/red]")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to see Method 2...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Method 2: Pre-compilation approach
+    console.print("\n[bold cyan]Method 2: Pre-compile and Reuse[/bold cyan]\n")
+    
+    # Advanced script with procedures
+    advanced_script = """# E-commerce automation with procedures
+PROC handle_popups
+  IF (EXISTS `.popup-overlay`) THEN CLICK `.popup-close`
+  IF (EXISTS `#newsletter-modal`) THEN CLICK `.modal-dismiss`
+ENDPROC
+
+PROC load_all_products  
+  # Keep scrolling until no more products load
+  REPEAT (SCROLL DOWN 1000, `document.querySelectorAll('.product').length < window.lastProductCount`)
+  EVAL `window.lastProductCount = document.querySelectorAll('.product').length`
+ENDPROC
+
+# Main flow
+WAIT `.products-container` 5
+handle_popups
+EVAL `window.lastProductCount = 0`
+load_all_products"""
+    
+    console.print("[bold]Advanced C4A Script with Procedures:[/bold]")
+    syntax = Syntax(advanced_script, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, border_style="cyan"))
+    
+    # Actually compile it
+    console.print("\n[cyan]Compiling the script...[/cyan]")
+    compilation_result = c4a_compile(advanced_script)
+    
+    if compilation_result.success:
+        console.print(f"[green]✅ Successfully compiled to {len(compilation_result.js_code)} JavaScript statements![/green]\n")
+        
+        # Show first few JS statements
+        console.print("[bold]Generated JavaScript (first 5 statements):[/bold]")
+        js_preview = "\n".join(compilation_result.js_code[:5])
+        if len(compilation_result.js_code) > 5:
+            js_preview += f"\n... and {len(compilation_result.js_code) - 5} more statements"
+        
+        js_syntax = Syntax(js_preview, "javascript", theme="monokai", line_numbers=True)
+        console.print(Panel(js_syntax, border_style="green"))
+        
+        # Create actual config with compiled code
+        config_with_js = CrawlerRunConfig(
+            js_code=compilation_result.js_code,
+            wait_for="css:.products-container",
+            cache_mode=CacheMode.BYPASS
+        )
+        
+        console.print("\n[green]✅ Config created with pre-compiled JavaScript![/green]")
+        
+        # Show the actual implementation
+        code_example2 = f'''# Live code showing pre-compilation:
+# Step 1: Compile once
+result = c4a_compile(advanced_script)
+if result.success:
+    js_code = result.js_code  # {len(compilation_result.js_code)} statements generated
+    
+    # Step 2: Use compiled code multiple times
+    config = CrawlerRunConfig(
+        js_code=js_code,
+        wait_for="css:.products-container",
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    # Step 3: Run crawler with compiled code
+    # async with AsyncWebCrawler() as crawler:
+    #     # Can reuse js_code for multiple URLs
+    #     for url in ["shop1.com", "shop2.com"]:
+    #         result = await crawler.arun(url, config=config)
+else:
+    print(f"Compilation error: {{result.first_error.message}}")'''
+        
+        syntax = Syntax(code_example2, "python", theme="monokai", line_numbers=True)
+        console.print(Panel(syntax, title="Method 2: Pre-compilation (Live Code)", border_style="green"))
+        
+    else:
+        console.print(f"[red]Compilation failed: {compilation_result.first_error.message}[/red]")
+    
+    if not auto_mode:
+        console.print("\n[dim]Press Enter to see a real-world example...[/dim]")
+        input()
+    else:
+        await asyncio.sleep(1)
+    
+    # Demo 5: Real-world example with actual crawling
+    console.print("\n[bold yellow]Demo 5: Real-World Example - News Site Automation[/bold yellow]\n")
+    
+    news_script = """# News site content extraction
+# Wait for main content
+WAIT `article, .article-content, main` 5
+
+# Handle common annoyances
+IF (EXISTS `.cookie-notice`) THEN CLICK `button[class*="accept"]`
+IF (EXISTS `.newsletter-popup`) THEN CLICK `.close, .dismiss`
+
+# Expand "Read More" sections
+IF (EXISTS `.read-more-button`) THEN CLICK `.read-more-button`
+
+# Load comments if available
+IF (EXISTS `.load-comments`) THEN CLICK `.load-comments`
+WAIT 2"""
+    
+    console.print("[bold]News Site Automation Script:[/bold]")
+    syntax = Syntax(news_script, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, border_style="cyan"))
+    
+    # Create and show actual working config
+    console.print("\n[cyan]Creating crawler configuration...[/cyan]")
+    
+    news_config = CrawlerRunConfig(
+        c4a_script=news_script,
+        wait_for="css:article",
+        only_text=True,
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    console.print("[green]✅ Configuration ready for crawling![/green]\n")
+    
+    # Show how to actually use it
+    usage_example = '''# Complete working example:
+async def crawl_news_site():
+    """Crawl a news site with C4A automation"""
+    
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        result = await crawler.arun(
+            url="https://example-news.com/article",
+            config=CrawlerRunConfig(
+                c4a_script=news_script,
+                wait_for="css:article",
+                only_text=True
+            )
+        )
+        
+        if result.success:
+            print(f"✓ Crawled: {result.url}")
+            print(f"✓ Content length: {len(result.markdown.raw_markdown)} chars")
+            print(f"✓ Links found: {len(result.links.get('internal', []))} internal")
+            
+            # The C4A script ensured we:
+            # - Handled cookie banners
+            # - Expanded collapsed content  
+            # - Loaded dynamic comments
+            # All automatically!
+        
+        return result
+
+# Run it:
+# result = await crawl_news_site()'''
+    
+    syntax = Syntax(usage_example, "python", theme="monokai", line_numbers=True)
+    console.print(Panel(syntax, title="Complete Working Example", border_style="green"))
+    
+    # Summary
+    console.print("\n[bold green]✨ What We Demonstrated:[/bold green]")
+    console.print("• C4A Script transpiles to optimized JavaScript automatically")
+    console.print("• Direct integration via `c4a_script` parameter - easiest approach")
+    console.print("• Pre-compilation via `c4a_compile()` - best for reuse")
+    console.print("• Real configs that you can copy and use immediately")
+    console.print("• Actual code running, not just examples!\n")
+
+
+async def interactive_menu():
+    """Interactive menu to select demos"""
+    from rich.prompt import Prompt
+    
+    demos = {
+        "1": ("Link Preview & Scoring", link_preview_demo),
+        "2": ("Adaptive Crawling", adaptive_crawling_demo),
+        "3": ("Virtual Scroll", virtual_scroll_demo),
+        "4": ("URL Seeder", url_seeder_demo),
+        "5": ("C4A Script", c4a_script_demo),
+        "6": ("LLM Context Builder", lambda auto: console.print("[yellow]LLM Context demo coming soon![/yellow]")),
+        "7": ("Run All Demos", None),  # Special case
+        "0": ("Exit", None)
+    }
+    
+    while True:
+        # Clear screen for better presentation
+        console.clear()
+        
+        print_banner(
+            "🚀 CRAWL4AI v0.7.0 SHOWCASE",
+            "Interactive Demo Menu"
+        )
+        
+        console.print("\n[bold cyan]Select a demo to run:[/bold cyan]\n")
+        
+        for key, (name, _) in demos.items():
+            if key == "0":
+                console.print(f"\n[dim]{key}. {name}[/dim]")
+            else:
+                console.print(f"{key}. {name}")
+        
+        choice = Prompt.ask("\n[bold]Enter your choice[/bold]", choices=list(demos.keys()))
+        
+        if choice == "0":
+            console.print("\n[yellow]Thanks for exploring Crawl4AI v0.7.0![/yellow]")
+            break
+        elif choice == "7":
+            # Run all demos
+            console.clear()
+            for key in ["1", "3", "4", "5"]:  # Link Preview, Virtual Scroll, URL Seeder, C4A Script
+                name, demo_func = demos[key]
+                if demo_func:
+                    await demo_func(auto_mode=True)
+                    console.print("\n[dim]Press Enter to continue...[/dim]")
+                    input()
+        else:
+            name, demo_func = demos[choice]
+            if demo_func:
+                console.clear()
+                await demo_func(auto_mode=False)
+                console.print("\n[dim]Press Enter to return to menu...[/dim]")
+                input()
+
+
+async def main():
+    """Run all feature demonstrations"""
+    import sys
+    
+    # Check command line arguments
+    interactive_mode = "--interactive" in sys.argv or "-i" in sys.argv
+    auto_mode = "--auto" in sys.argv
+    
+    if interactive_mode:
+        await interactive_menu()
+    elif auto_mode:
+        console.print("[yellow]Running in AUTO MODE - skipping user prompts[/yellow]\n")
+        
+        # Run demos automatically
+        await link_preview_demo(auto_mode=True)
+        await asyncio.sleep(2)
+        # await adaptive_crawling_demo(auto_mode=True)  # Skip for now
+        await virtual_scroll_demo(auto_mode=True)
+        await asyncio.sleep(2)
+        await url_seeder_demo(auto_mode=True)
+        await asyncio.sleep(2)
+        await c4a_script_demo(auto_mode=True)
+    else:
+        # Default: run all demos with prompts
+        try:
+            # 1. Link Preview Demo
+            await link_preview_demo(auto_mode=False)
+            
+            console.print("\n[dim]Press Enter to continue to Virtual Scroll demo...[/dim]")
+            input()
+            
+            # 2. Virtual Scroll Demo
+            await virtual_scroll_demo(auto_mode=False)
+            
+            console.print("\n[dim]Press Enter to continue to URL Seeder demo...[/dim]")
+            input()
+            
+            # 3. URL Seeder Demo
+            await url_seeder_demo(auto_mode=False)
+            
+            console.print("\n[dim]Press Enter to continue to C4A Script demo...[/dim]")
+            input()
+            
+            # 4. C4A Script Demo
+            await c4a_script_demo(auto_mode=False)
+            
+            # TODO: Add other demos here
+            # await llm_context_demo()
+            
+            console.print("\n[bold green]✨ All demos completed![/bold green]")
+            console.print("\nTo explore individual demos, run: [cyan]python crawl4ai_v0_7_0_showcase.py --interactive[/cyan]")
+            
+        except KeyboardInterrupt:
+            console.print("\n[yellow]Demo interrupted by user[/yellow]")
+        except Exception as e:
+            console.print(f"\n[red]Error: {str(e)}[/red]")
+            import traceback
+            traceback.print_exc()
+
+
+if __name__ == "__main__":
+    import sys
+    
+    # Show usage if --help is provided
+    if "--help" in sys.argv or "-h" in sys.argv:
+        console.print("\n[bold]Crawl4AI v0.7.0 Feature Showcase[/bold]\n")
+        console.print("Usage: python crawl4ai_v0_7_0_showcase.py [options]\n")
+        console.print("Options:")
+        console.print("  --interactive, -i    Interactive menu to select demos")
+        console.print("  --auto              Run all demos without user prompts")
+        console.print("  --help, -h          Show this help message\n")
+        console.print("Default: Run all demos with prompts between each\n")
+    else:
+        asyncio.run(main())
\ No newline at end of file
diff --git a/docs/releases_review/demo_v0.7.0.py b/docs/releases_review/demo_v0.7.0.py
new file mode 100644
index 0000000..53f4ccd
--- /dev/null
+++ b/docs/releases_review/demo_v0.7.0.py
@@ -0,0 +1,408 @@
+"""
+🚀 Crawl4AI v0.7.0 Release Demo
+================================
+This demo showcases all major features introduced in v0.7.0 release.
+
+Major Features:
+1. ✅ Adaptive Crawling - Intelligent crawling with confidence tracking
+2. ✅ Virtual Scroll Support - Handle infinite scroll pages
+3. ✅ Link Preview - Advanced link analysis with 3-layer scoring
+4. ✅ URL Seeder - Smart URL discovery and filtering
+5. ✅ C4A Script - Domain-specific language for web automation
+6. ✅ Chrome Extension Updates - Click2Crawl and instant schema extraction
+7. ✅ PDF Parsing Support - Extract content from PDF documents
+8. ✅ Nightly Builds - Automated nightly releases
+
+Run this demo to see all features in action!
+"""
+
+import asyncio
+import json
+from typing import List, Dict
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich import box
+
+from crawl4ai import (
+    AsyncWebCrawler, 
+    CrawlerRunConfig, 
+    BrowserConfig,
+    CacheMode,
+    AdaptiveCrawler, 
+    AdaptiveConfig,
+    AsyncUrlSeeder, 
+    SeedingConfig,
+    c4a_compile,
+    CompilationResult
+)
+from crawl4ai.async_configs import VirtualScrollConfig, LinkPreviewConfig
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+console = Console()
+
+def print_section(title: str, description: str = ""):
+    """Print a section header"""
+    console.print(f"\n[bold cyan]{'=' * 60}[/bold cyan]")
+    console.print(f"[bold yellow]{title}[/bold yellow]")
+    if description:
+        console.print(f"[dim]{description}[/dim]")
+    console.print(f"[bold cyan]{'=' * 60}[/bold cyan]\n")
+
+
+async def demo_1_adaptive_crawling():
+    """Demo 1: Adaptive Crawling - Intelligent content extraction"""
+    print_section(
+        "Demo 1: Adaptive Crawling",
+        "Intelligently learns and adapts to website patterns"
+    )
+    
+    # Create adaptive crawler with custom configuration
+    config = AdaptiveConfig(
+        strategy="statistical",  # or "embedding"
+        confidence_threshold=0.7,
+        max_pages=10,
+        top_k_links=3,
+        min_gain_threshold=0.1
+    )
+    
+    # Example: Learn from a product page
+    console.print("[cyan]Learning from product page patterns...[/cyan]")
+    
+    async with AsyncWebCrawler() as crawler:
+        adaptive = AdaptiveCrawler(crawler, config)
+        
+        # Start adaptive crawl
+        console.print("[cyan]Starting adaptive crawl...[/cyan]")
+        result = await adaptive.digest(
+            start_url="https://docs.python.org/3/",
+            query="python decorators tutorial"
+        )
+        
+        console.print("[green]✓ Adaptive crawl completed[/green]")
+        console.print(f"  - Confidence Level: {adaptive.confidence:.0%}")
+        console.print(f"  - Pages Crawled: {len(result.crawled_urls)}")
+        console.print(f"  - Knowledge Base: {len(adaptive.state.knowledge_base)} documents")
+        
+        # Get most relevant content
+        relevant = adaptive.get_relevant_content(top_k=3)
+        if relevant:
+            console.print("\nMost relevant pages:")
+            for i, page in enumerate(relevant, 1):
+                console.print(f"  {i}. {page['url']} (relevance: {page['score']:.2%})")
+
+
+async def demo_2_virtual_scroll():
+    """Demo 2: Virtual Scroll - Handle infinite scroll pages"""
+    print_section(
+        "Demo 2: Virtual Scroll Support",
+        "Capture content from modern infinite scroll pages"
+    )
+    
+    # Configure virtual scroll - using body as container for example.com
+    scroll_config = VirtualScrollConfig(
+        container_selector="body",  # Using body since example.com has simple structure
+        scroll_count=3,  # Just 3 scrolls for demo
+        scroll_by="container_height",  # or "page_height" or pixel value
+        wait_after_scroll=0.5  # Wait 500ms after each scroll
+    )
+    
+    config = CrawlerRunConfig(
+        virtual_scroll_config=scroll_config,
+        cache_mode=CacheMode.BYPASS,
+        wait_until="networkidle"
+    )
+    
+    console.print("[cyan]Virtual Scroll Configuration:[/cyan]")
+    console.print(f"  - Container: {scroll_config.container_selector}")
+    console.print(f"  - Scroll count: {scroll_config.scroll_count}")
+    console.print(f"  - Scroll by: {scroll_config.scroll_by}")
+    console.print(f"  - Wait after scroll: {scroll_config.wait_after_scroll}s")
+    
+    console.print("\n[dim]Note: Using example.com for demo - in production, use this[/dim]")
+    console.print("[dim]with actual infinite scroll pages like social media feeds.[/dim]\n")
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            config=config
+        )
+        
+        if result.success:
+            console.print("[green]✓ Virtual scroll executed successfully![/green]")
+            console.print(f"  - Content length: {len(result.markdown)} chars")
+            
+            # Show example of how to use with real infinite scroll sites
+            console.print("\n[yellow]Example for real infinite scroll sites:[/yellow]")
+            console.print("""
+# For Twitter-like feeds:
+scroll_config = VirtualScrollConfig(
+    container_selector="[data-testid='primaryColumn']",
+    scroll_count=20,
+    scroll_by="container_height",
+    wait_after_scroll=1.0
+)
+
+# For Instagram-like grids:
+scroll_config = VirtualScrollConfig(
+    container_selector="main article",
+    scroll_count=15,
+    scroll_by=1000,  # Fixed pixel amount
+    wait_after_scroll=1.5
+)""")
+
+
+async def demo_3_link_preview():
+    """Demo 3: Link Preview with 3-layer scoring"""
+    print_section(
+        "Demo 3: Link Preview & Scoring",
+        "Advanced link analysis with intrinsic, contextual, and total scoring"
+    )
+    
+    # Configure link preview
+    link_config = LinkPreviewConfig(
+        include_internal=True,
+        include_external=False,
+        max_links=10,
+        concurrency=5,
+        query="python tutorial",  # For contextual scoring
+        score_threshold=0.3,
+        verbose=True
+    )
+    
+    config = CrawlerRunConfig(
+        link_preview_config=link_config,
+        score_links=True,  # Enable intrinsic scoring
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    console.print("[cyan]Analyzing links with 3-layer scoring system...[/cyan]")
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://docs.python.org/3/", config=config)
+        
+        if result.success and result.links:
+            # Get scored links
+            internal_links = result.links.get("internal", [])
+            scored_links = [l for l in internal_links if l.get("total_score")]
+            scored_links.sort(key=lambda x: x.get("total_score", 0), reverse=True)
+            
+            # Create a scoring table
+            table = Table(title="Link Scoring Results", box=box.ROUNDED)
+            table.add_column("Link Text", style="cyan", width=40)
+            table.add_column("Intrinsic Score", justify="center")
+            table.add_column("Contextual Score", justify="center")
+            table.add_column("Total Score", justify="center", style="bold green")
+            
+            for link in scored_links[:5]:
+                text = link.get('text', 'No text')[:40]
+                table.add_row(
+                    text,
+                    f"{link.get('intrinsic_score', 0):.1f}/10",
+                    f"{link.get('contextual_score', 0):.2f}/1",
+                    f"{link.get('total_score', 0):.3f}"
+                )
+            
+            console.print(table)
+
+
+async def demo_4_url_seeder():
+    """Demo 4: URL Seeder - Smart URL discovery"""
+    print_section(
+        "Demo 4: URL Seeder",
+        "Intelligent URL discovery and filtering"
+    )
+    
+    # Configure seeding
+    seeding_config = SeedingConfig(
+        source="cc+sitemap",  # or "crawl"
+        pattern="*tutorial*",  # URL pattern filter
+        max_urls=50,
+        extract_head=True,  # Get metadata
+        query="python programming",  # For relevance scoring
+        scoring_method="bm25",
+        score_threshold=0.2,
+        force = True
+    )
+    
+    console.print("[cyan]URL Seeder Configuration:[/cyan]")
+    console.print(f"  - Source: {seeding_config.source}")
+    console.print(f"  - Pattern: {seeding_config.pattern}")
+    console.print(f"  - Max URLs: {seeding_config.max_urls}")
+    console.print(f"  - Query: {seeding_config.query}")
+    console.print(f"  - Scoring: {seeding_config.scoring_method}")
+    
+    # Use URL seeder to discover URLs
+    async with AsyncUrlSeeder() as seeder:
+        console.print("\n[cyan]Discovering URLs from Python docs...[/cyan]")
+        urls = await seeder.urls("docs.python.org", seeding_config)
+        
+        console.print(f"\n[green]✓ Discovered {len(urls)} URLs[/green]")
+        for i, url_info in enumerate(urls[:5], 1):
+            console.print(f"  {i}. {url_info['url']}")
+            if url_info.get('relevance_score'):
+                console.print(f"     Relevance: {url_info['relevance_score']:.3f}")
+
+
+async def demo_5_c4a_script():
+    """Demo 5: C4A Script - Domain-specific language"""
+    print_section(
+        "Demo 5: C4A Script Language",
+        "Domain-specific language for web automation"
+    )
+    
+    # Example C4A script
+    c4a_script = """
+# Simple C4A script example
+WAIT `body` 3
+IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
+CLICK `.search-button`
+TYPE "python tutorial"
+PRESS Enter
+WAIT `.results` 5
+"""
+    
+    console.print("[cyan]C4A Script Example:[/cyan]")
+    console.print(Panel(c4a_script, title="script.c4a", border_style="blue"))
+    
+    # Compile the script
+    compilation_result = c4a_compile(c4a_script)
+    
+    if compilation_result.success:
+        console.print("[green]✓ Script compiled successfully![/green]")
+        console.print(f"  - Generated {len(compilation_result.js_code)} JavaScript statements")
+        console.print("\nFirst 3 JS statements:")
+        for stmt in compilation_result.js_code[:3]:
+            console.print(f"  • {stmt}")
+    else:
+        console.print("[red]✗ Script compilation failed[/red]")
+        if compilation_result.first_error:
+            error = compilation_result.first_error
+            console.print(f"  Error at line {error.line}: {error.message}")
+
+
+async def demo_6_css_extraction():
+    """Demo 6: Enhanced CSS/JSON extraction"""
+    print_section(
+        "Demo 6: Enhanced Extraction",
+        "Improved CSS selector and JSON extraction"
+    )
+    
+    # Define extraction schema
+    schema = {
+        "name": "Example Page Data",
+        "baseSelector": "body",
+        "fields": [
+            {
+                "name": "title",
+                "selector": "h1",
+                "type": "text"
+            },
+            {
+                "name": "paragraphs",
+                "selector": "p",
+                "type": "list",
+                "fields": [
+                    {"name": "text", "type": "text"}
+                ]
+            }
+        ]
+    }
+    
+    extraction_strategy = JsonCssExtractionStrategy(schema)
+    
+    console.print("[cyan]Extraction Schema:[/cyan]")
+    console.print(json.dumps(schema, indent=2))
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            config=CrawlerRunConfig(
+                extraction_strategy=extraction_strategy,
+                cache_mode=CacheMode.BYPASS
+            )
+        )
+        
+        if result.success and result.extracted_content:
+            console.print("\n[green]✓ Content extracted successfully![/green]")
+            console.print(f"Extracted: {json.dumps(json.loads(result.extracted_content), indent=2)[:200]}...")
+
+
+async def demo_7_performance_improvements():
+    """Demo 7: Performance improvements"""
+    print_section(
+        "Demo 7: Performance Improvements",
+        "Faster crawling with better resource management"
+    )
+    
+    # Performance-optimized configuration
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.ENABLED,  # Use caching
+        wait_until="domcontentloaded",  # Faster than networkidle
+        page_timeout=10000,  # 10 second timeout
+        exclude_external_links=True,
+        exclude_social_media_links=True,
+        exclude_external_images=True
+    )
+    
+    console.print("[cyan]Performance Configuration:[/cyan]")
+    console.print("  - Cache: ENABLED")
+    console.print("  - Wait: domcontentloaded (faster)")
+    console.print("  - Timeout: 10s")
+    console.print("  - Excluding: external links, images, social media")
+    
+    # Measure performance
+    import time
+    start_time = time.time()
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com", config=config)
+        
+    elapsed = time.time() - start_time
+    
+    if result.success:
+        console.print(f"\n[green]✓ Page crawled in {elapsed:.2f} seconds[/green]")
+
+
+async def main():
+    """Run all demos"""
+    console.print(Panel(
+        "[bold cyan]Crawl4AI v0.7.0 Release Demo[/bold cyan]\n\n"
+        "This demo showcases all major features introduced in v0.7.0.\n"
+        "Each demo is self-contained and demonstrates a specific feature.",
+        title="Welcome",
+        border_style="blue"
+    ))
+    
+    demos = [
+        demo_1_adaptive_crawling,
+        demo_2_virtual_scroll,
+        demo_3_link_preview,
+        demo_4_url_seeder,
+        demo_5_c4a_script,
+        demo_6_css_extraction,
+        demo_7_performance_improvements
+    ]
+    
+    for i, demo in enumerate(demos, 1):
+        try:
+            await demo()
+            if i < len(demos):
+                console.print("\n[dim]Press Enter to continue to next demo...[/dim]")
+                input()
+        except Exception as e:
+            console.print(f"[red]Error in demo: {e}[/red]")
+            continue
+    
+    console.print(Panel(
+        "[bold green]Demo Complete![/bold green]\n\n"
+        "Thank you for trying Crawl4AI v0.7.0!\n"
+        "For more examples and documentation, visit:\n"
+        "https://github.com/unclecode/crawl4ai",
+        title="Complete",
+        border_style="green"
+    ))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/releases_review/demo_v0.7.5.py b/docs/releases_review/demo_v0.7.5.py
new file mode 100644
index 0000000..bda472a
--- /dev/null
+++ b/docs/releases_review/demo_v0.7.5.py
@@ -0,0 +1,338 @@
+"""
+🚀 Crawl4AI v0.7.5 Release Demo - Working Examples
+==================================================
+This demo showcases key features introduced in v0.7.5 with real, executable examples.
+
+Featured Demos:
+1. ✅ Docker Hooks System - Real API calls with custom hooks (string & function-based)
+2. ✅ Enhanced LLM Integration - Working LLM configurations
+3. ✅ HTTPS Preservation - Live crawling with HTTPS maintenance
+
+Requirements:
+- crawl4ai v0.7.5 installed
+- Docker running with crawl4ai image (optional for Docker demos)
+- Valid API keys for LLM demos (optional)
+"""
+
+import asyncio
+import requests
+import time
+import sys
+
+from crawl4ai import (AsyncWebCrawler, CrawlerRunConfig, BrowserConfig,
+                      CacheMode, FilterChain, URLPatternFilter, BFSDeepCrawlStrategy,
+                      hooks_to_string)
+from crawl4ai.docker_client import Crawl4aiDockerClient
+    
+
+def print_section(title: str, description: str = ""):
+    """Print a section header"""
+    print(f"\n{'=' * 60}")
+    print(f"{title}")
+    if description:
+        print(f"{description}")
+    print(f"{'=' * 60}\n")
+
+
+async def demo_1_docker_hooks_system():
+    """Demo 1: Docker Hooks System - Real API calls with custom hooks"""
+    print_section(
+        "Demo 1: Docker Hooks System",
+        "Testing both string-based and function-based hooks (NEW in v0.7.5!)"
+    )
+
+    # Check Docker service availability
+    def check_docker_service():
+        try:
+            response = requests.get("http://localhost:11235/", timeout=3)
+            return response.status_code == 200
+        except:
+            return False
+
+    print("Checking Docker service...")
+    docker_running = check_docker_service()
+
+    if not docker_running:
+        print("⚠️  Docker service not running on localhost:11235")
+        print("To test Docker hooks:")
+        print("1. Run: docker run -p 11235:11235 unclecode/crawl4ai:latest")
+        print("2. Wait for service to start")
+        print("3. Re-run this demo\n")
+        return
+
+    print("✓ Docker service detected!")
+
+    # ============================================================================
+    # PART 1: Traditional String-Based Hooks (Works with REST API)
+    # ============================================================================
+    print("\n" + "─" * 60)
+    print("Part 1: String-Based Hooks (REST API)")
+    print("─" * 60)
+
+    hooks_config_string = {
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("[String Hook] Setting up page context")
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    return page
+""",
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    print("[String Hook] Before retrieving HTML")
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    return page
+"""
+    }
+
+    payload = {
+        "urls": ["https://httpbin.org/html"],
+        "hooks": {
+            "code": hooks_config_string,
+            "timeout": 30
+        }
+    }
+
+    print("🔧 Using string-based hooks for REST API...")
+    try:
+        start_time = time.time()
+        response = requests.post("http://localhost:11235/crawl", json=payload, timeout=60)
+        execution_time = time.time() - start_time
+
+        if response.status_code == 200:
+            result = response.json()
+            print(f"✅ String-based hooks executed in {execution_time:.2f}s")
+            if result.get('results') and result['results'][0].get('success'):
+                html_length = len(result['results'][0].get('html', ''))
+                print(f"   📄 HTML length: {html_length} characters")
+        else:
+            print(f"❌ Request failed: {response.status_code}")
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+
+    # ============================================================================
+    # PART 2: NEW Function-Based Hooks with Docker Client (v0.7.5)
+    # ============================================================================
+    print("\n" + "─" * 60)
+    print("Part 2: Function-Based Hooks with Docker Client (✨ NEW!)")
+    print("─" * 60)
+
+    # Define hooks as regular Python functions
+    async def on_page_context_created_func(page, context, **kwargs):
+        """Block images to speed up crawling"""
+        print("[Function Hook] Setting up page context")
+        await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+        await page.set_viewport_size({"width": 1920, "height": 1080})
+        return page
+
+    async def before_goto_func(page, context, url, **kwargs):
+        """Add custom headers before navigation"""
+        print(f"[Function Hook] About to navigate to {url}")
+        await page.set_extra_http_headers({
+            'X-Crawl4AI': 'v0.7.5-function-hooks',
+            'X-Test-Header': 'demo'
+        })
+        return page
+
+    async def before_retrieve_html_func(page, context, **kwargs):
+        """Scroll to load lazy content"""
+        print("[Function Hook] Scrolling page for lazy-loaded content")
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        await page.wait_for_timeout(500)
+        await page.evaluate("window.scrollTo(0, 0)")
+        return page
+
+    # Use the hooks_to_string utility (can be used standalone)
+    print("\n📦 Converting functions to strings with hooks_to_string()...")
+    hooks_as_strings = hooks_to_string({
+        "on_page_context_created": on_page_context_created_func,
+        "before_goto": before_goto_func,
+        "before_retrieve_html": before_retrieve_html_func
+    })
+    print(f"   ✓ Converted {len(hooks_as_strings)} hooks to string format")
+
+    # OR use Docker Client which does conversion automatically!
+    print("\n🐳 Using Docker Client with automatic conversion...")
+    try:
+        client = Crawl4aiDockerClient(base_url="http://localhost:11235")
+
+        # Pass function objects directly - conversion happens automatically!
+        results = await client.crawl(
+            urls=["https://httpbin.org/html"],
+            hooks={
+                "on_page_context_created": on_page_context_created_func,
+                "before_goto": before_goto_func,
+                "before_retrieve_html": before_retrieve_html_func
+            },
+            hooks_timeout=30
+        )
+
+        if results and results.success:
+            print(f"✅ Function-based hooks executed successfully!")
+            print(f"   📄 HTML length: {len(results.html)} characters")
+            print(f"   🎯 URL: {results.url}")
+        else:
+            print("⚠️ Crawl completed but may have warnings")
+
+    except Exception as e:
+        print(f"❌ Docker client error: {str(e)}")
+
+    # Show the benefits
+    print("\n" + "=" * 60)
+    print("✨ Benefits of Function-Based Hooks:")
+    print("=" * 60)
+    print("✓ Full IDE support (autocomplete, syntax highlighting)")
+    print("✓ Type checking and linting")
+    print("✓ Easier to test and debug")
+    print("✓ Reusable across projects")
+    print("✓ Automatic conversion in Docker client")
+    print("=" * 60)
+
+
+async def demo_2_enhanced_llm_integration():
+    """Demo 2: Enhanced LLM Integration - Working LLM configurations"""
+    print_section(
+        "Demo 2: Enhanced LLM Integration",
+        "Testing custom LLM providers and configurations"
+    )
+
+    print("🤖 Testing Enhanced LLM Integration Features")
+
+    provider = "gemini/gemini-2.5-flash-lite"
+    payload = {
+        "url": "https://example.com",
+        "f": "llm",
+        "q": "Summarize this page in one sentence.",
+        "provider": provider,  # Explicitly set provider
+        "temperature": 0.7
+    }
+    try:
+        response = requests.post(
+            "http://localhost:11235/md",
+            json=payload,
+            timeout=60
+        )
+        if response.status_code == 200:
+            result = response.json()
+            print(f"✓ Request successful with provider: {provider}")
+            print(f"  - Response keys: {list(result.keys())}")
+            print(f"  - Content length: {len(result.get('markdown', ''))} characters")
+            print(f"  - Note: Actual LLM call may fail without valid API key")
+        else:
+            print(f"❌ Request failed: {response.status_code}")
+            print(f"  - Response: {response.text[:500]}")
+            
+    except Exception as e:
+        print(f"[red]Error: {e}[/]")
+
+
+async def demo_3_https_preservation():
+    """Demo 3: HTTPS Preservation - Live crawling with HTTPS maintenance"""
+    print_section(
+        "Demo 3: HTTPS Preservation",
+        "Testing HTTPS preservation for internal links"
+    )
+
+    print("🔒 Testing HTTPS Preservation Feature")
+
+    # Test with HTTPS preservation enabled
+    print("\nTest 1: HTTPS Preservation ENABLED")
+
+    url_filter = URLPatternFilter(
+        patterns=["^(https:\/\/)?quotes\.toscrape\.com(\/.*)?$"]
+    )
+    config = CrawlerRunConfig(
+        exclude_external_links=True, 
+        stream=True, 
+        verbose=False,
+        preserve_https_for_internal_links=True,
+        deep_crawl_strategy=BFSDeepCrawlStrategy(
+            max_depth=2, 
+            max_pages=5,
+            filter_chain=FilterChain([url_filter])
+        )
+    )
+
+    test_url = "https://quotes.toscrape.com"
+    print(f"🎯 Testing URL: {test_url}")
+
+    async with AsyncWebCrawler() as crawler:
+        async for result in await crawler.arun(url=test_url, config=config):
+            print("✓ HTTPS Preservation Test Completed")
+            internal_links = [i['href'] for i in result.links['internal']]
+            for link in internal_links:
+                print(f"  → {link}")
+
+
+async def main():
+    """Run all demos"""
+    print("\n" + "=" * 60)
+    print("🚀 Crawl4AI v0.7.5 Working Demo")
+    print("=" * 60)
+
+    # Check system requirements
+    print("🔍 System Requirements Check:")
+    print(f"  - Python version: {sys.version.split()[0]} {'✓' if sys.version_info >= (3, 10) else '❌ (3.10+ required)'}")
+
+    try:
+        import requests
+        print(f"  - Requests library: ✓")
+    except ImportError:
+        print(f"  - Requests library: ❌")
+
+    print()
+
+    demos = [
+        ("Docker Hooks System", demo_1_docker_hooks_system),
+        ("Enhanced LLM Integration", demo_2_enhanced_llm_integration),
+        ("HTTPS Preservation", demo_3_https_preservation),
+    ]
+
+    for i, (name, demo_func) in enumerate(demos, 1):
+        try:
+            print(f"\n📍 Starting Demo {i}/{len(demos)}: {name}")
+            await demo_func()
+
+            if i < len(demos):
+                print(f"\n✨ Demo {i} complete! Press Enter for next demo...")
+                input()
+
+        except KeyboardInterrupt:
+            print(f"\n⏹️  Demo interrupted by user")
+            break
+        except Exception as e:
+            print(f"❌ Demo {i} error: {str(e)}")
+            print("Continuing to next demo...")
+            continue
+
+    print("\n" + "=" * 60)
+    print("🎉 Demo Complete!")
+    print("=" * 60)
+    print("You've experienced the power of Crawl4AI v0.7.5!")
+    print("")
+    print("Key Features Demonstrated:")
+    print("🔧 Docker Hooks - String-based & function-based (NEW!)")
+    print("   • hooks_to_string() utility for function conversion")
+    print("   • Docker client with automatic conversion")
+    print("   • Full IDE support and type checking")
+    print("🤖 Enhanced LLM - Better AI integration")
+    print("🔒 HTTPS Preservation - Secure link handling")
+    print("")
+    print("Ready to build something amazing? 🚀")
+    print("")
+    print("📖 Docs: https://docs.crawl4ai.com/")
+    print("🐙 GitHub: https://github.com/unclecode/crawl4ai")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    print("🚀 Crawl4AI v0.7.5 Live Demo Starting...")
+    print("Press Ctrl+C anytime to exit\n")
+
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\n👋 Demo stopped by user. Thanks for trying Crawl4AI v0.7.5!")
+    except Exception as e:
+        print(f"\n❌ Demo error: {str(e)}")
+        print("Make sure you have the required dependencies installed.")
diff --git a/docs/releases_review/demo_v0.7.6.py b/docs/releases_review/demo_v0.7.6.py
new file mode 100644
index 0000000..5d59adf
--- /dev/null
+++ b/docs/releases_review/demo_v0.7.6.py
@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""
+Crawl4AI v0.7.6 Release Demo
+============================
+
+This demo showcases the major feature in v0.7.6:
+**Webhook Support for Docker Job Queue API**
+
+Features Demonstrated:
+1. Asynchronous job processing with webhook notifications
+2. Webhook support for /crawl/job endpoint
+3. Webhook support for /llm/job endpoint
+4. Notification-only vs data-in-payload modes
+5. Custom webhook headers for authentication
+6. Structured extraction with JSON schemas
+7. Exponential backoff retry for reliable delivery
+
+Prerequisites:
+- Crawl4AI Docker container running on localhost:11235
+- Flask installed: pip install flask requests
+- LLM API key configured (for LLM examples)
+
+Usage:
+python docs/releases_review/demo_v0.7.6.py
+"""
+
+import requests
+import json
+import time
+from flask import Flask, request, jsonify
+from threading import Thread
+
+# Configuration
+CRAWL4AI_BASE_URL = "http://localhost:11235"
+WEBHOOK_BASE_URL = "http://localhost:8080"
+
+# Flask app for webhook receiver
+app = Flask(__name__)
+received_webhooks = []
+
+
+@app.route('/webhook', methods=['POST'])
+def webhook_handler():
+    """Universal webhook handler for both crawl and LLM extraction jobs."""
+    payload = request.json
+    task_id = payload['task_id']
+    task_type = payload['task_type']
+    status = payload['status']
+
+    print(f"\n{'='*70}")
+    print(f"📬 Webhook Received!")
+    print(f"   Task ID: {task_id}")
+    print(f"   Task Type: {task_type}")
+    print(f"   Status: {status}")
+    print(f"   Timestamp: {payload['timestamp']}")
+
+    if status == 'completed':
+        if 'data' in payload:
+            print(f"   ✅ Data included in webhook")
+            if task_type == 'crawl':
+                results = payload['data'].get('results', [])
+                print(f"   📊 Crawled {len(results)} URL(s)")
+            elif task_type == 'llm_extraction':
+                extracted = payload['data'].get('extracted_content', {})
+                print(f"   🤖 Extracted: {json.dumps(extracted, indent=6)}")
+        else:
+            print(f"   📥 Notification only (fetch data separately)")
+    elif status == 'failed':
+        print(f"   ❌ Error: {payload.get('error', 'Unknown')}")
+
+    print(f"{'='*70}\n")
+    received_webhooks.append(payload)
+
+    return jsonify({"status": "received"}), 200
+
+
+def start_webhook_server():
+    """Start Flask webhook server in background."""
+    app.run(host='0.0.0.0', port=8080, debug=False, use_reloader=False)
+
+
+def demo_1_crawl_webhook_notification_only():
+    """Demo 1: Crawl job with webhook notification (data fetched separately)."""
+    print("\n" + "="*70)
+    print("DEMO 1: Crawl Job - Webhook Notification Only")
+    print("="*70)
+    print("Submitting crawl job with webhook notification...")
+
+    payload = {
+        "urls": ["https://example.com"],
+        "browser_config": {"headless": True},
+        "crawler_config": {"cache_mode": "bypass"},
+        "webhook_config": {
+            "webhook_url": f"{WEBHOOK_BASE_URL}/webhook",
+            "webhook_data_in_payload": False,
+            "webhook_headers": {
+                "X-Demo": "v0.7.6",
+                "X-Type": "crawl"
+            }
+        }
+    }
+
+    response = requests.post(f"{CRAWL4AI_BASE_URL}/crawl/job", json=payload)
+    if response.ok:
+        task_id = response.json()['task_id']
+        print(f"✅ Job submitted: {task_id}")
+        print("⏳ Webhook will notify when complete...")
+        return task_id
+    else:
+        print(f"❌ Failed: {response.text}")
+        return None
+
+
+def demo_2_crawl_webhook_with_data():
+    """Demo 2: Crawl job with full data in webhook payload."""
+    print("\n" + "="*70)
+    print("DEMO 2: Crawl Job - Webhook with Full Data")
+    print("="*70)
+    print("Submitting crawl job with data included in webhook...")
+
+    payload = {
+        "urls": ["https://www.python.org"],
+        "browser_config": {"headless": True},
+        "crawler_config": {"cache_mode": "bypass"},
+        "webhook_config": {
+            "webhook_url": f"{WEBHOOK_BASE_URL}/webhook",
+            "webhook_data_in_payload": True,
+            "webhook_headers": {
+                "X-Demo": "v0.7.6",
+                "X-Type": "crawl-with-data"
+            }
+        }
+    }
+
+    response = requests.post(f"{CRAWL4AI_BASE_URL}/crawl/job", json=payload)
+    if response.ok:
+        task_id = response.json()['task_id']
+        print(f"✅ Job submitted: {task_id}")
+        print("⏳ Webhook will include full results...")
+        return task_id
+    else:
+        print(f"❌ Failed: {response.text}")
+        return None
+
+
+def demo_3_llm_webhook_notification_only():
+    """Demo 3: LLM extraction with webhook notification (NEW in v0.7.6!)."""
+    print("\n" + "="*70)
+    print("DEMO 3: LLM Extraction - Webhook Notification Only (NEW!)")
+    print("="*70)
+    print("Submitting LLM extraction job with webhook notification...")
+
+    payload = {
+        "url": "https://www.example.com",
+        "q": "Extract the main heading and description from this page",
+        "provider": "openai/gpt-4o-mini",
+        "cache": False,
+        "webhook_config": {
+            "webhook_url": f"{WEBHOOK_BASE_URL}/webhook",
+            "webhook_data_in_payload": False,
+            "webhook_headers": {
+                "X-Demo": "v0.7.6",
+                "X-Type": "llm"
+            }
+        }
+    }
+
+    response = requests.post(f"{CRAWL4AI_BASE_URL}/llm/job", json=payload)
+    if response.ok:
+        task_id = response.json()['task_id']
+        print(f"✅ Job submitted: {task_id}")
+        print("⏳ Webhook will notify when LLM extraction completes...")
+        return task_id
+    else:
+        print(f"❌ Failed: {response.text}")
+        return None
+
+
+def demo_4_llm_webhook_with_schema():
+    """Demo 4: LLM extraction with JSON schema and data in webhook (NEW in v0.7.6!)."""
+    print("\n" + "="*70)
+    print("DEMO 4: LLM Extraction - Schema + Full Data in Webhook (NEW!)")
+    print("="*70)
+    print("Submitting LLM extraction with JSON schema...")
+
+    schema = {
+        "type": "object",
+        "properties": {
+            "title": {"type": "string", "description": "Page title"},
+            "description": {"type": "string", "description": "Page description"},
+            "main_topics": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Main topics covered"
+            }
+        },
+        "required": ["title"]
+    }
+
+    payload = {
+        "url": "https://www.python.org",
+        "q": "Extract the title, description, and main topics from this website",
+        "schema": json.dumps(schema),
+        "provider": "openai/gpt-4o-mini",
+        "cache": False,
+        "webhook_config": {
+            "webhook_url": f"{WEBHOOK_BASE_URL}/webhook",
+            "webhook_data_in_payload": True,
+            "webhook_headers": {
+                "X-Demo": "v0.7.6",
+                "X-Type": "llm-with-schema"
+            }
+        }
+    }
+
+    response = requests.post(f"{CRAWL4AI_BASE_URL}/llm/job", json=payload)
+    if response.ok:
+        task_id = response.json()['task_id']
+        print(f"✅ Job submitted: {task_id}")
+        print("⏳ Webhook will include structured extraction results...")
+        return task_id
+    else:
+        print(f"❌ Failed: {response.text}")
+        return None
+
+
+def demo_5_global_webhook_config():
+    """Demo 5: Using global webhook configuration from config.yml."""
+    print("\n" + "="*70)
+    print("DEMO 5: Global Webhook Configuration")
+    print("="*70)
+    print("💡 You can configure a default webhook URL in config.yml:")
+    print("""
+    webhooks:
+      enabled: true
+      default_url: "https://myapp.com/webhooks/default"
+      data_in_payload: false
+      retry:
+        max_attempts: 5
+        initial_delay_ms: 1000
+        max_delay_ms: 32000
+        timeout_ms: 30000
+    """)
+    print("Then submit jobs WITHOUT webhook_config - they'll use the default!")
+    print("This is useful for consistent webhook handling across all jobs.")
+
+
+def demo_6_webhook_retry_logic():
+    """Demo 6: Webhook retry mechanism with exponential backoff."""
+    print("\n" + "="*70)
+    print("DEMO 6: Webhook Retry Logic")
+    print("="*70)
+    print("🔄 Webhook delivery uses exponential backoff retry:")
+    print("   • Max attempts: 5")
+    print("   • Delays: 1s → 2s → 4s → 8s → 16s")
+    print("   • Timeout: 30s per attempt")
+    print("   • Retries on: 5xx errors, network errors, timeouts")
+    print("   • No retry on: 4xx client errors")
+    print("\nThis ensures reliable webhook delivery even with temporary failures!")
+
+
+def print_summary():
+    """Print demo summary and results."""
+    print("\n" + "="*70)
+    print("📊 DEMO SUMMARY")
+    print("="*70)
+    print(f"Total webhooks received: {len(received_webhooks)}")
+
+    crawl_webhooks = [w for w in received_webhooks if w['task_type'] == 'crawl']
+    llm_webhooks = [w for w in received_webhooks if w['task_type'] == 'llm_extraction']
+
+    print(f"\nBreakdown:")
+    print(f"  🕷️  Crawl jobs: {len(crawl_webhooks)}")
+    print(f"  🤖 LLM extraction jobs: {len(llm_webhooks)}")
+
+    print(f"\nDetails:")
+    for i, webhook in enumerate(received_webhooks, 1):
+        icon = "🕷️" if webhook['task_type'] == 'crawl' else "🤖"
+        print(f"  {i}. {icon} {webhook['task_id']}: {webhook['status']}")
+
+    print("\n" + "="*70)
+    print("✨ v0.7.6 KEY FEATURES DEMONSTRATED:")
+    print("="*70)
+    print("✅ Webhook support for /crawl/job")
+    print("✅ Webhook support for /llm/job (NEW!)")
+    print("✅ Notification-only mode (fetch data separately)")
+    print("✅ Data-in-payload mode (get full results in webhook)")
+    print("✅ Custom headers for authentication")
+    print("✅ JSON schema for structured LLM extraction")
+    print("✅ Exponential backoff retry for reliable delivery")
+    print("✅ Global webhook configuration support")
+    print("✅ Universal webhook handler for both job types")
+    print("\n💡 Benefits:")
+    print("   • No more polling - get instant notifications")
+    print("   • Better resource utilization")
+    print("   • Reliable delivery with automatic retries")
+    print("   • Consistent API across crawl and LLM jobs")
+    print("   • Production-ready webhook infrastructure")
+
+
+def main():
+    """Run all demos."""
+    print("\n" + "="*70)
+    print("🚀 Crawl4AI v0.7.6 Release Demo")
+    print("="*70)
+    print("Feature: Webhook Support for Docker Job Queue API")
+    print("="*70)
+
+    # Check if server is running
+    try:
+        health = requests.get(f"{CRAWL4AI_BASE_URL}/health", timeout=5)
+        print(f"✅ Crawl4AI server is running")
+    except:
+        print(f"❌ Cannot connect to Crawl4AI at {CRAWL4AI_BASE_URL}")
+        print("Please start Docker container:")
+        print("  docker run -d -p 11235:11235 --env-file .llm.env unclecode/crawl4ai:0.7.6")
+        return
+
+    # Start webhook server
+    print(f"\n🌐 Starting webhook server at {WEBHOOK_BASE_URL}...")
+    webhook_thread = Thread(target=start_webhook_server, daemon=True)
+    webhook_thread.start()
+    time.sleep(2)
+
+    # Run demos
+    demo_1_crawl_webhook_notification_only()
+    time.sleep(5)
+
+    demo_2_crawl_webhook_with_data()
+    time.sleep(5)
+
+    demo_3_llm_webhook_notification_only()
+    time.sleep(5)
+
+    demo_4_llm_webhook_with_schema()
+    time.sleep(5)
+
+    demo_5_global_webhook_config()
+    demo_6_webhook_retry_logic()
+
+    # Wait for webhooks
+    print("\n⏳ Waiting for all webhooks to arrive...")
+    time.sleep(30)
+
+    # Print summary
+    print_summary()
+
+    print("\n" + "="*70)
+    print("✅ Demo completed!")
+    print("="*70)
+    print("\n📚 Documentation:")
+    print("   • deploy/docker/WEBHOOK_EXAMPLES.md")
+    print("   • docs/examples/docker_webhook_example.py")
+    print("\n🔗 Upgrade:")
+    print("   docker pull unclecode/crawl4ai:0.7.6")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/docs/releases_review/demo_v0.7.7.py b/docs/releases_review/demo_v0.7.7.py
new file mode 100644
index 0000000..ea00146
--- /dev/null
+++ b/docs/releases_review/demo_v0.7.7.py
@@ -0,0 +1,628 @@
+#!/usr/bin/env python3
+"""
+Crawl4AI v0.7.7 Release Demo
+============================
+
+This demo showcases the major feature in v0.7.7:
+**Self-Hosting with Real-time Monitoring Dashboard**
+
+Features Demonstrated:
+1. System health monitoring with live metrics
+2. Real-time request tracking (active & completed)
+3. Browser pool management (permanent/hot/cold pools)
+4. Monitor API endpoints for programmatic access
+5. WebSocket streaming for real-time updates
+6. Control actions (kill browser, cleanup, restart)
+7. Production metrics (efficiency, reuse rates, memory)
+
+Prerequisites:
+- Crawl4AI Docker container running on localhost:11235
+- Python packages: pip install httpx websockets
+
+Usage:
+python docs/releases_review/demo_v0.7.7.py
+"""
+
+import asyncio
+import httpx
+import json
+import time
+from datetime import datetime
+from typing import Dict, Any
+
+# Configuration
+CRAWL4AI_BASE_URL = "http://localhost:11235"
+MONITOR_DASHBOARD_URL = f"{CRAWL4AI_BASE_URL}/dashboard"
+
+
+def print_section(title: str, description: str = ""):
+    """Print a formatted section header"""
+    print(f"\n{'=' * 70}")
+    print(f"📊 {title}")
+    if description:
+        print(f"{description}")
+    print(f"{'=' * 70}\n")
+
+
+def print_subsection(title: str):
+    """Print a formatted subsection header"""
+    print(f"\n{'-' * 70}")
+    print(f"{title}")
+    print(f"{'-' * 70}")
+
+
+async def check_server_health():
+    """Check if Crawl4AI server is running"""
+    try:
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            response = await client.get(f"{CRAWL4AI_BASE_URL}/health")
+            return response.status_code == 200
+    except:
+        return False
+
+
+async def demo_1_system_health_overview():
+    """Demo 1: System Health Overview - Live metrics and pool status"""
+    print_section(
+        "Demo 1: System Health Overview",
+        "Real-time monitoring of system resources and browser pool"
+    )
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        print("🔍 Fetching system health metrics...")
+
+        try:
+            response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/health")
+            health = response.json()
+
+            print("\n✅ System Health Report:")
+            print(f"\n🖥️  Container Metrics:")
+            print(f"   • CPU Usage: {health['container']['cpu_percent']:.1f}%")
+            print(f"   • Memory Usage: {health['container']['memory_percent']:.1f}% "
+                  f"({health['container']['memory_mb']:.0f} MB)")
+            print(f"   • Network RX: {health['container']['network_rx_mb']:.2f} MB")
+            print(f"   • Network TX: {health['container']['network_tx_mb']:.2f} MB")
+            print(f"   • Uptime: {health['container']['uptime_seconds']:.0f}s")
+
+            print(f"\n🌐 Browser Pool Status:")
+            print(f"   Permanent Browser:")
+            print(f"   • Active: {health['pool']['permanent']['active']}")
+            print(f"   • Total Requests: {health['pool']['permanent']['total_requests']}")
+
+            print(f"   Hot Pool (Frequently Used Configs):")
+            print(f"   • Count: {health['pool']['hot']['count']}")
+            print(f"   • Total Requests: {health['pool']['hot']['total_requests']}")
+
+            print(f"   Cold Pool (On-Demand Configs):")
+            print(f"   • Count: {health['pool']['cold']['count']}")
+            print(f"   • Total Requests: {health['pool']['cold']['total_requests']}")
+
+            print(f"\n📈 Overall Statistics:")
+            print(f"   • Total Requests: {health['stats']['total_requests']}")
+            print(f"   • Success Rate: {health['stats']['success_rate_percent']:.1f}%")
+            print(f"   • Avg Latency: {health['stats']['avg_latency_ms']:.0f}ms")
+
+            print(f"\n💡 Dashboard URL: {MONITOR_DASHBOARD_URL}")
+
+        except Exception as e:
+            print(f"❌ Error fetching health: {e}")
+
+
+async def demo_2_request_tracking():
+    """Demo 2: Real-time Request Tracking - Generate and monitor requests"""
+    print_section(
+        "Demo 2: Real-time Request Tracking",
+        "Submit crawl jobs and watch them in real-time"
+    )
+
+    async with httpx.AsyncClient(timeout=60.0) as client:
+        print("🚀 Submitting crawl requests...")
+
+        # Submit multiple requests
+        urls_to_crawl = [
+            "https://httpbin.org/html",
+            "https://httpbin.org/json",
+            "https://example.com"
+        ]
+
+        tasks = []
+        for url in urls_to_crawl:
+            task = client.post(
+                f"{CRAWL4AI_BASE_URL}/crawl",
+                json={"urls": [url], "crawler_config": {}}
+            )
+            tasks.append(task)
+
+        print(f"   • Submitting {len(urls_to_crawl)} requests in parallel...")
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        successful = sum(1 for r in results if not isinstance(r, Exception) and r.status_code == 200)
+        print(f"   ✅ {successful}/{len(urls_to_crawl)} requests submitted")
+
+        # Check request tracking
+        print("\n📊 Checking request tracking...")
+        await asyncio.sleep(2)  # Wait for requests to process
+
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/requests")
+        requests_data = response.json()
+
+        print(f"\n📋 Request Status:")
+        print(f"   • Active Requests: {len(requests_data['active'])}")
+        print(f"   • Completed Requests: {len(requests_data['completed'])}")
+
+        if requests_data['completed']:
+            print(f"\n📝 Recent Completed Requests:")
+            for req in requests_data['completed'][:3]:
+                status_icon = "✅" if req['success'] else "❌"
+                print(f"   {status_icon} {req['endpoint']} - {req['latency_ms']:.0f}ms")
+
+
+async def demo_3_browser_pool_management():
+    """Demo 3: Browser Pool Management - 3-tier architecture in action"""
+    print_section(
+        "Demo 3: Browser Pool Management",
+        "Understanding permanent, hot, and cold browser pools"
+    )
+
+    async with httpx.AsyncClient(timeout=60.0) as client:
+        print("🌊 Testing browser pool with different configurations...")
+
+        # Test 1: Default config (permanent browser)
+        print("\n🔥 Test 1: Default Config → Permanent Browser")
+        for i in range(3):
+            await client.post(
+                f"{CRAWL4AI_BASE_URL}/crawl",
+                json={"urls": [f"https://httpbin.org/html?req={i}"], "crawler_config": {}}
+            )
+            print(f"   • Request {i+1}/3 sent (should use permanent browser)")
+
+        await asyncio.sleep(2)
+
+        # Test 2: Custom viewport (cold → hot promotion after 3 uses)
+        print("\n♨️  Test 2: Custom Viewport → Cold Pool (promoting to Hot)")
+        viewport_config = {"viewport": {"width": 1280, "height": 720}}
+        for i in range(4):
+            await client.post(
+                f"{CRAWL4AI_BASE_URL}/crawl",
+                json={
+                    "urls": [f"https://httpbin.org/json?viewport={i}"],
+                    "browser_config": viewport_config,
+                    "crawler_config": {}
+                }
+            )
+            print(f"   • Request {i+1}/4 sent (cold→hot promotion after 3rd use)")
+
+        await asyncio.sleep(2)
+
+        # Check browser pool status
+        print("\n📊 Browser Pool Report:")
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/browsers")
+        browsers = response.json()
+
+        print(f"\n🎯 Pool Summary:")
+        print(f"   • Total Browsers: {browsers['summary']['total_count']}")
+        print(f"   • Total Memory: {browsers['summary']['total_memory_mb']} MB")
+        print(f"   • Reuse Rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
+
+        print(f"\n📋 Browser Pool Details:")
+        if browsers['permanent']:
+            for browser in browsers['permanent']:
+                print(f"   🔥 Permanent: {browser['browser_id'][:8]}... | "
+                      f"Requests: {browser['request_count']} | "
+                      f"Memory: {browser['memory_mb']:.0f} MB")
+
+        if browsers['hot']:
+            for browser in browsers['hot']:
+                print(f"   ♨️  Hot: {browser['browser_id'][:8]}... | "
+                      f"Requests: {browser['request_count']} | "
+                      f"Memory: {browser['memory_mb']:.0f} MB")
+
+        if browsers['cold']:
+            for browser in browsers['cold']:
+                print(f"   ❄️  Cold: {browser['browser_id'][:8]}... | "
+                      f"Requests: {browser['request_count']} | "
+                      f"Memory: {browser['memory_mb']:.0f} MB")
+
+
+async def demo_4_monitor_api_endpoints():
+    """Demo 4: Monitor API Endpoints - Complete API surface"""
+    print_section(
+        "Demo 4: Monitor API Endpoints",
+        "Programmatic access to all monitoring data"
+    )
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        print("🔌 Testing Monitor API endpoints...")
+
+        # Endpoint performance statistics
+        print_subsection("Endpoint Performance Statistics")
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/endpoints/stats")
+        endpoint_stats = response.json()
+
+        print("\n📊 Per-Endpoint Analytics:")
+        for endpoint, stats in endpoint_stats.items():
+            print(f"   {endpoint}:")
+            print(f"      • Requests: {stats['count']}")
+            print(f"      • Avg Latency: {stats['avg_latency_ms']:.0f}ms")
+            print(f"      • Success Rate: {stats['success_rate_percent']:.1f}%")
+
+        # Timeline data for charts
+        print_subsection("Timeline Data (for Charts)")
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/timeline?minutes=5")
+        timeline = response.json()
+
+        print(f"\n📈 Timeline Metrics (last 5 minutes):")
+        print(f"   • Data Points: {len(timeline['memory'])}")
+        if timeline['memory']:
+            latest = timeline['memory'][-1]
+            print(f"   • Latest Memory: {latest['value']:.1f}%")
+            print(f"   • Timestamp: {latest['timestamp']}")
+
+        # Janitor logs
+        print_subsection("Janitor Cleanup Events")
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/logs/janitor?limit=3")
+        janitor_logs = response.json()
+
+        print(f"\n🧹 Recent Cleanup Activities:")
+        if janitor_logs:
+            for log in janitor_logs[:3]:
+                print(f"   • {log['timestamp']}: {log['message']}")
+        else:
+            print("   (No cleanup events yet - janitor runs periodically)")
+
+        # Error logs
+        print_subsection("Error Monitoring")
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/logs/errors?limit=3")
+        error_logs = response.json()
+
+        print(f"\n❌ Recent Errors:")
+        if error_logs:
+            for log in error_logs[:3]:
+                print(f"   • {log['timestamp']}: {log['error_type']}")
+                print(f"     {log['message'][:100]}...")
+        else:
+            print("   ✅ No recent errors!")
+
+
+async def demo_5_websocket_streaming():
+    """Demo 5: WebSocket Streaming - Real-time updates"""
+    print_section(
+        "Demo 5: WebSocket Streaming",
+        "Live monitoring with 2-second update intervals"
+    )
+
+    print("⚡ WebSocket Streaming Demo")
+    print("\n💡 The monitoring dashboard uses WebSocket for real-time updates")
+    print(f"   • Connection: ws://localhost:11235/monitor/ws")
+    print(f"   • Update Interval: 2 seconds")
+    print(f"   • Data: Health, requests, browsers, memory, errors")
+
+    print("\n📝 Sample WebSocket Integration Code:")
+    print("""
+    import websockets
+    import json
+
+    async def monitor_realtime():
+        uri = "ws://localhost:11235/monitor/ws"
+        async with websockets.connect(uri) as websocket:
+            while True:
+                data = await websocket.recv()
+                update = json.loads(data)
+
+                print(f"Memory: {update['health']['container']['memory_percent']:.1f}%")
+                print(f"Active Requests: {len(update['requests']['active'])}")
+                print(f"Browser Pool: {update['health']['pool']['permanent']['active']}")
+    """)
+
+    print("\n🌐 Open the dashboard to see WebSocket in action:")
+    print(f"   {MONITOR_DASHBOARD_URL}")
+
+
+async def demo_6_control_actions():
+    """Demo 6: Control Actions - Manual browser management"""
+    print_section(
+        "Demo 6: Control Actions",
+        "Manual control over browser pool and cleanup"
+    )
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        print("🎮 Testing control actions...")
+
+        # Force cleanup
+        print_subsection("Force Immediate Cleanup")
+        print("🧹 Triggering manual cleanup...")
+        try:
+            response = await client.post(f"{CRAWL4AI_BASE_URL}/monitor/actions/cleanup")
+            if response.status_code == 200:
+                result = response.json()
+                print(f"   ✅ Cleanup completed")
+                print(f"   • Browsers cleaned: {result.get('cleaned_count', 0)}")
+                print(f"   • Memory freed: {result.get('memory_freed_mb', 0):.1f} MB")
+            else:
+                print(f"   ⚠️  Response: {response.status_code}")
+        except Exception as e:
+            print(f"   ℹ️  Cleanup action: {e}")
+
+        # Get browser list for potential kill/restart
+        print_subsection("Browser Management")
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/browsers")
+        browsers = response.json()
+
+        cold_browsers = browsers.get('cold', [])
+        if cold_browsers:
+            browser_id = cold_browsers[0]['browser_id']
+            print(f"\n🎯 Example: Kill specific browser")
+            print(f"   POST /monitor/actions/kill_browser")
+            print(f"   JSON: {{'browser_id': '{browser_id[:16]}...'}}")
+            print(f"   → Kills the browser and frees resources")
+
+        print(f"\n🔄 Example: Restart browser")
+        print(f"   POST /monitor/actions/restart_browser")
+        print(f"   JSON: {{'browser_id': 'browser_id_here'}}")
+        print(f"   → Restart a specific browser instance")
+
+        # Reset statistics
+        print_subsection("Reset Statistics")
+        print("📊 Statistics can be reset for fresh monitoring:")
+        print(f"   POST /monitor/stats/reset")
+        print(f"   → Clears all accumulated statistics")
+
+
+async def demo_7_production_metrics():
+    """Demo 7: Production Metrics - Key indicators for operations"""
+    print_section(
+        "Demo 7: Production Metrics",
+        "Critical metrics for production monitoring"
+    )
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        print("📊 Key Production Metrics:")
+
+        # Overall health
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/health")
+        health = response.json()
+
+        # Browser efficiency
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/browsers")
+        browsers = response.json()
+
+        print("\n🎯 Critical Metrics to Track:")
+
+        print(f"\n1️⃣  Memory Usage Trends")
+        print(f"   • Current: {health['container']['memory_percent']:.1f}%")
+        print(f"   • Alert if: >80%")
+        print(f"   • Action: Trigger cleanup or scale")
+
+        print(f"\n2️⃣  Request Success Rate")
+        print(f"   • Current: {health['stats']['success_rate_percent']:.1f}%")
+        print(f"   • Target: >95%")
+        print(f"   • Alert if: <90%")
+
+        print(f"\n3️⃣  Average Latency")
+        print(f"   • Current: {health['stats']['avg_latency_ms']:.0f}ms")
+        print(f"   • Target: <2000ms")
+        print(f"   • Alert if: >5000ms")
+
+        print(f"\n4️⃣  Browser Pool Efficiency")
+        print(f"   • Reuse Rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
+        print(f"   • Target: >80%")
+        print(f"   • Indicates: Effective browser pooling")
+
+        print(f"\n5️⃣  Total Browsers")
+        print(f"   • Current: {browsers['summary']['total_count']}")
+        print(f"   • Alert if: >20 (possible leak)")
+        print(f"   • Check: Janitor is running correctly")
+
+        print(f"\n6️⃣  Error Frequency")
+        response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/logs/errors?limit=10")
+        errors = response.json()
+        print(f"   • Recent Errors: {len(errors)}")
+        print(f"   • Alert if: >10 in last hour")
+        print(f"   • Action: Review error patterns")
+
+        print("\n💡 Integration Examples:")
+        print("   • Prometheus: Scrape /monitor/health")
+        print("   • Alerting: Monitor memory, success rate, latency")
+        print("   • Dashboards: WebSocket streaming to custom UI")
+        print("   • Log Aggregation: Collect /monitor/logs/* endpoints")
+
+
+async def demo_8_self_hosting_value():
+    """Demo 8: Self-Hosting Value Proposition"""
+    print_section(
+        "Demo 8: Why Self-Host Crawl4AI?",
+        "The value proposition of owning your infrastructure"
+    )
+
+    print("🎯 Self-Hosting Benefits:\n")
+
+    print("🔒 Data Privacy & Security")
+    print("   • Your data never leaves your infrastructure")
+    print("   • No third-party access to crawled content")
+    print("   • Keep sensitive workflows behind your firewall")
+
+    print("\n💰 Cost Control")
+    print("   • No per-request pricing or rate limits")
+    print("   • Predictable infrastructure costs")
+    print("   • Scale based on your actual needs")
+
+    print("\n🎯 Full Customization")
+    print("   • Complete control over browser configs")
+    print("   • Custom hooks and strategies")
+    print("   • Tailored monitoring and alerting")
+
+    print("\n📊 Complete Transparency")
+    print("   • Real-time monitoring dashboard")
+    print("   • Full visibility into system performance")
+    print("   • Detailed request and error tracking")
+
+    print("\n⚡ Performance & Flexibility")
+    print("   • Direct access, no network overhead")
+    print("   • Integrate with existing infrastructure")
+    print("   • Custom resource allocation")
+
+    print("\n🛡️ Enterprise-Grade Operations")
+    print("   • Prometheus integration ready")
+    print("   • WebSocket for real-time dashboards")
+    print("   • Full API for automation")
+    print("   • Manual controls for troubleshooting")
+
+    print(f"\n🌐 Get Started:")
+    print(f"   docker pull unclecode/crawl4ai:0.7.7")
+    print(f"   docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.7")
+    print(f"   # Visit: {MONITOR_DASHBOARD_URL}")
+
+
+def print_summary():
+    """Print comprehensive demo summary"""
+    print("\n" + "=" * 70)
+    print("📊 DEMO SUMMARY - Crawl4AI v0.7.7")
+    print("=" * 70)
+
+    print("\n✨ Features Demonstrated:")
+    print("=" * 70)
+    print("✅ System Health Overview")
+    print("   → Real-time CPU, memory, network, and uptime monitoring")
+
+    print("\n✅ Request Tracking")
+    print("   → Active and completed request monitoring with full details")
+
+    print("\n✅ Browser Pool Management")
+    print("   → 3-tier architecture: Permanent, Hot, and Cold pools")
+    print("   → Automatic promotion and cleanup")
+
+    print("\n✅ Monitor API Endpoints")
+    print("   → Complete REST API for programmatic access")
+    print("   → Health, requests, browsers, timeline, logs, errors")
+
+    print("\n✅ WebSocket Streaming")
+    print("   → Real-time updates every 2 seconds")
+    print("   → Build custom dashboards with live data")
+
+    print("\n✅ Control Actions")
+    print("   → Manual browser management (kill, restart)")
+    print("   → Force cleanup and statistics reset")
+
+    print("\n✅ Production Metrics")
+    print("   → 6 critical metrics for operational excellence")
+    print("   → Prometheus integration patterns")
+
+    print("\n✅ Self-Hosting Value")
+    print("   → Data privacy, cost control, full customization")
+    print("   → Enterprise-grade transparency and control")
+
+    print("\n" + "=" * 70)
+    print("🎯 What's New in v0.7.7?")
+    print("=" * 70)
+    print("• 📊 Complete Real-time Monitoring System")
+    print("• 🌐 Interactive Web Dashboard (/dashboard)")
+    print("• 🔌 Comprehensive Monitor API")
+    print("• ⚡ WebSocket Streaming (2-second updates)")
+    print("• 🎮 Manual Control Actions")
+    print("• 📈 Production Integration Examples")
+    print("• 🏭 Prometheus, Alerting, Log Aggregation")
+    print("• 🔥 Smart Browser Pool (Permanent/Hot/Cold)")
+    print("• 🧹 Automatic Janitor Cleanup")
+    print("• 📋 Full Request & Error Tracking")
+
+    print("\n" + "=" * 70)
+    print("💡 Why This Matters")
+    print("=" * 70)
+    print("Before v0.7.7: Docker was just a containerized crawler")
+    print("After v0.7.7: Complete self-hosting platform with enterprise monitoring")
+    print("\nYou now have:")
+    print("  • Full visibility into what's happening inside")
+    print("  • Real-time operational dashboards")
+    print("  • Complete control over browser resources")
+    print("  • Production-ready observability")
+    print("  • Zero external dependencies")
+
+    print("\n" + "=" * 70)
+    print("📚 Next Steps")
+    print("=" * 70)
+    print(f"1. Open the dashboard: {MONITOR_DASHBOARD_URL}")
+    print("2. Read the docs: https://docs.crawl4ai.com/basic/self-hosting/")
+    print("3. Try the Monitor API endpoints yourself")
+    print("4. Set up Prometheus integration for production")
+    print("5. Build custom dashboards with WebSocket streaming")
+
+    print("\n" + "=" * 70)
+    print("🔗 Resources")
+    print("=" * 70)
+    print(f"• Dashboard: {MONITOR_DASHBOARD_URL}")
+    print(f"• Health API: {CRAWL4AI_BASE_URL}/monitor/health")
+    print(f"• Documentation: https://docs.crawl4ai.com/")
+    print(f"• GitHub: https://github.com/unclecode/crawl4ai")
+
+    print("\n" + "=" * 70)
+    print("🎉 You're now in control of your web crawling destiny!")
+    print("=" * 70)
+
+
+async def main():
+    """Run all demos"""
+    print("\n" + "=" * 70)
+    print("🚀 Crawl4AI v0.7.7 Release Demo")
+    print("=" * 70)
+    print("Feature: Self-Hosting with Real-time Monitoring Dashboard")
+    print("=" * 70)
+
+    # Check if server is running
+    print("\n🔍 Checking Crawl4AI server...")
+    server_running = await check_server_health()
+
+    if not server_running:
+        print(f"❌ Cannot connect to Crawl4AI at {CRAWL4AI_BASE_URL}")
+        print("\nPlease start the Docker container:")
+        print("  docker pull unclecode/crawl4ai:0.7.7")
+        print("  docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.7")
+        print("\nThen re-run this demo.")
+        return
+
+    print(f"✅ Crawl4AI server is running!")
+    print(f"📊 Dashboard available at: {MONITOR_DASHBOARD_URL}")
+
+    # Run all demos
+    demos = [
+        demo_1_system_health_overview,
+        demo_2_request_tracking,
+        demo_3_browser_pool_management,
+        demo_4_monitor_api_endpoints,
+        demo_5_websocket_streaming,
+        demo_6_control_actions,
+        demo_7_production_metrics,
+        demo_8_self_hosting_value,
+    ]
+
+    for i, demo_func in enumerate(demos, 1):
+        try:
+            await demo_func()
+
+            if i < len(demos):
+                await asyncio.sleep(2)  # Brief pause between demos
+
+        except KeyboardInterrupt:
+            print(f"\n\n⚠️  Demo interrupted by user")
+            return
+        except Exception as e:
+            print(f"\n❌ Demo {i} error: {e}")
+            print("Continuing to next demo...\n")
+            continue
+
+    # Print comprehensive summary
+    print_summary()
+
+    print("\n" + "=" * 70)
+    print("✅ Demo completed!")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\n\n👋 Demo stopped by user. Thanks for trying Crawl4AI v0.7.7!")
+    except Exception as e:
+        print(f"\n\n❌ Demo failed: {e}")
+        print("Make sure the Docker container is running:")
+        print("  docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.7")
diff --git a/docs/releases_review/demo_v0.7.8.py b/docs/releases_review/demo_v0.7.8.py
new file mode 100644
index 0000000..5fb7521
--- /dev/null
+++ b/docs/releases_review/demo_v0.7.8.py
@@ -0,0 +1,910 @@
+#!/usr/bin/env python3
+"""
+Crawl4AI v0.7.8 Release Demo - Verification Tests
+==================================================
+
+This demo ACTUALLY RUNS and VERIFIES the bug fixes in v0.7.8.
+Each test executes real code and validates the fix is working.
+
+Bug Fixes Verified:
+1. ProxyConfig JSON serialization (#1629)
+2. Configurable backoff parameters (#1269)
+3. LLM Strategy input_format support (#1178)
+4. Raw HTML URL variable (#1116)
+5. Relative URLs after redirects (#1268)
+6. pypdf migration (#1412)
+7. Pydantic v2 ConfigDict (#678)
+8. Docker ContentRelevanceFilter (#1642) - requires Docker
+9. Docker .cache permissions (#1638) - requires Docker
+10. AdaptiveCrawler query expansion (#1621) - requires LLM API key
+11. Import statement formatting (#1181)
+
+Usage:
+    python docs/releases_review/demo_v0.7.8.py
+
+For Docker tests:
+    docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.8
+    python docs/releases_review/demo_v0.7.8.py
+"""
+
+import asyncio
+import json
+import sys
+import warnings
+import os
+import tempfile
+from typing import Tuple, Optional
+from dataclasses import dataclass
+
+# Test results tracking
+@dataclass
+class TestResult:
+    name: str
+    issue: str
+    passed: bool
+    message: str
+    skipped: bool = False
+
+
+results: list[TestResult] = []
+
+
+def print_header(title: str):
+    print(f"\n{'=' * 70}")
+    print(f"{title}")
+    print(f"{'=' * 70}")
+
+
+def print_test(name: str, issue: str):
+    print(f"\n[TEST] {name} ({issue})")
+    print("-" * 50)
+
+
+def record_result(name: str, issue: str, passed: bool, message: str, skipped: bool = False):
+    results.append(TestResult(name, issue, passed, message, skipped))
+    if skipped:
+        print(f"  SKIPPED: {message}")
+    elif passed:
+        print(f"  PASSED: {message}")
+    else:
+        print(f"  FAILED: {message}")
+
+
+# =============================================================================
+# TEST 1: ProxyConfig JSON Serialization (#1629)
+# =============================================================================
+async def test_proxy_config_serialization():
+    """
+    Verify BrowserConfig.to_dict() properly serializes ProxyConfig to JSON.
+
+    BEFORE: ProxyConfig was included as object, causing JSON serialization to fail
+    AFTER: ProxyConfig.to_dict() is called, producing valid JSON
+    """
+    print_test("ProxyConfig JSON Serialization", "#1629")
+
+    try:
+        from crawl4ai import BrowserConfig
+        from crawl4ai.async_configs import ProxyConfig
+
+        # Create config with ProxyConfig
+        proxy = ProxyConfig(
+            server="http://proxy.example.com:8080",
+            username="testuser",
+            password="testpass"
+        )
+        browser_config = BrowserConfig(headless=True, proxy_config=proxy)
+
+        # Test 1: to_dict() should return dict for proxy_config
+        config_dict = browser_config.to_dict()
+        proxy_dict = config_dict.get('proxy_config')
+
+        if not isinstance(proxy_dict, dict):
+            record_result("ProxyConfig Serialization", "#1629", False,
+                         f"proxy_config is {type(proxy_dict)}, expected dict")
+            return
+
+        # Test 2: Should be JSON serializable
+        try:
+            json_str = json.dumps(config_dict)
+            json.loads(json_str)  # Verify valid JSON
+        except (TypeError, json.JSONDecodeError) as e:
+            record_result("ProxyConfig Serialization", "#1629", False,
+                         f"JSON serialization failed: {e}")
+            return
+
+        # Test 3: Verify proxy data is preserved
+        if proxy_dict.get('server') != "http://proxy.example.com:8080":
+            record_result("ProxyConfig Serialization", "#1629", False,
+                         "Proxy server not preserved in serialization")
+            return
+
+        record_result("ProxyConfig Serialization", "#1629", True,
+                     "BrowserConfig with ProxyConfig serializes to valid JSON")
+
+    except Exception as e:
+        record_result("ProxyConfig Serialization", "#1629", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 2: Configurable Backoff Parameters (#1269)
+# =============================================================================
+async def test_configurable_backoff():
+    """
+    Verify LLMConfig accepts and stores backoff configuration parameters.
+
+    BEFORE: Backoff was hardcoded (delay=2, attempts=3, factor=2)
+    AFTER: LLMConfig accepts backoff_base_delay, backoff_max_attempts, backoff_exponential_factor
+    """
+    print_test("Configurable Backoff Parameters", "#1269")
+
+    try:
+        from crawl4ai import LLMConfig
+
+        # Test 1: Default values
+        default_config = LLMConfig(provider="openai/gpt-4o-mini")
+
+        if default_config.backoff_base_delay != 2:
+            record_result("Configurable Backoff", "#1269", False,
+                         f"Default base_delay is {default_config.backoff_base_delay}, expected 2")
+            return
+
+        if default_config.backoff_max_attempts != 3:
+            record_result("Configurable Backoff", "#1269", False,
+                         f"Default max_attempts is {default_config.backoff_max_attempts}, expected 3")
+            return
+
+        if default_config.backoff_exponential_factor != 2:
+            record_result("Configurable Backoff", "#1269", False,
+                         f"Default exponential_factor is {default_config.backoff_exponential_factor}, expected 2")
+            return
+
+        # Test 2: Custom values
+        custom_config = LLMConfig(
+            provider="openai/gpt-4o-mini",
+            backoff_base_delay=5,
+            backoff_max_attempts=10,
+            backoff_exponential_factor=3
+        )
+
+        if custom_config.backoff_base_delay != 5:
+            record_result("Configurable Backoff", "#1269", False,
+                         f"Custom base_delay is {custom_config.backoff_base_delay}, expected 5")
+            return
+
+        if custom_config.backoff_max_attempts != 10:
+            record_result("Configurable Backoff", "#1269", False,
+                         f"Custom max_attempts is {custom_config.backoff_max_attempts}, expected 10")
+            return
+
+        if custom_config.backoff_exponential_factor != 3:
+            record_result("Configurable Backoff", "#1269", False,
+                         f"Custom exponential_factor is {custom_config.backoff_exponential_factor}, expected 3")
+            return
+
+        # Test 3: to_dict() includes backoff params
+        config_dict = custom_config.to_dict()
+        if 'backoff_base_delay' not in config_dict:
+            record_result("Configurable Backoff", "#1269", False,
+                         "backoff_base_delay missing from to_dict()")
+            return
+
+        record_result("Configurable Backoff", "#1269", True,
+                     "LLMConfig accepts and stores custom backoff parameters")
+
+    except Exception as e:
+        record_result("Configurable Backoff", "#1269", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 3: LLM Strategy Input Format (#1178)
+# =============================================================================
+async def test_llm_input_format():
+    """
+    Verify LLMExtractionStrategy accepts input_format parameter.
+
+    BEFORE: Always used markdown input
+    AFTER: Supports "markdown", "html", "fit_markdown", "cleaned_html", "fit_html"
+    """
+    print_test("LLM Strategy Input Format", "#1178")
+
+    try:
+        from crawl4ai import LLMExtractionStrategy, LLMConfig
+
+        llm_config = LLMConfig(provider="openai/gpt-4o-mini")
+
+        # Test 1: Default is markdown
+        default_strategy = LLMExtractionStrategy(
+            llm_config=llm_config,
+            instruction="Extract data"
+        )
+
+        if default_strategy.input_format != "markdown":
+            record_result("LLM Input Format", "#1178", False,
+                         f"Default input_format is '{default_strategy.input_format}', expected 'markdown'")
+            return
+
+        # Test 2: Can set to html
+        html_strategy = LLMExtractionStrategy(
+            llm_config=llm_config,
+            instruction="Extract data",
+            input_format="html"
+        )
+
+        if html_strategy.input_format != "html":
+            record_result("LLM Input Format", "#1178", False,
+                         f"HTML input_format is '{html_strategy.input_format}', expected 'html'")
+            return
+
+        # Test 3: Can set to fit_markdown
+        fit_strategy = LLMExtractionStrategy(
+            llm_config=llm_config,
+            instruction="Extract data",
+            input_format="fit_markdown"
+        )
+
+        if fit_strategy.input_format != "fit_markdown":
+            record_result("LLM Input Format", "#1178", False,
+                         f"fit_markdown input_format is '{fit_strategy.input_format}'")
+            return
+
+        record_result("LLM Input Format", "#1178", True,
+                     "LLMExtractionStrategy accepts all input_format options")
+
+    except Exception as e:
+        record_result("LLM Input Format", "#1178", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 4: Raw HTML URL Variable (#1116)
+# =============================================================================
+async def test_raw_html_url_variable():
+    """
+    Verify that raw: prefix URLs pass "Raw HTML" to extraction strategy.
+
+    BEFORE: Entire HTML blob was passed as URL parameter
+    AFTER: "Raw HTML" string is passed as URL parameter
+    """
+    print_test("Raw HTML URL Variable", "#1116")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+        from crawl4ai.extraction_strategy import ExtractionStrategy
+
+        # Custom strategy to capture what URL is passed
+        class URLCapturingStrategy(ExtractionStrategy):
+            captured_url = None
+
+            def extract(self, url: str, html: str, *args, **kwargs):
+                URLCapturingStrategy.captured_url = url
+                return [{"content": "test"}]
+
+        html_content = "
    Test
    "
+        strategy = URLCapturingStrategy()
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=f"raw:{html_content}",
+                config=CrawlerRunConfig(
+                    extraction_strategy=strategy
+                )
+            )
+
+        captured = URLCapturingStrategy.captured_url
+
+        if captured is None:
+            record_result("Raw HTML URL Variable", "#1116", False,
+                         "Extraction strategy was not called")
+            return
+
+        if captured == html_content or captured.startswith("
+        
+        
    
+import os
+import sys
+from pathlib import Path
+from typing import List, Dict
+
+def main():
+    pass
+        
    
+        
+        
+        """
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=f"raw:{html_with_code}",
+                config=CrawlerRunConfig()
+            )
+
+        markdown = result.markdown.raw_markdown if result.markdown else ""
+
+        # Check that imports are not concatenated on the same line
+        # Bad: "import osimport sys" (no newline between statements)
+        # This is the actual bug - statements getting merged on same line
+        bad_patterns = [
+            "import os import sys",      # Space but no newline
+            "import osimport sys",       # No space or newline
+            "import os from pathlib",    # Space but no newline
+            "import osfrom pathlib",     # No space or newline
+        ]
+
+        markdown_single_line = markdown.replace('\n', ' ')  # Convert newlines to spaces
+
+        for pattern in bad_patterns:
+            # Check if pattern exists without proper line separation
+            if pattern.replace(' ', '') in markdown_single_line.replace(' ', ''):
+                # Verify it's actually on same line (not just adjacent after newline removal)
+                lines = markdown.split('\n')
+                for line in lines:
+                    if 'import' in line.lower():
+                        # Count import statements on this line
+                        import_count = line.lower().count('import ')
+                        if import_count > 1:
+                            record_result("Import Formatting", "#1181", False,
+                                         f"Multiple imports on same line: {line[:60]}...")
+                            return
+
+        # Verify imports are present
+        if "import" in markdown.lower():
+            record_result("Import Formatting", "#1181", True,
+                         "Import statements are properly line-separated")
+        else:
+            record_result("Import Formatting", "#1181", True,
+                         "No import statements found to verify (test HTML may have changed)")
+
+    except Exception as e:
+        record_result("Import Formatting", "#1181", False, f"Exception: {e}")
+
+
+# =============================================================================
+# COMPREHENSIVE CRAWL TEST
+# =============================================================================
+async def test_comprehensive_crawl():
+    """
+    Run a comprehensive crawl to verify overall stability.
+    """
+    print_test("Comprehensive Crawl Test", "Overall")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+
+        async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+            result = await crawler.arun(
+                url="https://httpbin.org/html",
+                config=CrawlerRunConfig()
+            )
+
+        # Verify result
+        checks = []
+
+        if result.success:
+            checks.append("success=True")
+        else:
+            record_result("Comprehensive Crawl", "Overall", False,
+                         f"Crawl failed: {result.error_message}")
+            return
+
+        if result.html and len(result.html) > 100:
+            checks.append(f"html={len(result.html)} chars")
+
+        if result.markdown and result.markdown.raw_markdown:
+            checks.append(f"markdown={len(result.markdown.raw_markdown)} chars")
+
+        if result.redirected_url:
+            checks.append("redirected_url present")
+
+        record_result("Comprehensive Crawl", "Overall", True,
+                     f"All checks passed: {', '.join(checks)}")
+
+    except Exception as e:
+        record_result("Comprehensive Crawl", "Overall", False, f"Exception: {e}")
+
+
+# =============================================================================
+# MAIN
+# =============================================================================
+
+def print_summary():
+    """Print test results summary"""
+    print_header("TEST RESULTS SUMMARY")
+
+    passed = sum(1 for r in results if r.passed and not r.skipped)
+    failed = sum(1 for r in results if not r.passed and not r.skipped)
+    skipped = sum(1 for r in results if r.skipped)
+
+    print(f"\nTotal: {len(results)} tests")
+    print(f"  Passed:  {passed}")
+    print(f"  Failed:  {failed}")
+    print(f"  Skipped: {skipped}")
+
+    if failed > 0:
+        print("\nFailed Tests:")
+        for r in results:
+            if not r.passed and not r.skipped:
+                print(f"  - {r.name} ({r.issue}): {r.message}")
+
+    if skipped > 0:
+        print("\nSkipped Tests:")
+        for r in results:
+            if r.skipped:
+                print(f"  - {r.name} ({r.issue}): {r.message}")
+
+    print("\n" + "=" * 70)
+    if failed == 0:
+        print("All tests passed! v0.7.8 bug fixes verified.")
+    else:
+        print(f"WARNING: {failed} test(s) failed!")
+    print("=" * 70)
+
+    return failed == 0
+
+
+async def main():
+    """Run all verification tests"""
+    print_header("Crawl4AI v0.7.8 - Bug Fix Verification Tests")
+    print("Running actual tests to verify bug fixes...")
+
+    # Run all tests
+    tests = [
+        test_proxy_config_serialization,       # #1629
+        test_configurable_backoff,             # #1269
+        test_llm_input_format,                 # #1178
+        test_raw_html_url_variable,            # #1116
+        test_redirect_url_handling,            # #1268
+        test_pypdf_migration,                  # #1412
+        test_pydantic_configdict,              # #678
+        test_docker_content_filter,            # #1642
+        test_docker_cache_permissions,         # #1638
+        test_adaptive_crawler_embedding,       # #1621
+        test_import_formatting,                # #1181
+        test_comprehensive_crawl,              # Overall
+    ]
+
+    for test_func in tests:
+        try:
+            await test_func()
+        except Exception as e:
+            print(f"\nTest {test_func.__name__} crashed: {e}")
+            results.append(TestResult(
+                test_func.__name__,
+                "Unknown",
+                False,
+                f"Crashed: {e}"
+            ))
+
+    # Print summary
+    all_passed = print_summary()
+
+    return 0 if all_passed else 1
+
+
+if __name__ == "__main__":
+    try:
+        exit_code = asyncio.run(main())
+        sys.exit(exit_code)
+    except KeyboardInterrupt:
+        print("\n\nTests interrupted by user.")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n\nTest suite failed: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
diff --git a/docs/releases_review/demo_v0.8.0.py b/docs/releases_review/demo_v0.8.0.py
new file mode 100644
index 0000000..bce8918
--- /dev/null
+++ b/docs/releases_review/demo_v0.8.0.py
@@ -0,0 +1,633 @@
+#!/usr/bin/env python3
+"""
+Crawl4AI v0.8.0 Release Demo - Feature Verification Tests
+==========================================================
+
+This demo ACTUALLY RUNS and VERIFIES the new features in v0.8.0.
+Each test executes real code and validates the feature is working.
+
+New Features Verified:
+1. Crash Recovery - on_state_change callback for real-time state persistence
+2. Crash Recovery - resume_state for resuming from checkpoint
+3. Crash Recovery - State is JSON serializable
+4. Prefetch Mode - Returns HTML and links only
+5. Prefetch Mode - Skips heavy processing (markdown, extraction)
+6. Prefetch Mode - Two-phase crawl pattern
+7. Security - Hooks disabled by default (Docker API)
+
+Breaking Changes in v0.8.0:
+- Docker API hooks disabled by default (CRAWL4AI_HOOKS_ENABLED=false)
+- file:// URLs blocked on Docker API endpoints
+
+Usage:
+    python docs/releases_review/demo_v0.8.0.py
+"""
+
+import asyncio
+import json
+import sys
+import time
+from typing import Dict, Any, List, Optional
+from dataclasses import dataclass
+
+
+# Test results tracking
+@dataclass
+class TestResult:
+    name: str
+    feature: str
+    passed: bool
+    message: str
+    skipped: bool = False
+
+
+results: list[TestResult] = []
+
+
+def print_header(title: str):
+    print(f"\n{'=' * 70}")
+    print(f"{title}")
+    print(f"{'=' * 70}")
+
+
+def print_test(name: str, feature: str):
+    print(f"\n[TEST] {name} ({feature})")
+    print("-" * 50)
+
+
+def record_result(name: str, feature: str, passed: bool, message: str, skipped: bool = False):
+    results.append(TestResult(name, feature, passed, message, skipped))
+    if skipped:
+        print(f"  SKIPPED: {message}")
+    elif passed:
+        print(f"  PASSED: {message}")
+    else:
+        print(f"  FAILED: {message}")
+
+
+# =============================================================================
+# TEST 1: Crash Recovery - State Capture with on_state_change
+# =============================================================================
+async def test_crash_recovery_state_capture():
+    """
+    Verify on_state_change callback is called after each URL is processed.
+
+    NEW in v0.8.0: Deep crawl strategies support on_state_change callback
+    for real-time state persistence (useful for cloud deployments).
+    """
+    print_test("Crash Recovery - State Capture", "on_state_change")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+        from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+        captured_states: List[Dict[str, Any]] = []
+
+        async def capture_state(state: Dict[str, Any]):
+            """Callback that fires after each URL is processed."""
+            captured_states.append(state.copy())
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=3,
+            on_state_change=capture_state,
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=strategy,
+            verbose=False,
+        )
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            await crawler.arun("https://books.toscrape.com", config=config)
+
+        # Verify states were captured
+        if len(captured_states) == 0:
+            record_result("State Capture", "on_state_change", False,
+                         "No states captured - callback not called")
+            return
+
+        # Verify callback was called for each page
+        pages_crawled = captured_states[-1].get("pages_crawled", 0)
+        if pages_crawled != len(captured_states):
+            record_result("State Capture", "on_state_change", False,
+                         f"Callback count {len(captured_states)} != pages_crawled {pages_crawled}")
+            return
+
+        record_result("State Capture", "on_state_change", True,
+                     f"Callback fired {len(captured_states)} times (once per URL)")
+
+    except Exception as e:
+        record_result("State Capture", "on_state_change", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 2: Crash Recovery - Resume from Checkpoint
+# =============================================================================
+async def test_crash_recovery_resume():
+    """
+    Verify crawl can resume from a saved checkpoint without re-crawling visited URLs.
+
+    NEW in v0.8.0: BFSDeepCrawlStrategy accepts resume_state parameter
+    to continue from a previously saved checkpoint.
+    """
+    print_test("Crash Recovery - Resume from Checkpoint", "resume_state")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+        from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+        # Phase 1: Start crawl and capture state after 2 pages
+        crash_after = 2
+        captured_states: List[Dict] = []
+        phase1_urls: List[str] = []
+
+        async def capture_until_crash(state: Dict[str, Any]):
+            captured_states.append(state.copy())
+            phase1_urls.clear()
+            phase1_urls.extend(state["visited"])
+            if state["pages_crawled"] >= crash_after:
+                raise Exception("Simulated crash")
+
+        strategy1 = BFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=5,
+            on_state_change=capture_until_crash,
+        )
+
+        config1 = CrawlerRunConfig(
+            deep_crawl_strategy=strategy1,
+            verbose=False,
+        )
+
+        # Run until "crash"
+        try:
+            async with AsyncWebCrawler(verbose=False) as crawler:
+                await crawler.arun("https://books.toscrape.com", config=config1)
+        except Exception:
+            pass  # Expected crash
+
+        if not captured_states:
+            record_result("Resume from Checkpoint", "resume_state", False,
+                         "No state captured before crash")
+            return
+
+        saved_state = captured_states[-1]
+        print(f"  Phase 1: Crawled {len(phase1_urls)} URLs before crash")
+
+        # Phase 2: Resume from checkpoint
+        phase2_urls: List[str] = []
+
+        async def track_phase2(state: Dict[str, Any]):
+            new_urls = set(state["visited"]) - set(saved_state["visited"])
+            for url in new_urls:
+                if url not in phase2_urls:
+                    phase2_urls.append(url)
+
+        strategy2 = BFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=5,
+            resume_state=saved_state,  # Resume from checkpoint!
+            on_state_change=track_phase2,
+        )
+
+        config2 = CrawlerRunConfig(
+            deep_crawl_strategy=strategy2,
+            verbose=False,
+        )
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            await crawler.arun("https://books.toscrape.com", config=config2)
+
+        print(f"  Phase 2: Crawled {len(phase2_urls)} new URLs after resume")
+
+        # Verify no duplicates
+        duplicates = set(phase2_urls) & set(phase1_urls)
+        if duplicates:
+            record_result("Resume from Checkpoint", "resume_state", False,
+                         f"Re-crawled {len(duplicates)} URLs: {list(duplicates)[:2]}")
+            return
+
+        record_result("Resume from Checkpoint", "resume_state", True,
+                     f"Resumed successfully, no duplicate crawls")
+
+    except Exception as e:
+        record_result("Resume from Checkpoint", "resume_state", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 3: Crash Recovery - State is JSON Serializable
+# =============================================================================
+async def test_crash_recovery_json_serializable():
+    """
+    Verify the state dictionary can be serialized to JSON (for Redis/DB storage).
+
+    NEW in v0.8.0: State dictionary is designed to be JSON-serializable
+    for easy storage in Redis, databases, or files.
+    """
+    print_test("Crash Recovery - JSON Serializable", "State Structure")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+        from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+        captured_state: Optional[Dict] = None
+
+        async def capture_state(state: Dict[str, Any]):
+            nonlocal captured_state
+            captured_state = state
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=2,
+            on_state_change=capture_state,
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=strategy,
+            verbose=False,
+        )
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            await crawler.arun("https://books.toscrape.com", config=config)
+
+        if not captured_state:
+            record_result("JSON Serializable", "State Structure", False,
+                         "No state captured")
+            return
+
+        # Test JSON serialization round-trip
+        try:
+            json_str = json.dumps(captured_state)
+            restored = json.loads(json_str)
+        except (TypeError, json.JSONDecodeError) as e:
+            record_result("JSON Serializable", "State Structure", False,
+                         f"JSON serialization failed: {e}")
+            return
+
+        # Verify state structure
+        required_fields = ["strategy_type", "visited", "pending", "depths", "pages_crawled"]
+        missing = [f for f in required_fields if f not in restored]
+        if missing:
+            record_result("JSON Serializable", "State Structure", False,
+                         f"Missing fields: {missing}")
+            return
+
+        # Verify types
+        if not isinstance(restored["visited"], list):
+            record_result("JSON Serializable", "State Structure", False,
+                         "visited is not a list")
+            return
+
+        if not isinstance(restored["pages_crawled"], int):
+            record_result("JSON Serializable", "State Structure", False,
+                         "pages_crawled is not an int")
+            return
+
+        record_result("JSON Serializable", "State Structure", True,
+                     f"State serializes to {len(json_str)} bytes, all fields present")
+
+    except Exception as e:
+        record_result("JSON Serializable", "State Structure", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 4: Prefetch Mode - Returns HTML and Links Only
+# =============================================================================
+async def test_prefetch_returns_html_links():
+    """
+    Verify prefetch mode returns HTML and links but skips markdown generation.
+
+    NEW in v0.8.0: CrawlerRunConfig accepts prefetch=True for fast
+    URL discovery without heavy processing.
+    """
+    print_test("Prefetch Mode - HTML and Links", "prefetch=True")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+        config = CrawlerRunConfig(prefetch=True)
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun("https://books.toscrape.com", config=config)
+
+        # Verify HTML is present
+        if not result.html or len(result.html) < 100:
+            record_result("Prefetch HTML/Links", "prefetch=True", False,
+                         "HTML not returned or too short")
+            return
+
+        # Verify links are present
+        if not result.links:
+            record_result("Prefetch HTML/Links", "prefetch=True", False,
+                         "Links not returned")
+            return
+
+        internal_count = len(result.links.get("internal", []))
+        external_count = len(result.links.get("external", []))
+
+        if internal_count == 0:
+            record_result("Prefetch HTML/Links", "prefetch=True", False,
+                         "No internal links extracted")
+            return
+
+        record_result("Prefetch HTML/Links", "prefetch=True", True,
+                     f"HTML: {len(result.html)} chars, Links: {internal_count} internal, {external_count} external")
+
+    except Exception as e:
+        record_result("Prefetch HTML/Links", "prefetch=True", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 5: Prefetch Mode - Skips Heavy Processing
+# =============================================================================
+async def test_prefetch_skips_processing():
+    """
+    Verify prefetch mode skips markdown generation and content extraction.
+
+    NEW in v0.8.0: prefetch=True skips markdown generation, content scraping,
+    media extraction, and LLM extraction for maximum speed.
+    """
+    print_test("Prefetch Mode - Skips Processing", "prefetch=True")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+        config = CrawlerRunConfig(prefetch=True)
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun("https://books.toscrape.com", config=config)
+
+        # Check that heavy processing was skipped
+        checks = []
+
+        # Markdown should be None or empty
+        if result.markdown is None:
+            checks.append("markdown=None")
+        elif hasattr(result.markdown, 'raw_markdown') and result.markdown.raw_markdown is None:
+            checks.append("raw_markdown=None")
+        else:
+            record_result("Prefetch Skips Processing", "prefetch=True", False,
+                         f"Markdown was generated (should be skipped)")
+            return
+
+        # cleaned_html should be None
+        if result.cleaned_html is None:
+            checks.append("cleaned_html=None")
+        else:
+            record_result("Prefetch Skips Processing", "prefetch=True", False,
+                         "cleaned_html was generated (should be skipped)")
+            return
+
+        # extracted_content should be None
+        if result.extracted_content is None:
+            checks.append("extracted_content=None")
+
+        record_result("Prefetch Skips Processing", "prefetch=True", True,
+                     f"Heavy processing skipped: {', '.join(checks)}")
+
+    except Exception as e:
+        record_result("Prefetch Skips Processing", "prefetch=True", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 6: Prefetch Mode - Two-Phase Crawl Pattern
+# =============================================================================
+async def test_prefetch_two_phase():
+    """
+    Verify the two-phase crawl pattern: prefetch for discovery, then full processing.
+
+    NEW in v0.8.0: Prefetch mode enables efficient two-phase crawling where
+    you discover URLs quickly, then selectively process important ones.
+    """
+    print_test("Prefetch Mode - Two-Phase Crawl", "Two-Phase Pattern")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            # Phase 1: Fast discovery with prefetch
+            prefetch_config = CrawlerRunConfig(prefetch=True)
+
+            start = time.time()
+            discovery = await crawler.arun("https://books.toscrape.com", config=prefetch_config)
+            prefetch_time = time.time() - start
+
+            all_urls = [link["href"] for link in discovery.links.get("internal", [])]
+
+            # Filter to specific pages (e.g., book detail pages)
+            book_urls = [
+                url for url in all_urls
+                if "catalogue/" in url and "category/" not in url
+            ][:2]  # Just 2 for demo
+
+            print(f"  Phase 1: Found {len(all_urls)} URLs in {prefetch_time:.2f}s")
+            print(f"  Filtered to {len(book_urls)} book pages for full processing")
+
+            if len(book_urls) == 0:
+                record_result("Two-Phase Crawl", "Two-Phase Pattern", False,
+                             "No book URLs found to process")
+                return
+
+            # Phase 2: Full processing on selected URLs
+            full_config = CrawlerRunConfig()  # Normal mode
+
+            start = time.time()
+            processed = []
+            for url in book_urls:
+                result = await crawler.arun(url, config=full_config)
+                if result.success and result.markdown:
+                    processed.append(result)
+
+            full_time = time.time() - start
+
+            print(f"  Phase 2: Processed {len(processed)} pages in {full_time:.2f}s")
+
+            if len(processed) == 0:
+                record_result("Two-Phase Crawl", "Two-Phase Pattern", False,
+                             "No pages successfully processed in phase 2")
+                return
+
+            # Verify full processing includes markdown
+            if not processed[0].markdown or not processed[0].markdown.raw_markdown:
+                record_result("Two-Phase Crawl", "Two-Phase Pattern", False,
+                             "Full processing did not generate markdown")
+                return
+
+            record_result("Two-Phase Crawl", "Two-Phase Pattern", True,
+                         f"Discovered {len(all_urls)} URLs (prefetch), processed {len(processed)} (full)")
+
+    except Exception as e:
+        record_result("Two-Phase Crawl", "Two-Phase Pattern", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 7: Security - Hooks Disabled by Default
+# =============================================================================
+async def test_security_hooks_disabled():
+    """
+    Verify hooks are disabled by default in Docker API for security.
+
+    NEW in v0.8.0: Docker API hooks are disabled by default to prevent
+    Remote Code Execution. Set CRAWL4AI_HOOKS_ENABLED=true to enable.
+    """
+    print_test("Security - Hooks Disabled", "CRAWL4AI_HOOKS_ENABLED")
+
+    try:
+        import os
+
+        # Check the default environment variable
+        hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower()
+
+        if hooks_enabled == "true":
+            record_result("Hooks Disabled Default", "Security", True,
+                         "CRAWL4AI_HOOKS_ENABLED is explicitly set to 'true' (user override)",
+                         skipped=True)
+            return
+
+        # Verify default is "false"
+        if hooks_enabled == "false":
+            record_result("Hooks Disabled Default", "Security", True,
+                         "Hooks disabled by default (CRAWL4AI_HOOKS_ENABLED=false)")
+        else:
+            record_result("Hooks Disabled Default", "Security", True,
+                         f"CRAWL4AI_HOOKS_ENABLED='{hooks_enabled}' (not 'true', hooks disabled)")
+
+    except Exception as e:
+        record_result("Hooks Disabled Default", "Security", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 8: Comprehensive Crawl Test
+# =============================================================================
+async def test_comprehensive_crawl():
+    """
+    Run a comprehensive crawl to verify overall stability with new features.
+    """
+    print_test("Comprehensive Crawl Test", "Overall")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+
+        async with AsyncWebCrawler(config=BrowserConfig(headless=True), verbose=False) as crawler:
+            result = await crawler.arun(
+                url="https://httpbin.org/html",
+                config=CrawlerRunConfig()
+            )
+
+        checks = []
+
+        if result.success:
+            checks.append("success=True")
+        else:
+            record_result("Comprehensive Crawl", "Overall", False,
+                         f"Crawl failed: {result.error_message}")
+            return
+
+        if result.html and len(result.html) > 100:
+            checks.append(f"html={len(result.html)} chars")
+
+        if result.markdown and result.markdown.raw_markdown:
+            checks.append(f"markdown={len(result.markdown.raw_markdown)} chars")
+
+        if result.links:
+            total_links = len(result.links.get("internal", [])) + len(result.links.get("external", []))
+            checks.append(f"links={total_links}")
+
+        record_result("Comprehensive Crawl", "Overall", True,
+                     f"All checks passed: {', '.join(checks)}")
+
+    except Exception as e:
+        record_result("Comprehensive Crawl", "Overall", False, f"Exception: {e}")
+
+
+# =============================================================================
+# MAIN
+# =============================================================================
+
+def print_summary():
+    """Print test results summary"""
+    print_header("TEST RESULTS SUMMARY")
+
+    passed = sum(1 for r in results if r.passed and not r.skipped)
+    failed = sum(1 for r in results if not r.passed and not r.skipped)
+    skipped = sum(1 for r in results if r.skipped)
+
+    print(f"\nTotal: {len(results)} tests")
+    print(f"  Passed:  {passed}")
+    print(f"  Failed:  {failed}")
+    print(f"  Skipped: {skipped}")
+
+    if failed > 0:
+        print("\nFailed Tests:")
+        for r in results:
+            if not r.passed and not r.skipped:
+                print(f"  - {r.name} ({r.feature}): {r.message}")
+
+    if skipped > 0:
+        print("\nSkipped Tests:")
+        for r in results:
+            if r.skipped:
+                print(f"  - {r.name} ({r.feature}): {r.message}")
+
+    print("\n" + "=" * 70)
+    if failed == 0:
+        print("All tests passed! v0.8.0 features verified.")
+    else:
+        print(f"WARNING: {failed} test(s) failed!")
+    print("=" * 70)
+
+    return failed == 0
+
+
+async def main():
+    """Run all verification tests"""
+    print_header("Crawl4AI v0.8.0 - Feature Verification Tests")
+    print("Running actual tests to verify new features...")
+    print("\nKey Features in v0.8.0:")
+    print("  - Crash Recovery for Deep Crawl (resume_state, on_state_change)")
+    print("  - Prefetch Mode for Fast URL Discovery (prefetch=True)")
+    print("  - Security: Hooks disabled by default on Docker API")
+
+    # Run all tests
+    tests = [
+        test_crash_recovery_state_capture,      # on_state_change
+        test_crash_recovery_resume,             # resume_state
+        test_crash_recovery_json_serializable,  # State structure
+        test_prefetch_returns_html_links,       # prefetch=True basics
+        test_prefetch_skips_processing,         # prefetch skips heavy work
+        test_prefetch_two_phase,                # Two-phase pattern
+        test_security_hooks_disabled,           # Security check
+        test_comprehensive_crawl,               # Overall stability
+    ]
+
+    for test_func in tests:
+        try:
+            await test_func()
+        except Exception as e:
+            print(f"\nTest {test_func.__name__} crashed: {e}")
+            results.append(TestResult(
+                test_func.__name__,
+                "Unknown",
+                False,
+                f"Crashed: {e}"
+            ))
+
+    # Print summary
+    all_passed = print_summary()
+
+    return 0 if all_passed else 1
+
+
+if __name__ == "__main__":
+    try:
+        exit_code = asyncio.run(main())
+        sys.exit(exit_code)
+    except KeyboardInterrupt:
+        print("\n\nTests interrupted by user.")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n\nTest suite failed: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
diff --git a/docs/releases_review/demo_v0.8.5.py b/docs/releases_review/demo_v0.8.5.py
new file mode 100644
index 0000000..b7697b5
--- /dev/null
+++ b/docs/releases_review/demo_v0.8.5.py
@@ -0,0 +1,913 @@
+#!/usr/bin/env python3
+"""
+Crawl4AI v0.8.5 Release Demo - Feature Verification Tests
+==========================================================
+
+This demo ACTUALLY RUNS and VERIFIES the new features in v0.8.5.
+Each test executes real crawls and validates the feature is working.
+
+New Features Verified:
+1. Anti-bot detection - Detects blocked pages and passes normal ones
+2. Anti-bot + crawl_stats - Real crawl produces crawl_stats tracking
+3. Proxy escalation chain - proxy_config accepts a list with DIRECT
+4. Config defaults API - set_defaults affects real crawls
+5. Shadow DOM flattening - Crawl a shadow-DOM site with/without flattening
+6. Deep crawl cancellation - DFS crawl stops at callback limit
+7. Consent popup removal - Crawl with remove_consent_popups enabled
+8. Source/sibling selector - Extract from sibling elements via "source" field
+9. GFM table compliance - Crawl a page with tables, verify pipe delimiters
+10. avoid_ads / avoid_css - Crawl with resource filtering enabled
+11. Browser recycling - Crawl multiple pages with memory_saving_mode
+12. BM25 content filter dedup - fit_markdown has no duplicate chunks
+13. cleaned_html preserves class/id - Verify attributes retained after crawl
+
+Usage:
+    python docs/releases_review/demo_v0.8.5.py
+"""
+
+import asyncio
+import json
+import sys
+import time
+from typing import Dict, Any, List, Optional
+from dataclasses import dataclass
+
+
+# Test results tracking
+@dataclass
+class TestResult:
+    name: str
+    feature: str
+    passed: bool
+    message: str
+    skipped: bool = False
+
+
+results: list[TestResult] = []
+
+
+def print_header(title: str):
+    print(f"\n{'=' * 70}")
+    print(f"{title}")
+    print(f"{'=' * 70}")
+
+
+def print_test(name: str, feature: str):
+    print(f"\n[TEST] {name} ({feature})")
+    print("-" * 50)
+
+
+def record_result(name: str, feature: str, passed: bool, message: str, skipped: bool = False):
+    results.append(TestResult(name, feature, passed, message, skipped))
+    if skipped:
+        print(f"  SKIPPED: {message}")
+    elif passed:
+        print(f"  PASSED: {message}")
+    else:
+        print(f"  FAILED: {message}")
+
+
+# =============================================================================
+# TEST 1: Anti-bot Detection - Unit + Live Crawl
+# =============================================================================
+async def test_antibot_detection():
+    """
+    Verify is_blocked() detects blocked pages and a real crawl to a normal
+    site succeeds without false positives.
+
+    NEW in v0.8.5: 3-tier anti-bot detection (status codes, content patterns,
+    structural integrity) with automatic retry and fallback.
+    """
+    print_test("Anti-bot Detection", "is_blocked() + live crawl")
+
+    try:
+        from crawl4ai.antibot_detector import is_blocked
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+        # Unit: blocked page detected
+        blocked, reason = is_blocked(
+            403,
+            '
    Please verify you are human
    '
+            '
    Checking your browser...
    ',
+        )
+        if not blocked:
+            record_result("Anti-bot Detection", "is_blocked()", False,
+                         "Failed to detect challenge page")
+            return
+
+        # Unit: JSON response not flagged
+        blocked, _ = is_blocked(
+            200,
+            '
    {"status":"ok"}
    ',
+        )
+        if blocked:
+            record_result("Anti-bot Detection", "is_blocked()", False,
+                         "False positive on JSON response")
+            return
+
+        # Live: crawl a normal site, verify no false positive
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun(
+                "https://quotes.toscrape.com",
+                config=CrawlerRunConfig(),
+            )
+
+        if not result.success:
+            record_result("Anti-bot Detection", "live crawl", False,
+                         f"Normal site crawl failed: {result.error_message}")
+            return
+
+        record_result("Anti-bot Detection", "is_blocked() + live crawl", True,
+                     f"Detects blocks, no false positive on live crawl "
+                     f"({len(result.html)} chars)")
+
+    except Exception as e:
+        record_result("Anti-bot Detection", "is_blocked()", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 2: Anti-bot crawl_stats Tracking
+# =============================================================================
+async def test_crawl_stats():
+    """
+    Verify a real crawl produces crawl_stats with proxy/fallback tracking.
+
+    NEW in v0.8.5: CrawlResult includes crawl_stats dict tracking which
+    proxies were used, whether fallback was invoked, and how it resolved.
+    """
+    print_test("Crawl Stats Tracking", "crawl_stats on CrawlResult")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun(
+                "https://example.com",
+                config=CrawlerRunConfig(),
+            )
+
+        if not result.success:
+            record_result("Crawl Stats", "crawl_stats", False,
+                         f"Crawl failed: {result.error_message}")
+            return
+
+        stats = getattr(result, "crawl_stats", None)
+        if stats is None:
+            record_result("Crawl Stats", "crawl_stats", False,
+                         "crawl_stats not present on CrawlResult")
+            return
+
+        # Check expected fields
+        has_proxies = "proxies_used" in stats
+        has_resolved = "resolved_by" in stats
+
+        if not has_proxies or not has_resolved:
+            record_result("Crawl Stats", "crawl_stats", False,
+                         f"Missing fields. Keys: {list(stats.keys())}")
+            return
+
+        record_result("Crawl Stats", "crawl_stats", True,
+                     f"Stats present: resolved_by={stats.get('resolved_by')}, "
+                     f"proxies_used={len(stats.get('proxies_used', []))} entries")
+
+    except Exception as e:
+        record_result("Crawl Stats", "crawl_stats", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 3: Proxy Escalation Chain + DIRECT Sentinel
+# =============================================================================
+async def test_proxy_escalation():
+    """
+    Verify proxy_config accepts a list and DIRECT sentinel, then crawl
+    with DIRECT-only to prove the escalation path works.
+
+    NEW in v0.8.5: proxy_config can be a list of ProxyConfig/None for
+    escalation. ProxyConfig.DIRECT normalizes to None (no proxy).
+    """
+    print_test("Proxy Escalation Chain", "list proxy_config + DIRECT crawl")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+        from crawl4ai.async_configs import ProxyConfig
+
+        # Verify DIRECT normalizes correctly in a list
+        config = CrawlerRunConfig(
+            proxy_config=[ProxyConfig.DIRECT],
+        )
+        if not isinstance(config.proxy_config, list):
+            record_result("Proxy Escalation", "list config", False,
+                         f"proxy_config is {type(config.proxy_config)}, expected list")
+            return
+
+        # Live crawl with DIRECT (no proxy)
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun(
+                "https://example.com",
+                config=config,
+            )
+
+        if not result.success:
+            record_result("Proxy Escalation", "DIRECT crawl", False,
+                         f"DIRECT crawl failed: {result.error_message}")
+            return
+
+        record_result("Proxy Escalation", "list + DIRECT crawl", True,
+                     f"List config accepted, DIRECT crawl succeeded "
+                     f"({len(result.html)} chars)")
+
+    except Exception as e:
+        record_result("Proxy Escalation", "proxy_config list", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 4: Config Defaults API — Real Crawl
+# =============================================================================
+async def test_config_defaults():
+    """
+    Set text_mode=True as a default, then crawl and verify it took effect.
+
+    NEW in v0.8.5: BrowserConfig.set_defaults() / get_defaults() /
+    reset_defaults() persist across all new instances.
+    """
+    print_test("Config Defaults API", "set_defaults → real crawl")
+
+    try:
+        from crawl4ai import AsyncWebCrawler
+        from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+        original = BrowserConfig.get_defaults()
+
+        try:
+            # Set text_mode as default (disables image loading)
+            BrowserConfig.set_defaults(text_mode=True, headless=True)
+
+            # Verify it applies
+            bc = BrowserConfig()
+            if not bc.text_mode:
+                record_result("Config Defaults", "set_defaults", False,
+                             "text_mode default not applied")
+                return
+
+            # Verify explicit override wins
+            bc2 = BrowserConfig(text_mode=False)
+            if bc2.text_mode:
+                record_result("Config Defaults", "set_defaults", False,
+                             "Explicit override didn't work")
+                return
+
+            # Real crawl with default text_mode
+            async with AsyncWebCrawler(config=BrowserConfig(), verbose=False) as crawler:
+                result = await crawler.arun(
+                    "https://example.com",
+                    config=CrawlerRunConfig(),
+                )
+
+            if not result.success:
+                record_result("Config Defaults", "crawl with defaults", False,
+                             f"Crawl failed: {result.error_message}")
+                return
+
+            # Verify reset works
+            BrowserConfig.reset_defaults()
+            if BrowserConfig.get_defaults():
+                record_result("Config Defaults", "reset_defaults", False,
+                             "Defaults not cleared after reset")
+                return
+
+            record_result("Config Defaults", "set/get/reset + crawl", True,
+                         f"Defaults applied to crawl, override works, reset clears "
+                         f"({len(result.markdown.raw_markdown)} chars markdown)")
+
+        finally:
+            BrowserConfig.reset_defaults()
+            if original:
+                BrowserConfig.set_defaults(**original)
+
+    except Exception as e:
+        record_result("Config Defaults", "set/get/reset_defaults", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 5: Shadow DOM Flattening — Comparison Crawl
+# =============================================================================
+async def test_shadow_dom_flattening():
+    """
+    Crawl a page with and without flatten_shadow_dom and compare content.
+
+    NEW in v0.8.5: CrawlerRunConfig.flatten_shadow_dom serializes shadow DOM
+    into the light DOM, exposing hidden content to extraction.
+    """
+    print_test("Shadow DOM Flattening", "comparison crawl")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+        # Use a page known to use web components / shadow DOM
+        # (GitHub uses shadow DOM for some components)
+        url = "https://books.toscrape.com"
+
+        async with AsyncWebCrawler(
+            config=BrowserConfig(headless=True),
+            verbose=False,
+        ) as crawler:
+            # Without flattening
+            result_normal = await crawler.arun(
+                url, config=CrawlerRunConfig(flatten_shadow_dom=False),
+            )
+
+            # With flattening
+            result_flat = await crawler.arun(
+                url, config=CrawlerRunConfig(flatten_shadow_dom=True),
+            )
+
+        if not result_normal.success or not result_flat.success:
+            record_result("Shadow DOM", "comparison crawl", False,
+                         "One or both crawls failed")
+            return
+
+        normal_len = len(result_normal.html or "")
+        flat_len = len(result_flat.html or "")
+
+        # Both should succeed (this page may not have shadow DOM, but
+        # the flattening pipeline should run without error)
+        record_result("Shadow DOM", "flatten_shadow_dom", True,
+                     f"Both crawls succeeded. Normal: {normal_len} chars, "
+                     f"Flattened: {flat_len} chars. Pipeline runs cleanly.")
+
+    except Exception as e:
+        record_result("Shadow DOM", "flatten_shadow_dom", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 6: Deep Crawl Cancellation — DFS with should_cancel
+# =============================================================================
+async def test_deep_crawl_cancellation():
+    """
+    Run a DFS deep crawl and cancel after 2 pages via should_cancel callback.
+
+    NEW in v0.8.5: All deep crawl strategies support cancel() method and
+    should_cancel callback for graceful cancellation.
+    """
+    print_test("Deep Crawl Cancellation", "DFS cancel after 2 pages")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+        from crawl4ai.deep_crawling import DFSDeepCrawlStrategy
+
+        pages_crawled = 0
+
+        def should_cancel():
+            return pages_crawled >= 2
+
+        async def track_state(state: Dict[str, Any]):
+            nonlocal pages_crawled
+            pages_crawled = state.get("pages_crawled", 0)
+
+        strategy = DFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=10,
+            should_cancel=should_cancel,
+            on_state_change=track_state,
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=strategy,
+            verbose=False,
+        )
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            await crawler.arun("https://books.toscrape.com", config=config)
+
+        if strategy.cancelled:
+            record_result("Deep Crawl Cancel", "should_cancel", True,
+                         f"Cancelled after {pages_crawled} pages (limit was 2)")
+        elif pages_crawled <= 3:
+            record_result("Deep Crawl Cancel", "should_cancel", True,
+                         f"Stopped at {pages_crawled} pages (callback triggered)")
+        else:
+            record_result("Deep Crawl Cancel", "should_cancel", False,
+                         f"Crawled {pages_crawled} pages — cancellation didn't work")
+
+    except Exception as e:
+        record_result("Deep Crawl Cancel", "should_cancel", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 7: Consent Popup Removal — Real Crawl
+# =============================================================================
+async def test_consent_popup_removal():
+    """
+    Crawl a site with remove_consent_popups=True and verify the JS runs
+    without errors and content is still captured.
+
+    NEW in v0.8.5: CrawlerRunConfig.remove_consent_popups runs a JS snippet
+    that clicks "Accept All" on 40+ CMP platforms.
+    """
+    print_test("Consent Popup Removal", "crawl with remove_consent_popups")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun(
+                "https://quotes.toscrape.com",
+                config=CrawlerRunConfig(remove_consent_popups=True),
+            )
+
+        if not result.success:
+            record_result("Consent Popup", "remove_consent_popups", False,
+                         f"Crawl failed: {result.error_message}")
+            return
+
+        md = result.markdown.raw_markdown if result.markdown else ""
+        if len(md) < 50:
+            record_result("Consent Popup", "remove_consent_popups", False,
+                         "Content too short — JS may have broken the page")
+            return
+
+        record_result("Consent Popup", "remove_consent_popups", True,
+                     f"Crawl succeeded with consent popup removal "
+                     f"({len(md)} chars markdown)")
+
+    except Exception as e:
+        record_result("Consent Popup", "remove_consent_popups", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 8: Source/Sibling Selector — Extract from Real Crawled HTML
+# =============================================================================
+async def test_source_sibling_selector():
+    """
+    Crawl a page, then use JsonCssExtractionStrategy with "source" field
+    to extract data spanning sibling elements.
+
+    NEW in v0.8.5: "source": "+ selector" navigates to sibling elements
+    before applying the field selector. Works in CSS and XPath strategies.
+    """
+    print_test("Source/Sibling Selector", "crawl + extract with source field")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+        from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+        # Use a schema with source field on synthetic HTML first to verify
+        # the feature works, then also run it through a real crawl pipeline
+        schema = {
+            "name": "SiblingItems",
+            "baseSelector": "tr.athing",
+            "fields": [
+                {"name": "title", "selector": ".titleline > a", "type": "text"},
+                {"name": "score", "selector": ".score", "type": "text", "source": "+ tr"},
+            ],
+        }
+
+        strategy = JsonCssExtractionStrategy(schema=schema)
+
+        # Test with sibling HTML structure
+        html = """
+        
+            
+                
+            
+            
+                
+            
+            
+                
+            
+            
+                
+            
+        
    Article One
    250 points
    Article Two
    180 points
    
+        """
+
+        # Run through the full crawl pipeline with raw: URL
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun(
+                f"raw:{html}",
+                config=CrawlerRunConfig(
+                    extraction_strategy=strategy,
+                ),
+            )
+
+        if not result.extracted_content:
+            record_result("Sibling Selector", "source field", False,
+                         "No extracted_content returned")
+            return
+
+        data = json.loads(result.extracted_content)
+
+        if len(data) < 2:
+            record_result("Sibling Selector", "source field", False,
+                         f"Expected 2 items, got {len(data)}")
+            return
+
+        if data[0].get("title") != "Article One":
+            record_result("Sibling Selector", "source field", False,
+                         f"Title mismatch: {data[0].get('title')}")
+            return
+
+        if data[0].get("score") != "250 points":
+            record_result("Sibling Selector", "source field", False,
+                         f"Sibling score not extracted: {data[0].get('score')}")
+            return
+
+        if data[1].get("score") != "180 points":
+            record_result("Sibling Selector", "source field", False,
+                         f"Second sibling score wrong: {data[1].get('score')}")
+            return
+
+        record_result("Sibling Selector", "source field via crawl pipeline", True,
+                     f"Extracted {len(data)} items with sibling scores through "
+                     f"full arun() pipeline")
+
+    except Exception as e:
+        record_result("Sibling Selector", "source field", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 9: GFM Table Compliance — Crawl Page with Tables
+# =============================================================================
+async def test_gfm_tables():
+    """
+    Crawl a page containing HTML tables and verify the markdown output
+    has proper GFM pipe delimiters.
+
+    NEW in v0.8.5: html2text now generates | col1 | col2 | with proper
+    leading/trailing pipes instead of col1 | col2.
+    """
+    print_test("GFM Table Compliance", "crawl page with tables")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+        # Use raw HTML with a table
+        html = """
+        
+            
    Product Comparison
    
+            
+                
+                
+                
+            
    ProductPriceRating
    Widget A$9.994.5
    Widget B$14.994.8
    
+        
+        """
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun(
+                f"raw:{html}",
+                config=CrawlerRunConfig(),
+            )
+
+        if not result.success or not result.markdown:
+            record_result("GFM Tables", "table crawl", False,
+                         "Crawl failed or no markdown")
+            return
+
+        md = result.markdown.raw_markdown
+        table_lines = [
+            l.strip() for l in md.split("\n")
+            if l.strip() and "|" in l
+        ]
+
+        if not table_lines:
+            record_result("GFM Tables", "pipe delimiters", False,
+                         f"No table lines found in markdown:\n{md}")
+            return
+
+        all_have_pipes = all(
+            l.startswith("|") and l.endswith("|")
+            for l in table_lines
+        )
+
+        if not all_have_pipes:
+            record_result("GFM Tables", "pipe delimiters", False,
+                         f"Missing leading/trailing pipes:\n" +
+                         "\n".join(table_lines))
+            return
+
+        record_result("GFM Tables", "pipe delimiters via crawl", True,
+                     f"Table has proper GFM pipes ({len(table_lines)} rows)")
+
+    except Exception as e:
+        record_result("GFM Tables", "pipe delimiters", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 10: avoid_ads / avoid_css — Real Crawl with Filtering
+# =============================================================================
+async def test_avoid_ads():
+    """
+    Crawl a real page with avoid_ads=True and verify content is still captured.
+
+    NEW in v0.8.5: BrowserConfig.avoid_ads blocks ad/tracker domains,
+    BrowserConfig.avoid_css blocks CSS resources at the network level.
+    """
+    print_test("Resource Filtering", "crawl with avoid_ads + avoid_css")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+
+        # Crawl with ad blocking enabled
+        async with AsyncWebCrawler(
+            config=BrowserConfig(
+                headless=True,
+                avoid_ads=True,
+                avoid_css=True,
+            ),
+            verbose=False,
+        ) as crawler:
+            result = await crawler.arun(
+                "https://quotes.toscrape.com",
+                config=CrawlerRunConfig(),
+            )
+
+        if not result.success:
+            record_result("Resource Filtering", "avoid_ads crawl", False,
+                         f"Crawl failed: {result.error_message}")
+            return
+
+        md = result.markdown.raw_markdown if result.markdown else ""
+
+        # Verify actual content was captured (quotes should be there)
+        has_quotes = "quote" in md.lower() or "albert einstein" in md.lower()
+        if not has_quotes and len(md) < 100:
+            record_result("Resource Filtering", "avoid_ads crawl", False,
+                         "Content missing — filtering may have broken the page")
+            return
+
+        record_result("Resource Filtering", "avoid_ads + avoid_css crawl", True,
+                     f"Content captured with ad/CSS blocking "
+                     f"({len(md)} chars markdown)")
+
+    except Exception as e:
+        record_result("Resource Filtering", "avoid_ads/css", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 11: Browser Recycling — Multi-page Crawl with memory_saving_mode
+# =============================================================================
+async def test_browser_recycling():
+    """
+    Crawl multiple pages with memory_saving_mode enabled and verify
+    all succeed without browser crashes.
+
+    NEW in v0.8.5: BrowserConfig.memory_saving_mode adds aggressive cache/V8
+    flags. max_pages_before_recycle triggers automatic browser restart.
+    """
+    print_test("Browser Recycling", "multi-page crawl with memory_saving_mode")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+
+        urls = [
+            "https://example.com",
+            "https://quotes.toscrape.com",
+            "https://httpbin.org/html",
+        ]
+
+        async with AsyncWebCrawler(
+            config=BrowserConfig(
+                headless=True,
+                memory_saving_mode=True,
+            ),
+            verbose=False,
+        ) as crawler:
+            succeeded = 0
+            for url in urls:
+                result = await crawler.arun(url, config=CrawlerRunConfig())
+                if result.success:
+                    succeeded += 1
+
+        if succeeded == len(urls):
+            record_result("Browser Recycling", "memory_saving_mode", True,
+                         f"All {succeeded}/{len(urls)} crawls succeeded with "
+                         f"memory_saving_mode")
+        else:
+            record_result("Browser Recycling", "memory_saving_mode", False,
+                         f"Only {succeeded}/{len(urls)} crawls succeeded")
+
+    except Exception as e:
+        record_result("Browser Recycling", "memory_saving_mode", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 12: BM25 Content Filter Deduplication
+# =============================================================================
+async def test_bm25_dedup():
+    """
+    Crawl a page using BM25ContentFilter and verify no duplicate chunks
+    in fit_markdown.
+
+    NEW in v0.8.5: BM25ContentFilter.filter_content() deduplicates output
+    chunks, keeping the first occurrence in document order.
+    """
+    print_test("BM25 Deduplication", "fit_markdown has no duplicates")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+        from crawl4ai.content_filter_strategy import BM25ContentFilter
+        from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun(
+                "https://quotes.toscrape.com",
+                config=CrawlerRunConfig(
+                    markdown_generator=DefaultMarkdownGenerator(
+                        content_filter=BM25ContentFilter(
+                            user_query="famous quotes about life",
+                        ),
+                    ),
+                ),
+            )
+
+        if not result.success:
+            record_result("BM25 Dedup", "fit_markdown", False,
+                         f"Crawl failed: {result.error_message}")
+            return
+
+        fit_md = result.markdown.fit_markdown if result.markdown else ""
+        if not fit_md:
+            record_result("BM25 Dedup", "fit_markdown", False,
+                         "No fit_markdown produced")
+            return
+
+        # Check for duplicate lines (non-empty, non-header)
+        lines = [l.strip() for l in fit_md.split("\n") if l.strip() and not l.startswith("#")]
+        unique_lines = list(dict.fromkeys(lines))  # preserves order
+        dupes = len(lines) - len(unique_lines)
+
+        if dupes > 0:
+            record_result("BM25 Dedup", "fit_markdown", False,
+                         f"{dupes} duplicate lines found in fit_markdown")
+            return
+
+        record_result("BM25 Dedup", "fit_markdown dedup", True,
+                     f"No duplicates in fit_markdown ({len(unique_lines)} unique lines)")
+
+    except Exception as e:
+        record_result("BM25 Dedup", "fit_markdown", False, f"Exception: {e}")
+
+
+# =============================================================================
+# TEST 13: cleaned_html Preserves class and id Attributes
+# =============================================================================
+async def test_cleaned_html_attrs():
+    """
+    Crawl a page and verify cleaned_html retains class and id attributes.
+
+    NEW in v0.8.5: 'class' and 'id' are now in IMPORTANT_ATTRS, so they
+    survive HTML cleaning. Previously they were stripped.
+    """
+    print_test("cleaned_html Attributes", "class and id preserved")
+
+    try:
+        from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+        html = """
+        
+            
    
+                
    Hello World
    
+                
    Introduction paragraph.
    
+            
    
+        
+        """
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            result = await crawler.arun(
+                f"raw:{html}",
+                config=CrawlerRunConfig(),
+            )
+
+        if not result.success or not result.cleaned_html:
+            record_result("cleaned_html Attrs", "class/id", False,
+                         "Crawl failed or no cleaned_html")
+            return
+
+        cleaned = result.cleaned_html
+        checks = []
+
+        if 'id="main-content"' in cleaned:
+            checks.append("id=main-content")
+        if 'class="container wide"' in cleaned or 'class="container' in cleaned:
+            checks.append("class=container")
+        if 'class="page-title"' in cleaned:
+            checks.append("class=page-title")
+        if 'id="intro"' in cleaned:
+            checks.append("id=intro")
+
+        if len(checks) < 2:
+            record_result("cleaned_html Attrs", "class/id", False,
+                         f"Only found {len(checks)} attrs: {checks}. "
+                         f"cleaned_html snippet: {cleaned[:200]}")
+            return
+
+        record_result("cleaned_html Attrs", "class/id preserved", True,
+                     f"Found {len(checks)} preserved attributes: {', '.join(checks)}")
+
+    except Exception as e:
+        record_result("cleaned_html Attrs", "class/id", False, f"Exception: {e}")
+
+
+# =============================================================================
+# MAIN
+# =============================================================================
+
+def print_summary():
+    """Print test results summary"""
+    print_header("TEST RESULTS SUMMARY")
+
+    passed = sum(1 for r in results if r.passed and not r.skipped)
+    failed = sum(1 for r in results if not r.passed and not r.skipped)
+    skipped = sum(1 for r in results if r.skipped)
+
+    print(f"\nTotal: {len(results)} tests")
+    print(f"  Passed:  {passed}")
+    print(f"  Failed:  {failed}")
+    print(f"  Skipped: {skipped}")
+
+    if failed > 0:
+        print("\nFailed Tests:")
+        for r in results:
+            if not r.passed and not r.skipped:
+                print(f"  - {r.name} ({r.feature}): {r.message}")
+
+    if skipped > 0:
+        print("\nSkipped Tests:")
+        for r in results:
+            if r.skipped:
+                print(f"  - {r.name} ({r.feature}): {r.message}")
+
+    print("\n" + "=" * 70)
+    if failed == 0:
+        print("All tests passed! v0.8.5 features verified.")
+    else:
+        print(f"WARNING: {failed} test(s) failed!")
+    print("=" * 70)
+
+    return failed == 0
+
+
+async def main():
+    """Run all verification tests"""
+    print_header("Crawl4AI v0.8.5 - Feature Verification Tests")
+    print("Running actual tests to verify new features...")
+    print("\nKey Features in v0.8.5:")
+    print("  - Anti-bot detection + retry + proxy escalation + fallback")
+    print("  - Shadow DOM flattening (flatten_shadow_dom)")
+    print("  - Deep crawl cancellation (cancel / should_cancel)")
+    print("  - Config defaults API (set_defaults / get_defaults / reset_defaults)")
+    print("  - Source/sibling selector in JSON extraction")
+    print("  - Consent popup removal (40+ CMP platforms)")
+    print("  - avoid_ads / avoid_css resource filtering")
+    print("  - Browser recycling + memory-saving mode")
+    print("  - GFM table compliance")
+    print("  - BM25 content filter deduplication")
+    print("  - cleaned_html preserves class/id attributes")
+    print("  - 49+ bug fixes including critical RCE and CVE patches")
+
+    tests = [
+        test_antibot_detection,             # Anti-bot + live crawl
+        test_crawl_stats,                   # crawl_stats tracking
+        test_proxy_escalation,              # Proxy chain + DIRECT crawl
+        test_config_defaults,               # set_defaults → real crawl
+        test_shadow_dom_flattening,         # Shadow DOM comparison crawl
+        test_deep_crawl_cancellation,       # DFS cancel at 2 pages
+        test_consent_popup_removal,         # Crawl with consent removal
+        test_source_sibling_selector,       # Sibling extraction via pipeline
+        test_gfm_tables,                    # Table crawl with pipe check
+        test_avoid_ads,                     # Crawl with ad/CSS blocking
+        test_browser_recycling,             # Multi-page memory_saving crawl
+        test_bm25_dedup,                    # BM25 fit_markdown dedup
+        test_cleaned_html_attrs,            # class/id preserved
+    ]
+
+    for test_func in tests:
+        try:
+            await test_func()
+        except Exception as e:
+            print(f"\nTest {test_func.__name__} crashed: {e}")
+            results.append(TestResult(
+                test_func.__name__,
+                "Unknown",
+                False,
+                f"Crashed: {e}"
+            ))
+
+    all_passed = print_summary()
+    return 0 if all_passed else 1
+
+
+if __name__ == "__main__":
+    try:
+        exit_code = asyncio.run(main())
+        sys.exit(exit_code)
+    except KeyboardInterrupt:
+        print("\n\nTests interrupted by user.")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n\nTest suite failed: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
diff --git a/docs/releases_review/demo_v0.9.1.py b/docs/releases_review/demo_v0.9.1.py
new file mode 100644
index 0000000..56d4fe7
--- /dev/null
+++ b/docs/releases_review/demo_v0.9.1.py
@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""
+Crawl4AI v0.9.1 Release Demo - Feature Verification Tests
+==========================================================
+
+This demo ACTUALLY RUNS and VERIFIES the key changes in v0.9.1.
+Each test executes real crawls or exercises the fix path end-to-end.
+
+Features / Fixes Verified:
+1. PruningContentFilter preserve_classes / preserve_tags whitelist
+2. Windows channel='chromium' fix (channel not passed for default)
+3. page_timeout ms-to-seconds conversion for HTTP mode
+4. html2text bypass_tables preserves table attributes
+5. Best-first batch ordering stability
+
+Usage:
+    python docs/releases_review/demo_v0.9.1.py
+"""
+
+import asyncio
+import sys
+from dataclasses import dataclass
+
+
+@dataclass
+class TestResult:
+    name: str
+    feature: str
+    passed: bool
+    message: str
+    skipped: bool = False
+
+
+results: list[TestResult] = []
+
+
+def print_header(title: str):
+    print(f"\n{'=' * 70}")
+    print(f"{title}")
+    print(f"{'=' * 70}")
+
+
+def print_test(name: str, feature: str):
+    print(f"\n[TEST] {name} ({feature})")
+    print("-" * 50)
+
+
+def record_result(name: str, feature: str, passed: bool, message: str, skipped: bool = False):
+    results.append(TestResult(name, feature, passed, message, skipped))
+    status = "SKIP" if skipped else ("PASS" if passed else "FAIL")
+    print(f"  [{status}] {message}")
+
+
+# ── Test 1: PruningContentFilter preserve_classes / preserve_tags ────
+
+async def test_preserve_whitelist():
+    """Verify that whitelisted classes/tags survive pruning."""
+    print_test("PruningContentFilter Whitelist", "preserve_classes / preserve_tags")
+
+    from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+    from crawl4ai.content_filter_strategy import PruningContentFilter
+    from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+    # HTML where short metadata elements would normally be pruned
+    html = """
+    
+    
    
+        
    Main Article Title
    
+        
    This is a long paragraph with enough content that the density-based
+        scoring will keep it. It contains multiple sentences and substantial text
+        that makes it clearly content rather than boilerplate navigation.
    
+        By John Doe
+        
+        
    Another substantial paragraph with enough text to be kept by the
+        pruning filter. This ensures the article has real content around the
+        short metadata elements we want to test preservation for.
    
+    
    
+    
+    """
+
+    # Without whitelist — author/time may be pruned
+    filter_no_wl = PruningContentFilter(threshold=0.48)
+    gen_no_wl = DefaultMarkdownGenerator(content_filter=filter_no_wl)
+
+    # With whitelist — author class and time tag protected
+    filter_wl = PruningContentFilter(
+        threshold=0.48,
+        preserve_classes=["author"],
+        preserve_tags=["time"],
+    )
+    gen_wl = DefaultMarkdownGenerator(content_filter=filter_wl)
+
+    config_no_wl = CrawlerRunConfig(
+        markdown_generator=gen_no_wl,
+    )
+    config_wl = CrawlerRunConfig(
+        markdown_generator=gen_wl,
+    )
+
+    async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
+        result_wl = await crawler.arun(url=f"raw://{html}", config=config_wl)
+
+    fit = result_wl.markdown.fit_markdown or ""
+    has_author = "John Doe" in fit
+    has_time = "July 8, 2026" in fit
+
+    if has_author and has_time:
+        record_result("preserve_whitelist", "PruningContentFilter", True,
+                       "Whitelisted class and tag preserved in fit_markdown")
+    else:
+        record_result("preserve_whitelist", "PruningContentFilter", False,
+                       f"author={'found' if has_author else 'MISSING'}, "
+                       f"time={'found' if has_time else 'MISSING'} in fit_markdown")
+
+
+# ── Test 2: channel='chromium' not passed to Playwright ──────────────
+
+async def test_channel_chromium_skipped():
+    """Verify that the default 'chromium' channel is not passed to launch args."""
+    print_test("Channel Chromium Skip", "Windows TargetClosedError fix")
+
+    from crawl4ai.browser_manager import BrowserManager
+    from crawl4ai.async_configs import BrowserConfig
+
+    config = BrowserConfig()  # default chrome_channel='chromium'
+    mgr = BrowserManager(config)
+    args = mgr._build_browser_args()
+
+    if "channel" not in args:
+        record_result("channel_skip", "browser_manager", True,
+                       "Default 'chromium' channel correctly omitted from launch args")
+    else:
+        record_result("channel_skip", "browser_manager", False,
+                       f"channel='{args['channel']}' should not be in launch args")
+
+    # Verify explicit non-default channel IS passed
+    config2 = BrowserConfig(chrome_channel="chrome")
+    mgr2 = BrowserManager(config2)
+    args2 = mgr2._build_browser_args()
+
+    if args2.get("channel") == "chrome":
+        record_result("channel_explicit", "browser_manager", True,
+                       "Explicit 'chrome' channel correctly passed")
+    else:
+        record_result("channel_explicit", "browser_manager", False,
+                       f"Expected channel='chrome', got {args2.get('channel')}")
+
+
+# ── Test 3: page_timeout ms→s conversion ─────────────────────────────
+
+async def test_page_timeout_conversion():
+    """Verify page_timeout is converted from ms to seconds for aiohttp."""
+    print_test("HTTP Timeout Conversion", "page_timeout ms to seconds")
+
+    from crawl4ai.async_configs import CrawlerRunConfig
+
+    config = CrawlerRunConfig()
+
+    # Default page_timeout is 60000 (ms). aiohttp timeout should be 60s, not 60000s.
+    import aiohttp
+    timeout = aiohttp.ClientTimeout(total=config.page_timeout / 1000)
+
+    if timeout.total == 60.0:
+        record_result("timeout_conversion", "HTTP mode", True,
+                       f"page_timeout {config.page_timeout}ms → {timeout.total}s correctly")
+    else:
+        record_result("timeout_conversion", "HTTP mode", False,
+                       f"Expected 60.0s, got {timeout.total}s")
+
+
+# ── Test 4: html2text bypass_tables preserves attributes ──────────────
+
+async def test_bypass_tables_attributes():
+    """Verify table tag attributes are preserved when bypass_tables is enabled."""
+    print_test("Table Attribute Preservation", "html2text bypass_tables fix")
+
+    from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+    from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+    html = """
+    
+    
+        
+        
+        
+    
    NameValue
    Alpha100
    Beta200
    
+    
+    """
+
+    config = CrawlerRunConfig(
+        markdown_generator=DefaultMarkdownGenerator(),
+    )
+
+    async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
+        result = await crawler.arun(url=f"raw://{html}", config=config)
+
+    md = result.markdown.raw_markdown or ""
+    # Tables should render and contain content
+    has_content = "Alpha" in md and "Beta" in md
+    if has_content:
+        record_result("bypass_tables", "html2text", True,
+                       "Table content preserved in markdown output")
+    else:
+        record_result("bypass_tables", "html2text", False,
+                       "Table content missing from markdown")
+
+
+# ── Test 5: Best-first batch ordering stability ──────────────────────
+
+async def test_bestfirst_ordering():
+    """Verify best-first scorer produces deterministic ordering."""
+    print_test("Best-First Ordering", "Stable batch ordering fix")
+
+    from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer, CompositeScorer
+
+    scorer = CompositeScorer([
+        KeywordRelevanceScorer(["python", "crawl"], weight=1.0),
+    ])
+
+    urls = [
+        "https://example.com/python-guide",
+        "https://example.com/crawl-tutorial",
+        "https://example.com/about",
+        "https://example.com/python-crawl-tips",
+        "https://example.com/contact",
+    ]
+
+    # Score multiple times — should be deterministic
+    scores_run1 = [scorer.score(u) for u in urls]
+    scores_run2 = [scorer.score(u) for u in urls]
+
+    if scores_run1 == scores_run2:
+        record_result("bestfirst_stable", "deep_crawling", True,
+                       "Scorer produces deterministic results across runs")
+    else:
+        record_result("bestfirst_stable", "deep_crawling", False,
+                       "Scorer results differ between runs")
+
+
+# ── Main ──────────────────────────────────────────────────────────────
+
+async def main():
+    print_header("Crawl4AI v0.9.1 Release Verification")
+
+    await test_preserve_whitelist()
+    await test_channel_chromium_skipped()
+    await test_page_timeout_conversion()
+    await test_bypass_tables_attributes()
+    await test_bestfirst_ordering()
+
+    # Summary
+    print_header("RESULTS SUMMARY")
+    total = len(results)
+    passed = sum(1 for r in results if r.passed and not r.skipped)
+    failed = sum(1 for r in results if not r.passed and not r.skipped)
+    skipped = sum(1 for r in results if r.skipped)
+
+    for r in results:
+        status = "SKIP" if r.skipped else ("PASS" if r.passed else "FAIL")
+        print(f"  [{status}] {r.name}: {r.message}")
+
+    print(f"\nTotal: {total} | Passed: {passed} | Failed: {failed} | Skipped: {skipped}")
+
+    if failed > 0:
+        print("\n⚠️  Some tests FAILED. Review before release.")
+        sys.exit(1)
+    else:
+        print("\n✅ All tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/releases_review/v0.3.74.overview.py b/docs/releases_review/v0.3.74.overview.py
new file mode 100644
index 0000000..4938db7
--- /dev/null
+++ b/docs/releases_review/v0.3.74.overview.py
@@ -0,0 +1,276 @@
+import os, sys
+
+# append the parent directory to the sys.path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+parent_parent_dir = os.path.dirname(parent_dir)
+sys.path.append(parent_parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+__data__ = os.path.join(__location__, "__data")
+import asyncio
+from pathlib import Path
+import aiohttp
+import json
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+
+# 1. File Download Processing Example
+async def download_example():
+    """Example of downloading files from Python.org"""
+    # downloads_path = os.path.join(os.getcwd(), "downloads")
+    downloads_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
+    os.makedirs(downloads_path, exist_ok=True)
+
+    print(f"Downloads will be saved to: {downloads_path}")
+
+    async with AsyncWebCrawler(
+        accept_downloads=True, downloads_path=downloads_path, verbose=True
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.python.org/downloads/",
+            js_code="""
+            // Find and click the first Windows installer link
+            const downloadLink = document.querySelector('a[href$=".exe"]');
+            if (downloadLink) {
+                console.log('Found download link:', downloadLink.href);
+                downloadLink.click();
+            } else {
+                console.log('No .exe download link found');
+            }
+            """,
+            delay_before_return_html=1,  # Wait 5 seconds to ensure download starts
+            cache_mode=CacheMode.BYPASS,
+        )
+
+        if result.downloaded_files:
+            print("\nDownload successful!")
+            print("Downloaded files:")
+            for file_path in result.downloaded_files:
+                print(f"- {file_path}")
+                print(f"  File size: {os.path.getsize(file_path) / (1024*1024):.2f} MB")
+        else:
+            print("\nNo files were downloaded")
+
+
+# 2. Local File and Raw HTML Processing Example
+async def local_and_raw_html_example():
+    """Example of processing local files and raw HTML"""
+    # Create a sample HTML file
+    sample_file = os.path.join(__data__, "sample.html")
+    with open(sample_file, "w") as f:
+        f.write(
+            """
+        
+            
    Test Content
    
+            
    This is a test paragraph.
    
+        
+        """
+        )
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        # Process local file
+        local_result = await crawler.arun(url=f"file://{os.path.abspath(sample_file)}")
+
+        # Process raw HTML
+        raw_html = """
+        
+            
    Raw HTML Test
    
+            
    This is a test of raw HTML processing.
    
+        
+        """
+        raw_result = await crawler.arun(url=f"raw:{raw_html}")
+
+        # Clean up
+        os.remove(sample_file)
+
+        print("Local file content:", local_result.markdown)
+        print("\nRaw HTML content:", raw_result.markdown)
+
+
+# 3. Enhanced Markdown Generation Example
+async def markdown_generation_example():
+    """Example of enhanced markdown generation with citations and LLM-friendly features"""
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        # Create a content filter (optional)
+        content_filter = BM25ContentFilter(
+            # user_query="History and cultivation",
+            bm25_threshold=1.0
+        )
+
+        result = await crawler.arun(
+            url="https://en.wikipedia.org/wiki/Apple",
+            css_selector="main div#bodyContent",
+            content_filter=content_filter,
+            cache_mode=CacheMode.BYPASS,
+        )
+
+        from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+        result = await crawler.arun(
+            url="https://en.wikipedia.org/wiki/Apple",
+            css_selector="main div#bodyContent",
+            content_filter=BM25ContentFilter(),
+        )
+        print(result.markdown_v2.fit_markdown)
+
+        print("\nMarkdown Generation Results:")
+        print(f"1. Original markdown length: {len(result.markdown)}")
+        print("2. New markdown versions (markdown_v2):")
+        print(f"   - Raw markdown length: {len(result.markdown_v2.raw_markdown)}")
+        print(
+            f"   - Citations markdown length: {len(result.markdown_v2.markdown_with_citations)}"
+        )
+        print(
+            f"   - References section length: {len(result.markdown_v2.references_markdown)}"
+        )
+        if result.markdown_v2.fit_markdown:
+            print(
+                f"   - Filtered markdown length: {len(result.markdown_v2.fit_markdown)}"
+            )
+
+        # Save examples to files
+        output_dir = os.path.join(__data__, "markdown_examples")
+        os.makedirs(output_dir, exist_ok=True)
+
+        # Save different versions
+        with open(os.path.join(output_dir, "1_raw_markdown.md"), "w") as f:
+            f.write(result.markdown_v2.raw_markdown)
+
+        with open(os.path.join(output_dir, "2_citations_markdown.md"), "w") as f:
+            f.write(result.markdown_v2.markdown_with_citations)
+
+        with open(os.path.join(output_dir, "3_references.md"), "w") as f:
+            f.write(result.markdown_v2.references_markdown)
+
+        if result.markdown_v2.fit_markdown:
+            with open(os.path.join(output_dir, "4_filtered_markdown.md"), "w") as f:
+                f.write(result.markdown_v2.fit_markdown)
+
+        print(f"\nMarkdown examples saved to: {output_dir}")
+
+        # Show a sample of citations and references
+        print("\nSample of markdown with citations:")
+        print(result.markdown_v2.markdown_with_citations[:500] + "...\n")
+        print("Sample of references:")
+        print(
+            "\n".join(result.markdown_v2.references_markdown.split("\n")[:10]) + "..."
+        )
+
+
+# 4. Browser Management Example
+async def browser_management_example():
+    """Example of using enhanced browser management features"""
+    # Use the specified user directory path
+    user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
+    os.makedirs(user_data_dir, exist_ok=True)
+
+    print(f"Browser profile will be saved to: {user_data_dir}")
+
+    async with AsyncWebCrawler(
+        use_managed_browser=True,
+        user_data_dir=user_data_dir,
+        headless=False,
+        verbose=True,
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://crawl4ai.com",
+            # session_id="persistent_session_1",
+            cache_mode=CacheMode.BYPASS,
+        )
+        # Use GitHub as an example - it's a good test for browser management
+        # because it requires proper browser handling
+        result = await crawler.arun(
+            url="https://github.com/trending",
+            # session_id="persistent_session_1",
+            cache_mode=CacheMode.BYPASS,
+        )
+
+        print("\nBrowser session result:", result.success)
+        if result.success:
+            print("Page title:", result.metadata.get("title", "No title found"))
+
+
+# 5. API Usage Example
+async def api_example():
+    """Example of using the new API endpoints"""
+    api_token = os.getenv("CRAWL4AI_API_TOKEN") or "test_api_code"
+    headers = {"Authorization": f"Bearer {api_token}"}
+    async with aiohttp.ClientSession() as session:
+        # Submit crawl job
+        crawl_request = {
+            "urls": ["https://news.ycombinator.com"],  # Hacker News as an example
+            "extraction_config": {
+                "type": "json_css",
+                "params": {
+                    "schema": {
+                        "name": "Hacker News Articles",
+                        "baseSelector": ".athing",
+                        "fields": [
+                            {"name": "title", "selector": ".title a", "type": "text"},
+                            {"name": "score", "selector": ".score", "type": "text"},
+                            {
+                                "name": "url",
+                                "selector": ".title a",
+                                "type": "attribute",
+                                "attribute": "href",
+                            },
+                        ],
+                    }
+                },
+            },
+            "crawler_params": {
+                "headless": True,
+                # "use_managed_browser": True
+            },
+            "cache_mode": "bypass",
+            # "screenshot": True,
+            # "magic": True
+        }
+
+        async with session.post(
+            "http://localhost:11235/crawl", json=crawl_request, headers=headers
+        ) as response:
+            task_data = await response.json()
+            task_id = task_data["task_id"]
+
+            # Check task status
+            while True:
+                async with session.get(
+                    f"http://localhost:11235/task/{task_id}", headers=headers
+                ) as status_response:
+                    result = await status_response.json()
+                    print(f"Task status: {result['status']}")
+
+                    if result["status"] == "completed":
+                        print("Task completed!")
+                        print("Results:")
+                        news = json.loads(result["results"][0]["extracted_content"])
+                        print(json.dumps(news[:4], indent=2))
+                        break
+                    else:
+                        await asyncio.sleep(1)
+
+
+# Main execution
+async def main():
+    # print("Running Crawl4AI feature examples...")
+
+    # print("\n1. Running Download Example:")
+    # await download_example()
+
+    # print("\n2. Running Markdown Generation Example:")
+    # await markdown_generation_example()
+
+    # # print("\n3. Running Local and Raw HTML Example:")
+    # await local_and_raw_html_example()
+
+    # # print("\n4. Running Browser Management Example:")
+    await browser_management_example()
+
+    # print("\n5. Running API Example:")
+    await api_example()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/releases_review/v0.7.5_docker_hooks_demo.py b/docs/releases_review/v0.7.5_docker_hooks_demo.py
new file mode 100644
index 0000000..6dbe23f
--- /dev/null
+++ b/docs/releases_review/v0.7.5_docker_hooks_demo.py
@@ -0,0 +1,655 @@
+#!/usr/bin/env python3
+"""
+🚀 Crawl4AI v0.7.5 - Docker Hooks System Complete Demonstration
+================================================================
+
+This file demonstrates the NEW Docker Hooks System introduced in v0.7.5.
+
+The Docker Hooks System is a completely NEW feature that provides pipeline
+customization through user-provided Python functions. It offers three approaches:
+
+1. String-based hooks for REST API
+2. hooks_to_string() utility to convert functions
+3. Docker Client with automatic conversion (most convenient)
+
+All three approaches are part of this NEW v0.7.5 feature!
+
+Perfect for video recording and demonstration purposes.
+
+Requirements:
+- Docker container running: docker run -p 11235:11235 unclecode/crawl4ai:latest
+- crawl4ai v0.7.5 installed: pip install crawl4ai==0.7.5
+"""
+
+import asyncio
+import requests
+import json
+import time
+from typing import Dict, Any
+
+# Import Crawl4AI components
+from crawl4ai import hooks_to_string
+from crawl4ai.docker_client import Crawl4aiDockerClient
+
+# Configuration
+DOCKER_URL = "http://localhost:11235"
+# DOCKER_URL = "http://localhost:11234"
+TEST_URLS = [
+    # "https://httpbin.org/html",
+    "https://www.kidocode.com",
+    "https://quotes.toscrape.com",
+]
+
+
+def print_section(title: str, description: str = ""):
+    """Print a formatted section header"""
+    print("\n" + "=" * 70)
+    print(f"  {title}")
+    if description:
+        print(f"  {description}")
+    print("=" * 70 + "\n")
+
+
+def check_docker_service() -> bool:
+    """Check if Docker service is running"""
+    try:
+        response = requests.get(f"{DOCKER_URL}/health", timeout=3)
+        return response.status_code == 200
+    except:
+        return False
+
+
+# ============================================================================
+# REUSABLE HOOK LIBRARY (NEW in v0.7.5)
+# ============================================================================
+
+async def performance_optimization_hook(page, context, **kwargs):
+    """
+    Performance Hook: Block unnecessary resources to speed up crawling
+    """
+    print("  [Hook] 🚀 Optimizing performance - blocking images and ads...")
+
+    # Block images
+    await context.route(
+        "**/*.{png,jpg,jpeg,gif,webp,svg,ico}",
+        lambda route: route.abort()
+    )
+
+    # Block ads and analytics
+    await context.route("**/analytics/*", lambda route: route.abort())
+    await context.route("**/ads/*", lambda route: route.abort())
+    await context.route("**/google-analytics.com/*", lambda route: route.abort())
+
+    print("  [Hook] ✓ Performance optimization applied")
+    return page
+
+
+async def viewport_setup_hook(page, context, **kwargs):
+    """
+    Viewport Hook: Set consistent viewport size for rendering
+    """
+    print("  [Hook] 🖥️  Setting viewport to 1920x1080...")
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    print("  [Hook] ✓ Viewport configured")
+    return page
+
+
+async def authentication_headers_hook(page, context, url, **kwargs):
+    """
+    Headers Hook: Add custom authentication and tracking headers
+    """
+    print(f"  [Hook] 🔐 Adding custom headers for {url[:50]}...")
+
+    await page.set_extra_http_headers({
+        'X-Crawl4AI-Version': '0.7.5',
+        'X-Custom-Hook': 'function-based-demo',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'User-Agent': 'Crawl4AI/0.7.5 (Educational Demo)'
+    })
+
+    print("  [Hook] ✓ Custom headers added")
+    return page
+
+
+async def lazy_loading_handler_hook(page, context, **kwargs):
+    """
+    Content Hook: Handle lazy-loaded content by scrolling
+    """
+    print("  [Hook] 📜 Scrolling to load lazy content...")
+
+    # Scroll to bottom
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+
+    # Scroll to middle
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2)")
+    await page.wait_for_timeout(500)
+
+    # Scroll back to top
+    await page.evaluate("window.scrollTo(0, 0)")
+    await page.wait_for_timeout(500)
+
+    print("  [Hook] ✓ Lazy content loaded")
+    return page
+
+
+async def page_analytics_hook(page, context, **kwargs):
+    """
+    Analytics Hook: Log page metrics before extraction
+    """
+    print("  [Hook] 📊 Collecting page analytics...")
+
+    metrics = await page.evaluate('''
+        () => ({
+            title: document.title,
+            images: document.images.length,
+            links: document.links.length,
+            scripts: document.scripts.length,
+            headings: document.querySelectorAll('h1, h2, h3').length,
+            paragraphs: document.querySelectorAll('p').length
+        })
+    ''')
+
+    print(f"  [Hook] 📈 Page: {metrics['title'][:50]}...")
+    print(f"         Links: {metrics['links']}, Images: {metrics['images']}, "
+          f"Headings: {metrics['headings']}, Paragraphs: {metrics['paragraphs']}")
+
+    return page
+
+
+# ============================================================================
+# DEMO 1: String-Based Hooks (NEW Docker Hooks System)
+# ============================================================================
+
+def demo_1_string_based_hooks():
+    """
+    Demonstrate string-based hooks with REST API (part of NEW Docker Hooks System)
+    """
+    print_section(
+        "DEMO 1: String-Based Hooks (REST API)",
+        "Part of the NEW Docker Hooks System - hooks as strings"
+    )
+
+    # Define hooks as strings
+    hooks_config = {
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("  [String Hook] Setting up page context...")
+    # Block images for performance
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+""",
+
+        "before_goto": """
+async def hook(page, context, url, **kwargs):
+    print(f"  [String Hook] Navigating to {url[:50]}...")
+    await page.set_extra_http_headers({
+        'X-Crawl4AI': 'string-based-hooks',
+        'X-Demo': 'v0.7.5'
+    })
+    return page
+""",
+
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    print("  [String Hook] Scrolling page...")
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    return page
+"""
+    }
+
+    # Prepare request payload
+    payload = {
+        "urls": [TEST_URLS[0]],
+        "hooks": {
+            "code": hooks_config,
+            "timeout": 30
+        },
+        "crawler_config": {
+            "cache_mode": "bypass"
+        }
+    }
+
+    print(f"🎯 Target URL: {TEST_URLS[0]}")
+    print(f"🔧 Configured {len(hooks_config)} string-based hooks")
+    print(f"📡 Sending request to Docker API...\n")
+
+    try:
+        start_time = time.time()
+        response = requests.post(f"{DOCKER_URL}/crawl", json=payload, timeout=60)
+        execution_time = time.time() - start_time
+
+        if response.status_code == 200:
+            result = response.json()
+
+            print(f"\n✅ Request successful! (took {execution_time:.2f}s)")
+
+            # Display results
+            if result.get('results') and result['results'][0].get('success'):
+                crawl_result = result['results'][0]
+                html_length = len(crawl_result.get('html', ''))
+                markdown_length = len(crawl_result.get('markdown', ''))
+
+                print(f"\n📊 Results:")
+                print(f"   • HTML length: {html_length:,} characters")
+                print(f"   • Markdown length: {markdown_length:,} characters")
+                print(f"   • URL: {crawl_result.get('url')}")
+
+                # Check hooks execution
+                if 'hooks' in result:
+                    hooks_info = result['hooks']
+                    print(f"\n🎣 Hooks Execution:")
+                    print(f"   • Status: {hooks_info['status']['status']}")
+                    print(f"   • Attached hooks: {len(hooks_info['status']['attached_hooks'])}")
+
+                    if 'summary' in hooks_info:
+                        summary = hooks_info['summary']
+                        print(f"   • Total executions: {summary['total_executions']}")
+                        print(f"   • Successful: {summary['successful']}")
+                        print(f"   • Success rate: {summary['success_rate']:.1f}%")
+            else:
+                print(f"⚠️ Crawl completed but no results")
+
+        else:
+            print(f"❌ Request failed with status {response.status_code}")
+            print(f"   Error: {response.text[:200]}")
+
+    except requests.exceptions.Timeout:
+        print("⏰ Request timed out after 60 seconds")
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+
+    print("\n" + "─" * 70)
+    print("✓ String-based hooks demo complete\n")
+
+
+# ============================================================================
+# DEMO 2: Function-Based Hooks with hooks_to_string() Utility
+# ============================================================================
+
+def demo_2_hooks_to_string_utility():
+    """
+    Demonstrate the new hooks_to_string() utility for converting functions
+    """
+    print_section(
+        "DEMO 2: hooks_to_string() Utility (NEW! ✨)",
+        "Convert Python functions to strings for REST API"
+    )
+
+    print("📦 Creating hook functions...")
+    print("   • performance_optimization_hook")
+    print("   • viewport_setup_hook")
+    print("   • authentication_headers_hook")
+    print("   • lazy_loading_handler_hook")
+
+    # Convert function objects to strings using the NEW utility
+    print("\n🔄 Converting functions to strings with hooks_to_string()...")
+
+    hooks_dict = {
+        "on_page_context_created": performance_optimization_hook,
+        "before_goto": authentication_headers_hook,
+        "before_retrieve_html": lazy_loading_handler_hook,
+    }
+
+    hooks_as_strings = hooks_to_string(hooks_dict)
+
+    print(f"✅ Successfully converted {len(hooks_as_strings)} functions to strings")
+
+    # Show a preview
+    print("\n📝 Sample converted hook (first 250 characters):")
+    print("─" * 70)
+    sample_hook = list(hooks_as_strings.values())[0]
+    print(sample_hook[:250] + "...")
+    print("─" * 70)
+
+    # Use the converted hooks with REST API
+    print("\n📡 Using converted hooks with REST API...")
+
+    payload = {
+        "urls": [TEST_URLS[0]],
+        "hooks": {
+            "code": hooks_as_strings,
+            "timeout": 30
+        }
+    }
+
+    try:
+        start_time = time.time()
+        response = requests.post(f"{DOCKER_URL}/crawl", json=payload, timeout=60)
+        execution_time = time.time() - start_time
+
+        if response.status_code == 200:
+            result = response.json()
+            print(f"\n✅ Request successful! (took {execution_time:.2f}s)")
+
+            if result.get('results') and result['results'][0].get('success'):
+                crawl_result = result['results'][0]
+                print(f"   • HTML length: {len(crawl_result.get('html', '')):,} characters")
+                print(f"   • Hooks executed successfully!")
+        else:
+            print(f"❌ Request failed: {response.status_code}")
+
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+
+    print("\n💡 Benefits of hooks_to_string():")
+    print("   ✓ Write hooks as regular Python functions")
+    print("   ✓ Full IDE support (autocomplete, syntax highlighting)")
+    print("   ✓ Type checking and linting")
+    print("   ✓ Easy to test and debug")
+    print("   ✓ Reusable across projects")
+    print("   ✓ Works with any REST API client")
+
+    print("\n" + "─" * 70)
+    print("✓ hooks_to_string() utility demo complete\n")
+
+
+# ============================================================================
+# DEMO 3: Docker Client with Automatic Conversion (RECOMMENDED! 🌟)
+# ============================================================================
+
+async def demo_3_docker_client_auto_conversion():
+    """
+    Demonstrate Docker Client with automatic hook conversion (RECOMMENDED)
+    """
+    print_section(
+        "DEMO 3: Docker Client with Auto-Conversion (RECOMMENDED! 🌟)",
+        "Pass function objects directly - conversion happens automatically!"
+    )
+
+    print("🐳 Initializing Crawl4AI Docker Client...")
+    client = Crawl4aiDockerClient(base_url=DOCKER_URL)
+
+    print("✅ Client ready!\n")
+
+    # Use our reusable hook library - just pass the function objects!
+    print("📚 Using reusable hook library:")
+    print("   • performance_optimization_hook")
+    print("   • viewport_setup_hook")
+    print("   • authentication_headers_hook")
+    print("   • lazy_loading_handler_hook")
+    print("   • page_analytics_hook")
+
+    print("\n🎯 Target URL: " + TEST_URLS[1])
+    print("🚀 Starting crawl with automatic hook conversion...\n")
+
+    try:
+        start_time = time.time()
+
+        # Pass function objects directly - NO manual conversion needed! ✨
+        results = await client.crawl(
+            urls=[TEST_URLS[0]],
+            hooks={
+                "on_page_context_created": performance_optimization_hook,
+                "before_goto": authentication_headers_hook,
+                "before_retrieve_html": lazy_loading_handler_hook,
+                "before_return_html": page_analytics_hook,
+            },
+            hooks_timeout=30
+        )
+
+        execution_time = time.time() - start_time
+
+        print(f"\n✅ Crawl completed! (took {execution_time:.2f}s)\n")
+
+        # Display results
+        if results and results.success:
+            result = results
+            print(f"📊 Results:")
+            print(f"   • URL: {result.url}")
+            print(f"   • Success: {result.success}")
+            print(f"   • HTML length: {len(result.html):,} characters")
+            print(f"   • Markdown length: {len(result.markdown):,} characters")
+
+            # Show metadata
+            if result.metadata:
+                print(f"\n📋 Metadata:")
+                print(f"   • Title: {result.metadata.get('title', 'N/A')}")
+                print(f"   • Description: {result.metadata.get('description', 'N/A')}")
+
+            # Show links
+            if result.links:
+                internal_count = len(result.links.get('internal', []))
+                external_count = len(result.links.get('external', []))
+                print(f"\n🔗 Links Found:")
+                print(f"   • Internal: {internal_count}")
+                print(f"   • External: {external_count}")
+        else:
+            print(f"⚠️ Crawl completed but no successful results")
+            if results:
+                print(f"   Error: {results.error_message}")
+
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+        import traceback
+        traceback.print_exc()
+
+    print("\n🌟 Why Docker Client is RECOMMENDED:")
+    print("   ✓ Automatic function-to-string conversion")
+    print("   ✓ No manual hooks_to_string() calls needed")
+    print("   ✓ Cleaner, more Pythonic code")
+    print("   ✓ Full type hints and IDE support")
+    print("   ✓ Built-in error handling")
+    print("   ✓ Async/await support")
+
+    print("\n" + "─" * 70)
+    print("✓ Docker Client auto-conversion demo complete\n")
+
+
+# ============================================================================
+# DEMO 4: Advanced Use Case - Complete Hook Pipeline
+# ============================================================================
+
+async def demo_4_complete_hook_pipeline():
+    """
+    Demonstrate a complete hook pipeline using all 8 hook points
+    """
+    print_section(
+        "DEMO 4: Complete Hook Pipeline",
+        "Using all 8 available hook points for comprehensive control"
+    )
+
+    # Define all 8 hooks
+    async def on_browser_created_hook(browser, **kwargs):
+        """Hook 1: Called after browser is created"""
+        print("  [Pipeline] 1/8 Browser created")
+        return browser
+
+    async def on_page_context_created_hook(page, context, **kwargs):
+        """Hook 2: Called after page context is created"""
+        print("  [Pipeline] 2/8 Page context created - setting up...")
+        await page.set_viewport_size({"width": 1920, "height": 1080})
+        return page
+
+    async def on_user_agent_updated_hook(page, context, user_agent, **kwargs):
+        """Hook 3: Called when user agent is updated"""
+        print(f"  [Pipeline] 3/8 User agent updated: {user_agent[:50]}...")
+        return page
+
+    async def before_goto_hook(page, context, url, **kwargs):
+        """Hook 4: Called before navigating to URL"""
+        print(f"  [Pipeline] 4/8 Before navigation to: {url[:60]}...")
+        return page
+
+    async def after_goto_hook(page, context, url, response, **kwargs):
+        """Hook 5: Called after navigation completes"""
+        print(f"  [Pipeline] 5/8 After navigation - Status: {response.status if response else 'N/A'}")
+        await page.wait_for_timeout(1000)
+        return page
+
+    async def on_execution_started_hook(page, context, **kwargs):
+        """Hook 6: Called when JavaScript execution starts"""
+        print("  [Pipeline] 6/8 JavaScript execution started")
+        return page
+
+    async def before_retrieve_html_hook(page, context, **kwargs):
+        """Hook 7: Called before retrieving HTML"""
+        print("  [Pipeline] 7/8 Before HTML retrieval - scrolling...")
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        return page
+
+    async def before_return_html_hook(page, context, html, **kwargs):
+        """Hook 8: Called before returning HTML"""
+        print(f"  [Pipeline] 8/8 Before return - HTML length: {len(html):,} chars")
+        return page
+
+    print("🎯 Target URL: " + TEST_URLS[0])
+    print("🔧 Configured ALL 8 hook points for complete pipeline control\n")
+
+    client = Crawl4aiDockerClient(base_url=DOCKER_URL)
+
+    try:
+        print("🚀 Starting complete pipeline crawl...\n")
+        start_time = time.time()
+
+        results = await client.crawl(
+            urls=[TEST_URLS[0]],
+            hooks={
+                "on_browser_created": on_browser_created_hook,
+                "on_page_context_created": on_page_context_created_hook,
+                "on_user_agent_updated": on_user_agent_updated_hook,
+                "before_goto": before_goto_hook,
+                "after_goto": after_goto_hook,
+                "on_execution_started": on_execution_started_hook,
+                "before_retrieve_html": before_retrieve_html_hook,
+                "before_return_html": before_return_html_hook,
+            },
+            hooks_timeout=45
+        )
+
+        execution_time = time.time() - start_time
+
+        if results and results.success:
+            print(f"\n✅ Complete pipeline executed successfully! (took {execution_time:.2f}s)")
+            print(f"   • All 8 hooks executed in sequence")
+            print(f"   • HTML length: {len(results.html):,} characters")
+        else:
+            print(f"⚠️ Pipeline completed with warnings")
+
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+
+    print("\n📚 Available Hook Points:")
+    print("   1. on_browser_created - Browser initialization")
+    print("   2. on_page_context_created - Page context setup")
+    print("   3. on_user_agent_updated - User agent configuration")
+    print("   4. before_goto - Pre-navigation setup")
+    print("   5. after_goto - Post-navigation processing")
+    print("   6. on_execution_started - JavaScript execution start")
+    print("   7. before_retrieve_html - Pre-extraction processing")
+    print("   8. before_return_html - Final HTML processing")
+
+    print("\n" + "─" * 70)
+    print("✓ Complete hook pipeline demo complete\n")
+
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+async def main():
+    """
+    Run all demonstrations
+    """
+    print("\n" + "=" * 70)
+    print("  🚀 Crawl4AI v0.7.5 - Docker Hooks Complete Demonstration")
+    print("=" * 70)
+
+    # Check Docker service
+    print("\n🔍 Checking Docker service status...")
+    if not check_docker_service():
+        print("❌ Docker service is not running!")
+        print("\n📋 To start the Docker service:")
+        print("   docker run -p 11235:11235 unclecode/crawl4ai:latest")
+        print("\nPlease start the service and run this demo again.")
+        return
+
+    print("✅ Docker service is running!\n")
+
+    # Run all demos
+    demos = [
+        ("String-Based Hooks (REST API)", demo_1_string_based_hooks, False),
+        ("hooks_to_string() Utility", demo_2_hooks_to_string_utility, False),
+        ("Docker Client Auto-Conversion", demo_3_docker_client_auto_conversion, True),
+        # ("Complete Hook Pipeline", demo_4_complete_hook_pipeline, True),
+    ]
+
+    for i, (name, demo_func, is_async) in enumerate(demos, 1):
+        print(f"\n{'🔷' * 35}")
+        print(f"Starting Demo {i}/{len(demos)}: {name}")
+        print(f"{'🔷' * 35}\n")
+
+        try:
+            if is_async:
+                await demo_func()
+            else:
+                demo_func()
+
+            print(f"✅ Demo {i} completed successfully!")
+
+            # Pause between demos (except the last one)
+            if i < len(demos):
+                print("\n⏸️  Press Enter to continue to next demo...")
+                # input()
+
+        except KeyboardInterrupt:
+            print(f"\n⏹️  Demo interrupted by user")
+            break
+        except Exception as e:
+            print(f"\n❌ Demo {i} failed: {str(e)}")
+            import traceback
+            traceback.print_exc()
+            print("\nContinuing to next demo...\n")
+            continue
+
+    # Final summary
+    print("\n" + "=" * 70)
+    print("  🎉 All Demonstrations Complete!")
+    print("=" * 70)
+
+    print("\n📊 Summary of v0.7.5 Docker Hooks System:")
+    print("\n🆕 COMPLETELY NEW FEATURE in v0.7.5:")
+    print("   The Docker Hooks System lets you customize the crawling pipeline")
+    print("   with user-provided Python functions at 8 strategic points.")
+
+    print("\n✨ Three Ways to Use Docker Hooks (All NEW!):")
+    print("   1. String-based - Write hooks as strings for REST API")
+    print("   2. hooks_to_string() - Convert Python functions to strings")
+    print("   3. Docker Client - Automatic conversion (RECOMMENDED)")
+
+    print("\n💡 Key Benefits:")
+    print("   ✓ Full IDE support (autocomplete, syntax highlighting)")
+    print("   ✓ Type checking and linting")
+    print("   ✓ Easy to test and debug")
+    print("   ✓ Reusable across projects")
+    print("   ✓ Complete pipeline control")
+
+    print("\n🎯 8 Hook Points Available:")
+    print("   • on_browser_created, on_page_context_created")
+    print("   • on_user_agent_updated, before_goto, after_goto")
+    print("   • on_execution_started, before_retrieve_html, before_return_html")
+
+    print("\n📚 Resources:")
+    print("   • Docs: https://docs.crawl4ai.com")
+    print("   • GitHub: https://github.com/unclecode/crawl4ai")
+    print("   • Discord: https://discord.gg/jP8KfhDhyN")
+
+    print("\n" + "=" * 70)
+    print("  Happy Crawling with v0.7.5! 🕷️")
+    print("=" * 70 + "\n")
+
+
+if __name__ == "__main__":
+    print("\n🎬 Starting Crawl4AI v0.7.5 Docker Hooks Demonstration...")
+    print("Press Ctrl+C anytime to exit\n")
+
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\n\n👋 Demo stopped by user. Thanks for exploring Crawl4AI v0.7.5!")
+    except Exception as e:
+        print(f"\n\n❌ Demo error: {str(e)}")
+        import traceback
+        traceback.print_exc()
diff --git a/docs/releases_review/v0.7.5_video_walkthrough.ipynb b/docs/releases_review/v0.7.5_video_walkthrough.ipynb
new file mode 100644
index 0000000..16738cc
--- /dev/null
+++ b/docs/releases_review/v0.7.5_video_walkthrough.ipynb
@@ -0,0 +1,1516 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 🚀 Crawl4AI v0.7.5 - Complete Feature Walkthrough\n",
+    "\n",
+    "Welcome to Crawl4AI v0.7.5! This notebook demonstrates all the new features introduced in this release.\n",
+    "\n",
+    "## 📋 What's New in v0.7.5\n",
+    "\n",
+    "1. **🔧 Docker Hooks System** - NEW! Complete pipeline customization with user-provided Python functions\n",
+    "2. **🤖 Enhanced LLM Integration** - Custom providers with temperature control\n",
+    "3. **🔒 HTTPS Preservation** - Secure internal link handling\n",
+    "4. **🛠️ Multiple Bug Fixes** - Community-reported issues resolved\n",
+    "\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 📦 Setup and Installation\n",
+    "\n",
+    "First, let's make sure we have the latest version installed:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Crawl4AI v0.7.5 ready!\n"
+     ]
+    }
+   ],
+   "source": [
+    "# # Install or upgrade to v0.7.5\n",
+    "# !pip install -U crawl4ai==0.7.5 --quiet\n",
+    "\n",
+    "# Import required modules\n",
+    "import asyncio\n",
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()  # For Jupyter compatibility\n",
+    "\n",
+    "from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode\n",
+    "from crawl4ai import FilterChain, URLPatternFilter, BFSDeepCrawlStrategy\n",
+    "from crawl4ai import hooks_to_string\n",
+    "\n",
+    "print(\"✅ Crawl4AI v0.7.5 ready!\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🔧 Feature 1: Docker Hooks System (NEW! 🆕)\n",
+    "\n",
+    "### What is it?\n",
+    "v0.7.5 introduces a **completely new Docker Hooks System** that lets you inject custom Python functions at 8 key points in the crawling pipeline. This gives you full control over:\n",
+    "- Authentication setup\n",
+    "- Performance optimization\n",
+    "- Content processing\n",
+    "- Custom behavior at each stage\n",
+    "\n",
+    "### Three Ways to Use Docker Hooks\n",
+    "\n",
+    "The Docker Hooks System offers three approaches, all part of this new feature:\n",
+    "\n",
+    "1. **String-based hooks** - Write hooks as strings for REST API\n",
+    "2. **Using `hooks_to_string()` utility** - Convert Python functions to strings\n",
+    "3. **Docker Client auto-conversion** - Pass functions directly (most convenient)\n",
+    "\n",
+    "All three approaches are NEW in v0.7.5!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🔒 Feature 2: HTTPS Preservation for Internal Links\n",
+    "\n",
+    "### Problem\n",
+    "When crawling HTTPS sites, internal links sometimes get downgraded to HTTP, breaking authentication and causing security warnings.\n",
+    "\n",
+    "### Solution  \n",
+    "The new `preserve_https_for_internal_links=True` parameter maintains HTTPS protocol for all internal links."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🔒 Testing HTTPS Preservation\n",
+      "\n",
+      "============================================================\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [INIT].... → Crawl4AI 0.7.5 \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m[\u001b[0m\u001b[36mINIT\u001b[0m\u001b[1;36m]\u001b[0m\u001b[36m...\u001b[0m\u001b[36m. → Crawl4AI \u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[36m.\u001b[0m\u001b[1;36m5\u001b[0m\u001b[36m \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [FETCH]... ↓ https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 1.98s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m                                                                          |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m98s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [SCRAPE].. ◆ https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 0.01s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m                                                                          |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m01s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [COMPLETE] ● https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 2.00s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m                                                                          |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m2.\u001b[0m\u001b[32m00s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [FETCH]... ↓ https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 0.72s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m                                                                          |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m72s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [SCRAPE].. ◆ https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 0.01s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m                                                                          |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m01s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [COMPLETE] ● https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 0.73s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m                                                                          |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m73s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [FETCH]... ↓ https://quotes.toscrape.com/login                                                                    |\n",
+       "✓ | ⏱: 0.83s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/login\u001b[0m\u001b[32m                                                                    |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m83s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [SCRAPE].. ◆ https://quotes.toscrape.com/login                                                                    |\n",
+       "✓ | ⏱: 0.00s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/login\u001b[0m\u001b[32m                                                                    |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m00s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [COMPLETE] ● https://quotes.toscrape.com/login                                                                    |\n",
+       "✓ | ⏱: 0.83s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/login\u001b[0m\u001b[32m                                                                    |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m83s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [FETCH]... ↓ https://quotes.toscrape.com/tag/change/page/1                                                        |\n",
+       "✓ | ⏱: 1.11s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/tag/change/page/1\u001b[0m\u001b[32m                                                        |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m11s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [SCRAPE].. ◆ https://quotes.toscrape.com/tag/change/page/1                                                        |\n",
+       "✓ | ⏱: 0.00s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/tag/change/page/1\u001b[0m\u001b[32m                                                        |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m00s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [COMPLETE] ● https://quotes.toscrape.com/tag/change/page/1                                                        |\n",
+       "✓ | ⏱: 1.12s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/tag/change/page/1\u001b[0m\u001b[32m                                                        |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m12s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [FETCH]... ↓ https://quotes.toscrape.com/author/Albert-Einstein                                                   |\n",
+       "✓ | ⏱: 1.32s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/author/Albert-Einstein\u001b[0m\u001b[32m                                                   |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m32s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [SCRAPE].. ◆ https://quotes.toscrape.com/author/Albert-Einstein                                                   |\n",
+       "✓ | ⏱: 0.00s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/author/Albert-Einstein\u001b[0m\u001b[32m                                                   |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m00s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [COMPLETE] ● https://quotes.toscrape.com/author/Albert-Einstein                                                   |\n",
+       "✓ | ⏱: 1.33s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/author/Albert-Einstein\u001b[0m\u001b[32m                                                   |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m33s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "📊 Results:\n",
+      "  Pages crawled: 5\n",
+      "  Total internal links (from first page): 47\n",
+      "  HTTPS links: 47 ✅\n",
+      "  HTTP links: 0 \n",
+      "  HTTPS preservation rate: 100.0%\n",
+      "\n",
+      "🔗 Sample HTTPS-preserved links:\n",
+      "  → https://quotes.toscrape.com/\n",
+      "  → https://quotes.toscrape.com/login\n",
+      "  → https://quotes.toscrape.com/author/Albert-Einstein\n",
+      "  → https://quotes.toscrape.com/tag/change/page/1\n",
+      "  → https://quotes.toscrape.com/tag/deep-thoughts/page/1\n",
+      "\n",
+      "============================================================\n",
+      "✅ HTTPS Preservation Demo Complete!\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "async def demo_https_preservation():\n",
+    "    \"\"\"\n",
+    "    Demonstrate HTTPS preservation with deep crawling\n",
+    "    \"\"\"\n",
+    "    print(\"🔒 Testing HTTPS Preservation\\n\")\n",
+    "    print(\"=\" * 60)\n",
+    "    \n",
+    "    # Setup URL filter for quotes.toscrape.com\n",
+    "    url_filter = URLPatternFilter(\n",
+    "        patterns=[r\"^(https:\\/\\/)?quotes\\.toscrape\\.com(\\/.*)?$\"]\n",
+    "    )\n",
+    "    \n",
+    "    # Configure crawler with HTTPS preservation\n",
+    "    config = CrawlerRunConfig(\n",
+    "        exclude_external_links=True,\n",
+    "        preserve_https_for_internal_links=True,  # 🆕 NEW in v0.7.5\n",
+    "        cache_mode=CacheMode.BYPASS,\n",
+    "        deep_crawl_strategy=BFSDeepCrawlStrategy(\n",
+    "            max_depth=2,\n",
+    "            max_pages=5,\n",
+    "            filter_chain=FilterChain([url_filter])\n",
+    "        )\n",
+    "    )\n",
+    "    \n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        # With deep_crawl_strategy, arun() returns a list of CrawlResult objects\n",
+    "        results = await crawler.arun(\n",
+    "            url=\"https://quotes.toscrape.com\",\n",
+    "            config=config\n",
+    "        )\n",
+    "        \n",
+    "        # Analyze the first result\n",
+    "        if results and len(results) > 0:\n",
+    "            first_result = results[0]\n",
+    "            internal_links = [link['href'] for link in first_result.links['internal']]\n",
+    "            \n",
+    "            # Check HTTPS preservation\n",
+    "            https_links = [link for link in internal_links if link.startswith('https://')]\n",
+    "            http_links = [link for link in internal_links if link.startswith('http://') and not link.startswith('https://')]\n",
+    "            \n",
+    "            print(f\"\\n📊 Results:\")\n",
+    "            print(f\"  Pages crawled: {len(results)}\")\n",
+    "            print(f\"  Total internal links (from first page): {len(internal_links)}\")\n",
+    "            print(f\"  HTTPS links: {len(https_links)} ✅\")\n",
+    "            print(f\"  HTTP links: {len(http_links)} {'⚠️' if http_links else ''}\")\n",
+    "            if internal_links:\n",
+    "                print(f\"  HTTPS preservation rate: {len(https_links)/len(internal_links)*100:.1f}%\")\n",
+    "            \n",
+    "            print(f\"\\n🔗 Sample HTTPS-preserved links:\")\n",
+    "            for link in https_links[:5]:\n",
+    "                print(f\"  → {link}\")\n",
+    "        else:\n",
+    "            print(f\"\\n⚠️ No results returned\")\n",
+    "    \n",
+    "    print(\"\\n\" + \"=\" * 60)\n",
+    "    print(\"✅ HTTPS Preservation Demo Complete!\\n\")\n",
+    "\n",
+    "# Run the demo\n",
+    "await demo_https_preservation()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🤖 Feature 3: Enhanced LLM Integration\n",
+    "\n",
+    "### What's New\n",
+    "- Custom `temperature` parameter for creativity control\n",
+    "- `base_url` for custom API endpoints\n",
+    "- Better multi-provider support\n",
+    "\n",
+    "### Example with Custom Temperature"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🤖 Testing Enhanced LLM Integration\n",
+      "\n",
+      "============================================================\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/k0/7502j87n0_q4f9g82c0w8ks80000gn/T/ipykernel_15029/173393508.py:47: PydanticDeprecatedSince20: The `schema` method is deprecated; use `model_json_schema` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/\n",
+      "  schema=Article.schema(),\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [INIT].... → Crawl4AI 0.7.5 \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m[\u001b[0m\u001b[36mINIT\u001b[0m\u001b[1;36m]\u001b[0m\u001b[36m...\u001b[0m\u001b[36m. → Crawl4AI \u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[36m.\u001b[0m\u001b[1;36m5\u001b[0m\u001b[36m \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [FETCH]... ↓ https://en.wikipedia.org/wiki/Artificial_intelligence                                                |\n",
+       "✓ | ⏱: 3.05s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://en.wikipedia.org/wiki/Artificial_intelligence\u001b[0m\u001b[32m                                                |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m3.\u001b[0m\u001b[32m05s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [SCRAPE].. ◆ https://en.wikipedia.org/wiki/Artificial_intelligence                                                |\n",
+       "✓ | ⏱: 0.63s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://en.wikipedia.org/wiki/Artificial_intelligence\u001b[0m\u001b[32m                                                |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m63s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [EXTRACT]. ■ https://en.wikipedia.org/wiki/Artificial_intelligence                                                |\n",
+       "✓ | ⏱: 20.74s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mEXTRACT\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m. ■ \u001b[0m\u001b[4;32mhttps://en.wikipedia.org/wiki/Artificial_intelligence\u001b[0m\u001b[32m                                                |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m20.\u001b[0m\u001b[32m74s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [COMPLETE] ● https://en.wikipedia.org/wiki/Artificial_intelligence                                                |\n",
+       "✓ | ⏱: 24.42s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://en.wikipedia.org/wiki/Artificial_intelligence\u001b[0m\u001b[32m                                                |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m24.\u001b[0m\u001b[32m42s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "✅ LLM Extraction Successful!\n",
+      "\n",
+      "📄 Extracted Content:\n",
+      "[\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"Artificial intelligence (AI) refers to the simulation of human intelligence in machines that are programmed to think and learn like humans. AI can be applied in various fields and has numerous applications, including health, finance, and military.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Goals\",\n",
+      "            \"Techniques\",\n",
+      "            \"Applications\",\n",
+      "            \"Ethics\",\n",
+      "            \"History\",\n",
+      "            \"Philosophy\",\n",
+      "            \"Future\",\n",
+      "            \"In fiction\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"The article discusses artificial intelligence (AI), its various techniques, applications, and advancements, particularly focusing on machine learning, deep learning, and neural networks. It highlights the evolution of AI technologies, including generative pre-trained transformers (GPT), and their impact on fields such as healthcare, gaming, and mathematics.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Classifiers and pattern matching\",\n",
+      "            \"Artificial neural networks\",\n",
+      "            \"Deep learning\",\n",
+      "            \"Generative pre-trained transformers (GPT)\",\n",
+      "            \"Hardware and software for AI\",\n",
+      "            \"Applications of AI\",\n",
+      "            \"AI in healthcare\",\n",
+      "            \"AI in games\",\n",
+      "            \"AI in mathematics\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"Artificial intelligence (AI) is the capability of computational systems to perform tasks typically associated with human intelligence, such as learning, reasoning, problem-solving, perception, and decision-making. It is a field of research in computer science that develops methods and software enabling machines to perceive their environment and take actions to achieve defined goals. AI has seen significant advancements and applications in various domains, including web search engines, recommendation systems, virtual assistants, and autonomous vehicles, among others.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Goals\",\n",
+      "            \"Reasoning and problem-solving\",\n",
+      "            \"Knowledge representation\",\n",
+      "            \"Planning and decision-making\",\n",
+      "            \"Learning\",\n",
+      "            \"Applications\",\n",
+      "            \"Philosophy\",\n",
+      "            \"History\",\n",
+      "            \"Controversies\",\n",
+      "            \"Ethics\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"The article discusses artificial intelligence (AI), its various techniques, and applications. It covers the foundational concepts of AI, including machine learning, natural language processing, perception, social intelligence, and general intelligence. The article also highlights the methods used in AI research, such as search and optimization, logic, probabilistic methods, and classifiers.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Markov decision processes\",\n",
+      "            \"Machine learning\",\n",
+      "            \"Supervised learning\",\n",
+      "            \"Unsupervised learning\",\n",
+      "            \"Reinforcement learning\",\n",
+      "            \"Transfer learning\",\n",
+      "            \"Deep learning\",\n",
+      "            \"Natural language processing\",\n",
+      "            \"Machine perception\",\n",
+      "            \"Social intelligence\",\n",
+      "            \"Artificial general intelligence\",\n",
+      "            \"Search and optimization\",\n",
+      "            \"Logic\",\n",
+      "            \"Probabilistic methods\",\n",
+      "            \"Classifiers and statistical learning methods\",\n",
+      "            \"Artificial neural networks\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the complexities and challenges associated with artificial intelligence (AI), particularly focusing on issues of bias, fairness, transparency, and the potential risks posed by AI technologies. It highlights the ethical implications of AI systems, the lack of diversity among AI developers, and the existential risks associated with advanced AI.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Bias and fairness in AI\",\n",
+      "            \"Lack of transparency in AI systems\",\n",
+      "            \"Weaponization of AI\",\n",
+      "            \"Technological unemployment due to AI\",\n",
+      "            \"Existential risk from AI\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"The article discusses the advancements and applications of artificial intelligence (AI) across various fields, including mathematics, finance, military, generative AI, and more. It highlights the capabilities of AI models, their limitations, and the ethical considerations surrounding their use.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Mathematics\",\n",
+      "            \"Finance\",\n",
+      "            \"Military applications\",\n",
+      "            \"Generative AI\",\n",
+      "            \"AI agents\",\n",
+      "            \"Web search\",\n",
+      "            \"Sexuality\",\n",
+      "            \"Industry-specific tasks\",\n",
+      "            \"Ethics\",\n",
+      "            \"Privacy and copyright\",\n",
+      "            \"Dominance by tech giants\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses various aspects of artificial intelligence (AI), including its impact on privacy, copyright issues, environmental concerns, misinformation, and algorithmic bias. It highlights the dominance of big tech companies in the AI landscape and the increasing power demands of AI technologies.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Privacy and Fairness\",\n",
+      "            \"Generative AI and Copyright\",\n",
+      "            \"Dominance by Tech Giants\",\n",
+      "            \"Power Needs and Environmental Impacts\",\n",
+      "            \"Misinformation\",\n",
+      "            \"Algorithmic Bias and Fairness\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"The article discusses the mixed opinions among experts regarding the risks associated with artificial intelligence (AI), particularly concerning superintelligent AI. It highlights concerns from notable figures in the field about existential risks, the importance of establishing safety guidelines, and the ongoing debate between pessimistic and optimistic views on AI's future impact. The article also covers ethical considerations, open-source developments, regulatory efforts, and the historical context of AI research.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Expert opinions on AI risks\",\n",
+      "            \"Existential risk from superintelligent AI\",\n",
+      "            \"Ethical machines and alignment\",\n",
+      "            \"Open-source AI\",\n",
+      "            \"Regulation of artificial intelligence\",\n",
+      "            \"History of artificial intelligence\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the history, development, and various approaches to artificial intelligence (AI), highlighting key milestones, challenges, and philosophical debates surrounding the field. It covers the evolution from early optimism and funding cuts to the resurgence of interest through expert systems and deep learning, as well as the implications of AI advancements on society.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"History of AI\",\n",
+      "            \"AI winter\",\n",
+      "            \"Expert systems\",\n",
+      "            \"Deep learning\",\n",
+      "            \"Artificial general intelligence (AGI)\",\n",
+      "            \"Philosophy of AI\",\n",
+      "            \"Defining artificial intelligence\",\n",
+      "            \"Evaluating approaches to AI\",\n",
+      "            \"Symbolic AI vs. sub-symbolic AI\",\n",
+      "            \"Narrow AI vs. general AI\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"Artificial intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. It encompasses various subfields including machine learning, natural language processing, and robotics, and aims to create systems that can perform tasks that typically require human intelligence.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Organoid intelligence\",\n",
+      "            \"Robotic process automation\",\n",
+      "            \"Wetware computer\",\n",
+      "            \"DARWIN EU\",\n",
+      "            \"Artificial intelligence in Wikimedia projects\",\n",
+      "            \"AI-generated content on Wikipedia\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"The article discusses the field of artificial intelligence (AI), exploring its various branches, methodologies, and philosophical implications. It highlights the ongoing debates within the AI community regarding the pursuit of general versus narrow AI, the nature of consciousness in machines, and the ethical considerations surrounding AI rights and welfare.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Soft vs. hard computing\",\n",
+      "            \"Narrow vs. general AI\",\n",
+      "            \"Philosophy of artificial intelligence\",\n",
+      "            \"Consciousness\",\n",
+      "            \"Computationalism and functionalism\",\n",
+      "            \"AI welfare and rights\",\n",
+      "            \"Superintelligence and the singularity\",\n",
+      "            \"Transhumanism\",\n",
+      "            \"Artificial intelligence in fiction\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the field of artificial intelligence (AI), covering its definitions, history, methodologies, and applications. It explores various aspects of AI, including machine learning, natural language processing, and robotics, as well as the challenges and ethical considerations associated with AI technologies.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Definitions of AI\",\n",
+      "            \"History of AI\",\n",
+      "            \"Machine Learning\",\n",
+      "            \"Natural Language Processing\",\n",
+      "            \"Robotics\",\n",
+      "            \"Ethical Considerations\",\n",
+      "            \"Applications of AI\",\n",
+      "            \"AI Methodologies\",\n",
+      "            \"Challenges in AI\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the advancements and implications of artificial intelligence (AI), particularly focusing on generative AI and its impact across various sectors including healthcare, finance, entertainment, and environmental concerns.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Generative AI in software development\",\n",
+      "            \"AI in healthcare\",\n",
+      "            \"AI in financial services\",\n",
+      "            \"Impact of AI on Hollywood and entertainment\",\n",
+      "            \"AI and environmental issues\",\n",
+      "            \"AI's role in creativity\",\n",
+      "            \"AI in search technologies\",\n",
+      "            \"AI's energy consumption\",\n",
+      "            \"AI and societal implications\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the concept of artificial intelligence (AI), its development, applications, and the ethical implications surrounding its use. It highlights the advancements in AI technology, including synthetic media and computational capitalism, and addresses concerns regarding misinformation and media manipulation through AI tools.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Definition of Artificial Intelligence\",\n",
+      "            \"Advancements in AI technology\",\n",
+      "            \"Synthetic media and computational capitalism\",\n",
+      "            \"Ethical implications of AI\",\n",
+      "            \"Misinformation and media manipulation\",\n",
+      "            \"AI in surveillance and security\",\n",
+      "            \"AI's impact on employment\",\n",
+      "            \"Global regulatory frameworks for AI\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the concept of artificial intelligence (AI), its applications, advancements, and implications across various fields, including healthcare, programming, and national security. It highlights the evolution of AI technologies, notable achievements, and the ongoing debates surrounding ethical considerations and the future of AI.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Definition of Artificial Intelligence\",\n",
+      "            \"Applications in Healthcare\",\n",
+      "            \"AI Programming Languages\",\n",
+      "            \"Ethical Considerations\",\n",
+      "            \"AI in National Security\",\n",
+      "            \"Generative AI\",\n",
+      "            \"Recent Advancements in AI Technologies\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the field of artificial intelligence (AI), its development, applications, and the ethical considerations surrounding its use. It highlights the advancements in AI technologies, the impact on various sectors, and the ongoing debates regarding the implications of AI on society.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Definition of Artificial Intelligence\",\n",
+      "            \"History and Development of AI\",\n",
+      "            \"Applications of AI\",\n",
+      "            \"Ethical Considerations in AI\",\n",
+      "            \"Impact of AI on Employment\",\n",
+      "            \"Governance and Regulation of AI\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article provides an overview of artificial intelligence (AI), its history, development, and various applications. It discusses the evolution of AI from its inception to its current state, highlighting key milestones and influential figures in the field. The article also addresses the philosophical implications of AI, its impact on society, and the ongoing debates surrounding its future.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"History of AI\",\n",
+      "            \"Key figures in AI development\",\n",
+      "            \"Philosophical implications of AI\",\n",
+      "            \"Applications of AI\",\n",
+      "            \"Current trends in AI\",\n",
+      "            \"Ethical considerations in AI\",\n",
+      "            \"Future of AI\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses various aspects of artificial intelligence (AI), including its implications, challenges, and the need for regulatory frameworks to ensure ethical use. It highlights the perspectives of experts on the responsibilities of tech companies and governments in managing AI technologies.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Ethical implications of AI\",\n",
+      "            \"Regulatory frameworks for AI\",\n",
+      "            \"Transparency in AI systems\",\n",
+      "            \"Compensation for data usage\",\n",
+      "            \"Professional licensing for AI engineers\",\n",
+      "            \"Limitations of natural language processing\",\n",
+      "            \"AI in media and misinformation\",\n",
+      "            \"AI technologies and their reliability\",\n",
+      "            \"Generative AI and its understanding\",\n",
+      "            \"AI applications in various fields\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the field of artificial intelligence (AI), its history, development, and various applications. It highlights the concerns and ethical considerations surrounding AI, as well as the potential impact on society and the economy.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"History of AI\",\n",
+      "            \"Applications of AI\",\n",
+      "            \"Ethical considerations\",\n",
+      "            \"Impact on society\",\n",
+      "            \"Machine learning\",\n",
+      "            \"Regulation of AI\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"The article discusses the field of artificial intelligence (AI), covering its history, development, and various applications. It highlights the advancements in AI technologies, the ethical implications, and the ongoing debates surrounding AI's impact on society.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"History of AI\",\n",
+      "            \"Development of AI technologies\",\n",
+      "            \"Applications of AI\",\n",
+      "            \"Ethical implications of AI\",\n",
+      "            \"Debates on AI's societal impact\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"Artificial intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction. AI applications include expert systems, natural language processing, speech recognition, and machine vision.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Neural networks\",\n",
+      "            \"Deep learning\",\n",
+      "            \"Language models\",\n",
+      "            \"Artificial general intelligence (AGI)\",\n",
+      "            \"Computer vision\",\n",
+      "            \"Speech recognition\",\n",
+      "            \"Natural language processing\",\n",
+      "            \"Robotics\",\n",
+      "            \"Philosophy of artificial intelligence\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"Artificial intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Definition of AI\",\n",
+      "            \"Processes involved in AI\",\n",
+      "            \"Applications of AI\",\n",
+      "            \"Types of AI\",\n",
+      "            \"Ethical considerations in AI\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial Intelligence\",\n",
+      "        \"summary\": \"Artificial intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction. AI applications include expert systems, natural language processing, speech recognition, and machine vision.\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Natural language processing\",\n",
+      "            \"Knowledge representation and reasoning\",\n",
+      "            \"Computer vision\",\n",
+      "            \"Automated planning and scheduling\",\n",
+      "            \"Search methodology\",\n",
+      "            \"Control method\",\n",
+      "            \"Philosophy of artificial intelligence\",\n",
+      "            \"Distributed artificial intelligence\",\n",
+      "            \"Machine learning\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    },\n",
+      "    {\n",
+      "        \"title\": \"Artificial intelligence\",\n",
+      "        \"summary\": \"Artificial intelligence (AI) is intelligence demonstrated by machines, in contrast to the natural intelligence displayed by humans and animals. Leading AI textbooks define the field as the study of \\\"intelligent agents\\\": any device that perceives its environment and takes actions that maximize its chance of successfully achieving its goals. Colloquially, the term \\\"artificial intelligence\\\" is often used to describe machines (or computers) that mimic \\\"cognitive\\\" functions that humans associate with the human mind, such as \\\"learning\\\" and \\\"problem-solving\\\".\",\n",
+      "        \"main_topics\": [\n",
+      "            \"Automation\",\n",
+      "            \"Ethics of technology\",\n",
+      "            \"AI alignment\",\n",
+      "            \"AI safety\",\n",
+      "            \"Technological singularity\",\n",
+      "            \"Machine ethics\",\n",
+      "            \"Existential risk from artificial intelligence\",\n",
+      "            \"Artificial general intelligence\",\n",
+      "            \"AI takeover\",\n",
+      "            \"AI capability control\"\n",
+      "        ],\n",
+      "        \"error\": false\n",
+      "    }\n",
+      "]\n",
+      "\n",
+      "============================================================\n",
+      "✅ Enhanced LLM Demo Complete!\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from crawl4ai import LLMExtractionStrategy, LLMConfig\n",
+    "from pydantic import BaseModel, Field\n",
+    "import os\n",
+    "\n",
+    "# Define extraction schema\n",
+    "class Article(BaseModel):\n",
+    "    title: str = Field(description=\"Article title\")\n",
+    "    summary: str = Field(description=\"Brief summary of the article\")\n",
+    "    main_topics: list[str] = Field(description=\"List of main topics covered\")\n",
+    "\n",
+    "async def demo_enhanced_llm():\n",
+    "    \"\"\"\n",
+    "    Demonstrate enhanced LLM integration with custom temperature\n",
+    "    \"\"\"\n",
+    "    print(\"🤖 Testing Enhanced LLM Integration\\n\")\n",
+    "    print(\"=\" * 60)\n",
+    "    \n",
+    "    # Check for API key\n",
+    "    api_key = os.getenv('OPENAI_API_KEY')\n",
+    "    if not api_key:\n",
+    "        print(\"⚠️ Note: Set OPENAI_API_KEY environment variable to test LLM extraction\")\n",
+    "        print(\"For this demo, we'll show the configuration only.\\n\")\n",
+    "        \n",
+    "        print(\"📝 Example LLM Configuration with new v0.7.5 features:\")\n",
+    "        print(\"\"\"\n",
+    "llm_strategy = LLMExtractionStrategy(\n",
+    "    llm_config=LLMConfig(\n",
+    "        provider=\"openai/gpt-4o-mini\",\n",
+    "        api_token=\"your-api-key\",\n",
+    "        temperature=0.7,  # 🆕 NEW: Control creativity (0.0-2.0)\n",
+    "        base_url=\"custom-endpoint\"  # 🆕 NEW: Custom API endpoint\n",
+    "    ),\n",
+    "    schema=Article.schema(),\n",
+    "    extraction_type=\"schema\",\n",
+    "    instruction=\"Extract article information\"\n",
+    ")\n",
+    "        \"\"\")\n",
+    "        return\n",
+    "    \n",
+    "    # Create LLM extraction strategy with custom temperature\n",
+    "    llm_strategy = LLMExtractionStrategy(\n",
+    "        llm_config=LLMConfig(\n",
+    "            provider=\"openai/gpt-4o-mini\",\n",
+    "            api_token=api_key,\n",
+    "            temperature=0.3,  # 🆕 Lower temperature for more focused extraction\n",
+    "        ),\n",
+    "        schema=Article.schema(),\n",
+    "        extraction_type=\"schema\",\n",
+    "        instruction=\"Extract the article title, a brief summary, and main topics discussed.\"\n",
+    "    )\n",
+    "    \n",
+    "    config = CrawlerRunConfig(\n",
+    "        extraction_strategy=llm_strategy,\n",
+    "        cache_mode=CacheMode.BYPASS\n",
+    "    )\n",
+    "    \n",
+    "    async with AsyncWebCrawler() as crawler:\n",
+    "        result = await crawler.arun(\n",
+    "            url=\"https://en.wikipedia.org/wiki/Artificial_intelligence\",\n",
+    "            config=config\n",
+    "        )\n",
+    "        \n",
+    "        if result.success:\n",
+    "            print(\"\\n✅ LLM Extraction Successful!\")\n",
+    "            print(f\"\\n📄 Extracted Content:\")\n",
+    "            print(result.extracted_content)\n",
+    "        else:\n",
+    "            print(f\"\\n❌ Extraction failed: {result.error_message}\")\n",
+    "    \n",
+    "    print(\"\\n\" + \"=\" * 60)\n",
+    "    print(\"✅ Enhanced LLM Demo Complete!\\n\")\n",
+    "\n",
+    "# Run the demo\n",
+    "await demo_enhanced_llm()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Creating Reusable Hook Functions\n",
+    "\n",
+    "First, let's create some hook functions that we can reuse:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Reusable hook library created!\n",
+      "\n",
+      "📚 Available hooks:\n",
+      "  • block_images_hook - Speed optimization\n",
+      "  • set_viewport_hook - Consistent rendering\n",
+      "  • add_custom_headers_hook - Custom headers\n",
+      "  • scroll_page_hook - Lazy content loading\n",
+      "  • log_page_metrics_hook - Page analytics\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define reusable hooks as Python functions\n",
+    "\n",
+    "async def block_images_hook(page, context, **kwargs):\n",
+    "    \"\"\"\n",
+    "    Performance optimization: Block images to speed up crawling\n",
+    "    \"\"\"\n",
+    "    print(\"[Hook] Blocking images for faster loading...\")\n",
+    "    await context.route(\n",
+    "        \"**/*.{png,jpg,jpeg,gif,webp,svg,ico}\",\n",
+    "        lambda route: route.abort()\n",
+    "    )\n",
+    "    return page\n",
+    "\n",
+    "async def set_viewport_hook(page, context, **kwargs):\n",
+    "    \"\"\"\n",
+    "    Set consistent viewport size for rendering\n",
+    "    \"\"\"\n",
+    "    print(\"[Hook] Setting viewport to 1920x1080...\")\n",
+    "    await page.set_viewport_size({\"width\": 1920, \"height\": 1080})\n",
+    "    return page\n",
+    "\n",
+    "async def add_custom_headers_hook(page, context, url, **kwargs):\n",
+    "    \"\"\"\n",
+    "    Add custom headers before navigation\n",
+    "    \"\"\"\n",
+    "    print(f\"[Hook] Adding custom headers for {url}...\")\n",
+    "    await page.set_extra_http_headers({\n",
+    "        'X-Crawl4AI-Version': '0.7.5',\n",
+    "        'X-Custom-Header': 'docker-hooks-demo',\n",
+    "        'Accept-Language': 'en-US,en;q=0.9'\n",
+    "    })\n",
+    "    return page\n",
+    "\n",
+    "async def scroll_page_hook(page, context, **kwargs):\n",
+    "    \"\"\"\n",
+    "    Scroll page to load lazy-loaded content\n",
+    "    \"\"\"\n",
+    "    print(\"[Hook] Scrolling page to load lazy content...\")\n",
+    "    await page.evaluate(\"window.scrollTo(0, document.body.scrollHeight)\")\n",
+    "    await page.wait_for_timeout(1000)\n",
+    "    await page.evaluate(\"window.scrollTo(0, 0)\")\n",
+    "    await page.wait_for_timeout(500)\n",
+    "    return page\n",
+    "\n",
+    "async def log_page_metrics_hook(page, context, **kwargs):\n",
+    "    \"\"\"\n",
+    "    Log page metrics before extracting HTML\n",
+    "    \"\"\"\n",
+    "    metrics = await page.evaluate('''\n",
+    "        () => ({\n",
+    "            images: document.images.length,\n",
+    "            links: document.links.length,\n",
+    "            scripts: document.scripts.length,\n",
+    "            title: document.title\n",
+    "        })\n",
+    "    ''')\n",
+    "    print(f\"[Hook] Page Metrics - Title: {metrics['title']}\")\n",
+    "    print(f\"        Images: {metrics['images']}, Links: {metrics['links']}, Scripts: {metrics['scripts']}\")\n",
+    "    return page\n",
+    "\n",
+    "print(\"✅ Reusable hook library created!\")\n",
+    "print(\"\\n📚 Available hooks:\")\n",
+    "print(\"  • block_images_hook - Speed optimization\")\n",
+    "print(\"  • set_viewport_hook - Consistent rendering\")\n",
+    "print(\"  • add_custom_headers_hook - Custom headers\")\n",
+    "print(\"  • scroll_page_hook - Lazy content loading\")\n",
+    "print(\"  • log_page_metrics_hook - Page analytics\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Using hooks_to_string() Utility\n",
+    "\n",
+    "The new `hooks_to_string()` utility converts Python function objects to strings that can be sent to the Docker API:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Converted 3 hook functions to string format\n",
+      "\n",
+      "📝 Example of converted hook (first 200 chars):\n",
+      "async def block_images_hook(page, context, **kwargs):\n",
+      "    \"\"\"\n",
+      "    Performance optimization: Block images to speed up crawling\n",
+      "    \"\"\"\n",
+      "    print(\"[Hook] Blocking images for faster loading...\")\n",
+      "    awai...\n",
+      "\n",
+      "💡 Benefits of hooks_to_string():\n",
+      "  ✓ Write hooks as Python functions (IDE support, type checking)\n",
+      "  ✓ Automatically converts to string format for Docker API\n",
+      "  ✓ Reusable across projects\n",
+      "  ✓ Easy to test and debug\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Convert functions to strings using the NEW utility\n",
+    "hooks_as_strings = hooks_to_string({\n",
+    "    \"on_page_context_created\": block_images_hook,\n",
+    "    \"before_goto\": add_custom_headers_hook,\n",
+    "    \"before_retrieve_html\": scroll_page_hook,\n",
+    "})\n",
+    "\n",
+    "print(\"✅ Converted 3 hook functions to string format\")\n",
+    "print(\"\\n📝 Example of converted hook (first 200 chars):\")\n",
+    "print(hooks_as_strings[\"on_page_context_created\"][:200] + \"...\")\n",
+    "\n",
+    "print(\"\\n💡 Benefits of hooks_to_string():\")\n",
+    "print(\"  ✓ Write hooks as Python functions (IDE support, type checking)\")\n",
+    "print(\"  ✓ Automatically converts to string format for Docker API\")\n",
+    "print(\"  ✓ Reusable across projects\")\n",
+    "print(\"  ✓ Easy to test and debug\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 8 Available Hook Points\n",
+    "\n",
+    "The Docker Hooks System provides 8 strategic points where you can inject custom behavior:\n",
+    "\n",
+    "1. **on_browser_created** - Browser initialization\n",
+    "2. **on_page_context_created** - Page context setup\n",
+    "3. **on_user_agent_updated** - User agent configuration\n",
+    "4. **before_goto** - Pre-navigation setup\n",
+    "5. **after_goto** - Post-navigation processing\n",
+    "6. **on_execution_started** - JavaScript execution start\n",
+    "7. **before_retrieve_html** - Pre-extraction processing\n",
+    "8. **before_return_html** - Final HTML processing"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Complete Docker Hooks Demo\n",
+    "\n",
+    "**Note**: For a complete demonstration of all Docker Hooks approaches including:\n",
+    "- String-based hooks with REST API\n",
+    "- hooks_to_string() utility usage\n",
+    "- Docker Client with automatic conversion\n",
+    "- Complete pipeline with all 8 hook points\n",
+    "\n",
+    "See the separate file: **`v0.7.5_docker_hooks_demo.py`**\n",
+    "\n",
+    "This standalone Python script provides comprehensive, runnable examples of the entire Docker Hooks System."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🛠️ Feature 4: Bug Fixes Summary\n",
+    "\n",
+    "### Major Fixes in v0.7.5\n",
+    "\n",
+    "1. **URL Processing** - Fixed '+' sign preservation in query parameters\n",
+    "2. **Proxy Configuration** - Enhanced proxy string parsing (old parameter deprecated)\n",
+    "3. **Docker Error Handling** - Better error messages with status codes\n",
+    "4. **Memory Management** - Fixed leaks in long-running sessions\n",
+    "5. **JWT Authentication** - Fixed Docker JWT validation\n",
+    "6. **Playwright Stealth** - Fixed stealth features\n",
+    "7. **API Configuration** - Fixed config handling\n",
+    "8. **Deep Crawl Strategy** - Resolved JSON encoding errors\n",
+    "9. **LLM Provider Support** - Fixed custom provider integration\n",
+    "10. **Performance** - Resolved backoff strategy failures\n",
+    "\n",
+    "### New Proxy Configuration Example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ New proxy configuration format demonstrated\n",
+      "\n",
+      "📝 Benefits:\n",
+      "  • More explicit and clear\n",
+      "  • Better authentication support\n",
+      "  • Consistent with industry standards\n"
+     ]
+    }
+   ],
+   "source": [
+    "# OLD WAY (Deprecated)\n",
+    "# browser_config = BrowserConfig(proxy=\"http://proxy:8080\")\n",
+    "\n",
+    "# NEW WAY (v0.7.5)\n",
+    "browser_config_with_proxy = BrowserConfig(\n",
+    "    proxy_config={\n",
+    "        \"server\": \"http://proxy.example.com:8080\",\n",
+    "        \"username\": \"optional-username\",  # Optional\n",
+    "        \"password\": \"optional-password\"   # Optional\n",
+    "    }\n",
+    ")\n",
+    "\n",
+    "print(\"✅ New proxy configuration format demonstrated\")\n",
+    "print(\"\\n📝 Benefits:\")\n",
+    "print(\"  • More explicit and clear\")\n",
+    "print(\"  • Better authentication support\")\n",
+    "print(\"  • Consistent with industry standards\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🎯 Complete Example: Combining Multiple Features\n",
+    "\n",
+    "Let's create a real-world example that uses multiple v0.7.5 features together:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🎯 Complete v0.7.5 Feature Demo\n",
+      "\n",
+      "============================================================\n",
+      "\n",
+      "1️⃣ Using Docker Hooks System (NEW!)\n",
+      "   ✓ Converted 3 hooks to string format\n",
+      "   ✓ Ready to send to Docker API\n",
+      "\n",
+      "2️⃣ Enabling HTTPS Preservation\n",
+      "   ✓ HTTPS preservation enabled\n",
+      "\n",
+      "3️⃣ Using New Proxy Configuration Format\n",
+      "   ✓ New proxy config format ready\n",
+      "\n",
+      "4️⃣ Executing Crawl with All Features\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [INIT].... → Crawl4AI 0.7.5 \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m[\u001b[0m\u001b[36mINIT\u001b[0m\u001b[1;36m]\u001b[0m\u001b[36m...\u001b[0m\u001b[36m. → Crawl4AI \u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[36m.\u001b[0m\u001b[1;36m5\u001b[0m\u001b[36m \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [FETCH]... ↓ https://example.com                                                                                  |\n",
+       "✓ | ⏱: 1.29s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://example.com\u001b[0m\u001b[32m                                                                                  |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m29s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [SCRAPE].. ◆ https://example.com                                                                                  |\n",
+       "✓ | ⏱: 0.00s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://example.com\u001b[0m\u001b[32m                                                                                  |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m00s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
    [COMPLETE] ● https://example.com                                                                                  |\n",
+       "✓ | ⏱: 1.29s \n",
+       "
    \n"
+      ],
+      "text/plain": [
+       "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://example.com\u001b[0m\u001b[32m                                                                                  |\u001b[0m\n",
+       "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m29s \u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   ✓ Crawl successful!\n",
+      "\n",
+      "📊 Results:\n",
+      "   • Pages crawled: 1\n",
+      "   • Title: Example Domain\n",
+      "   • Content length: 119 characters\n",
+      "   • Links found: 0\n",
+      "\n",
+      "============================================================\n",
+      "✅ Complete Feature Demo Finished!\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "async def complete_demo():\n",
+    "    \"\"\"\n",
+    "    Comprehensive demo combining multiple v0.7.5 features\n",
+    "    \"\"\"\n",
+    "    print(\"🎯 Complete v0.7.5 Feature Demo\\n\")\n",
+    "    print(\"=\" * 60)\n",
+    "    \n",
+    "    # Use function-based hooks (NEW Docker Hooks System)\n",
+    "    print(\"\\n1️⃣ Using Docker Hooks System (NEW!)\")\n",
+    "    hooks = {\n",
+    "        \"on_page_context_created\": set_viewport_hook,\n",
+    "        \"before_goto\": add_custom_headers_hook,\n",
+    "        \"before_retrieve_html\": log_page_metrics_hook\n",
+    "    }\n",
+    "    \n",
+    "    # Convert to strings using the NEW utility\n",
+    "    hooks_strings = hooks_to_string(hooks)\n",
+    "    print(f\"   ✓ Converted {len(hooks_strings)} hooks to string format\")\n",
+    "    print(\"   ✓ Ready to send to Docker API\")\n",
+    "    \n",
+    "    # Use HTTPS preservation\n",
+    "    print(\"\\n2️⃣ Enabling HTTPS Preservation\")\n",
+    "    url_filter = URLPatternFilter(\n",
+    "        patterns=[r\"^(https:\\/\\/)?example\\.com(\\/.*)?$\"]\n",
+    "    )\n",
+    "    \n",
+    "    config = CrawlerRunConfig(\n",
+    "        exclude_external_links=True,\n",
+    "        preserve_https_for_internal_links=True,  # v0.7.5 feature\n",
+    "        cache_mode=CacheMode.BYPASS,\n",
+    "        deep_crawl_strategy=BFSDeepCrawlStrategy(\n",
+    "            max_depth=1,\n",
+    "            max_pages=3,\n",
+    "            filter_chain=FilterChain([url_filter])\n",
+    "        )\n",
+    "    )\n",
+    "    print(\"   ✓ HTTPS preservation enabled\")\n",
+    "    \n",
+    "    # Use new proxy config format\n",
+    "    print(\"\\n3️⃣ Using New Proxy Configuration Format\")\n",
+    "    browser_config = BrowserConfig(\n",
+    "        headless=True,\n",
+    "        # proxy_config={  # Uncomment if you have a proxy\n",
+    "        #     \"server\": \"http://proxy:8080\"\n",
+    "        # }\n",
+    "    )\n",
+    "    print(\"   ✓ New proxy config format ready\")\n",
+    "    \n",
+    "    # Run the crawl\n",
+    "    print(\"\\n4️⃣ Executing Crawl with All Features\")\n",
+    "    async with AsyncWebCrawler(config=browser_config) as crawler:\n",
+    "        # With deep_crawl_strategy, returns a list\n",
+    "        results = await crawler.arun(\n",
+    "            url=\"https://example.com\",\n",
+    "            config=config\n",
+    "        )\n",
+    "        \n",
+    "        if results and len(results) > 0:\n",
+    "            result = results[0]  # Get first result\n",
+    "            print(\"   ✓ Crawl successful!\")\n",
+    "            print(f\"\\n📊 Results:\")\n",
+    "            print(f\"   • Pages crawled: {len(results)}\")\n",
+    "            print(f\"   • Title: {result.metadata.get('title', 'N/A')}\")\n",
+    "            print(f\"   • Content length: {len(result.markdown.raw_markdown)} characters\")\n",
+    "            print(f\"   • Links found: {len(result.links['internal']) + len(result.links['external'])}\")\n",
+    "        else:\n",
+    "            print(f\"   ⚠️ No results returned\")\n",
+    "    \n",
+    "    print(\"\\n\" + \"=\" * 60)\n",
+    "    print(\"✅ Complete Feature Demo Finished!\\n\")\n",
+    "\n",
+    "# Run complete demo\n",
+    "await complete_demo()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## 🎓 Summary\n",
+    "\n",
+    "### What We Covered\n",
+    "\n",
+    "✅ **HTTPS Preservation** - Maintain secure protocols throughout crawling  \n",
+    "✅ **Enhanced LLM Integration** - Custom temperature and provider configuration  \n",
+    "✅ **Docker Hooks System (NEW!)** - Complete pipeline customization with 3 approaches  \n",
+    "✅ **hooks_to_string() Utility (NEW!)** - Convert functions for Docker API  \n",
+    "✅ **Bug Fixes** - New proxy config and multiple improvements  \n",
+    "\n",
+    "### Key Highlight: Docker Hooks System 🌟\n",
+    "\n",
+    "The **Docker Hooks System** is completely NEW in v0.7.5. It offers:\n",
+    "- 8 strategic hook points in the pipeline\n",
+    "- 3 ways to use hooks (strings, utility, auto-conversion)\n",
+    "- Full control over crawling behavior\n",
+    "- Support for authentication, optimization, and custom processing\n",
+    "\n",
+    "### Next Steps\n",
+    "\n",
+    "1. **Docker Hooks Demo** - See `v0.7.5_docker_hooks_demo.py` for complete Docker Hooks examples\n",
+    "2. **Documentation** - Visit [docs.crawl4ai.com](https://docs.crawl4ai.com) for full reference\n",
+    "3. **Examples** - Check [GitHub examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples)\n",
+    "4. **Community** - Join [Discord](https://discord.gg/jP8KfhDhyN) for support\n",
+    "\n",
+    "---\n",
+    "\n",
+    "## 📚 Resources\n",
+    "\n",
+    "- 📖 [Full Documentation](https://docs.crawl4ai.com)\n",
+    "- 🐙 [GitHub Repository](https://github.com/unclecode/crawl4ai)\n",
+    "- 📝 [Release Notes](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.5.md)\n",
+    "- 💬 [Discord Community](https://discord.gg/jP8KfhDhyN)\n",
+    "- 🐦 [Twitter](https://x.com/unclecode)\n",
+    "\n",
+    "---\n",
+    "\n",
+    "**Happy Crawling with v0.7.5! 🚀**"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/docs/releases_review/v0_4_24_walkthrough.py b/docs/releases_review/v0_4_24_walkthrough.py
new file mode 100644
index 0000000..996e7b0
--- /dev/null
+++ b/docs/releases_review/v0_4_24_walkthrough.py
@@ -0,0 +1,464 @@
+"""
+Crawl4AI v0.4.24 Feature Walkthrough
+===================================
+
+This script demonstrates the new features introduced in Crawl4AI v0.4.24.
+Each section includes detailed examples and explanations of the new capabilities.
+"""
+
+import asyncio
+import os
+import json
+import re
+from typing import List
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    LLMExtractionStrategy,
+    JsonCssExtractionStrategy,
+)
+from crawl4ai.content_filter_strategy import RelevantContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from bs4 import BeautifulSoup
+
+# Sample HTML for demonstrations
+SAMPLE_HTML = """
+
    
+    
+    
+
    
+"""
+
+
+async def demo_ssl_features():
+    """
+    Enhanced SSL & Security Features Demo
+    -----------------------------------
+
+    This example demonstrates the new SSL certificate handling and security features:
+    1. Custom certificate paths
+    2. SSL verification options
+    3. HTTPS error handling
+    4. Certificate validation configurations
+
+    These features are particularly useful when:
+    - Working with self-signed certificates
+    - Dealing with corporate proxies
+    - Handling mixed content websites
+    - Managing different SSL security levels
+    """
+    print("\n1. Enhanced SSL & Security Demo")
+    print("--------------------------------")
+
+    browser_config = BrowserConfig()
+
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        fetch_ssl_certificate=True,  # Enable SSL certificate fetching
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com", config=run_config)
+        print(f"SSL Crawl Success: {result.success}")
+        result.ssl_certificate.to_json(
+            os.path.join(os.getcwd(), "ssl_certificate.json")
+        )
+        if not result.success:
+            print(f"SSL Error: {result.error_message}")
+
+
+async def demo_content_filtering():
+    """
+    Smart Content Filtering Demo
+    ----------------------
+
+    Demonstrates advanced content filtering capabilities:
+    1. Custom filter to identify and extract specific content
+    2. Integration with markdown generation
+    3. Flexible pruning rules
+    """
+    print("\n2. Smart Content Filtering Demo")
+    print("--------------------------------")
+
+    # Create a custom content filter
+    class CustomNewsFilter(RelevantContentFilter):
+        def __init__(self):
+            super().__init__()
+            # Add news-specific patterns
+            self.negative_patterns = re.compile(
+                r"nav|footer|header|sidebar|ads|comment|share|related|recommended|popular|trending",
+                re.I,
+            )
+            self.min_word_count = 30  # Higher threshold for news content
+
+        def filter_content(
+            self, html: str, min_word_threshold: int = None
+        ) -> List[str]:
+            """
+            Implements news-specific content filtering logic.
+
+            Args:
+                html (str): HTML content to be filtered
+                min_word_threshold (int, optional): Minimum word count threshold
+
+            Returns:
+                List[str]: List of filtered HTML content blocks
+            """
+            if not html or not isinstance(html, str):
+                return []
+
+            soup = BeautifulSoup(html, "lxml")
+            if not soup.body:
+                soup = BeautifulSoup(f"{html}", "lxml")
+
+            body = soup.find("body")
+
+            # Extract chunks with metadata
+            chunks = self.extract_text_chunks(
+                body, min_word_threshold or self.min_word_count
+            )
+
+            # Filter chunks based on news-specific criteria
+            filtered_chunks = []
+            for _, text, tag_type, element in chunks:
+                # Skip if element has negative class/id
+                if self.is_excluded(element):
+                    continue
+
+                # Headers are important in news articles
+                if tag_type == "header":
+                    filtered_chunks.append(self.clean_element(element))
+                    continue
+
+                # For content, check word count and link density
+                text = element.get_text(strip=True)
+                if len(text.split()) >= (min_word_threshold or self.min_word_count):
+                    # Calculate link density
+                    links_text = " ".join(
+                        a.get_text(strip=True) for a in element.find_all("a")
+                    )
+                    link_density = len(links_text) / len(text) if text else 1
+
+                    # Accept if link density is reasonable
+                    if link_density < 0.5:
+                        filtered_chunks.append(self.clean_element(element))
+
+            return filtered_chunks
+
+    # Create markdown generator with custom filter
+    markdown_gen = DefaultMarkdownGenerator(content_filter=CustomNewsFilter())
+
+    run_config = CrawlerRunConfig(
+        markdown_generator=markdown_gen, cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://news.ycombinator.com", config=run_config
+        )
+        print("Filtered Content Sample:")
+        print(result.markdown[:500])  # Show first 500 chars
+
+
+async def demo_json_extraction():
+    """
+    Improved JSON Extraction Demo
+    ---------------------------
+
+    Demonstrates the enhanced JSON extraction capabilities:
+    1. Base element attributes extraction
+    2. Complex nested structures
+    3. Multiple extraction patterns
+
+    Key features shown:
+    - Extracting attributes from base elements (href, data-* attributes)
+    - Processing repeated patterns
+    - Handling optional fields
+    """
+    print("\n3. Improved JSON Extraction Demo")
+    print("--------------------------------")
+
+    # Define the extraction schema with base element attributes
+    json_strategy = JsonCssExtractionStrategy(
+        schema={
+            "name": "Blog Posts",
+            "baseSelector": "div.article-list",
+            "baseFields": [
+                {"name": "list_id", "type": "attribute", "attribute": "data-list-id"},
+                {"name": "category", "type": "attribute", "attribute": "data-category"},
+            ],
+            "fields": [
+                {
+                    "name": "posts",
+                    "selector": "article.post",
+                    "type": "nested_list",
+                    "baseFields": [
+                        {
+                            "name": "post_id",
+                            "type": "attribute",
+                            "attribute": "data-post-id",
+                        },
+                        {
+                            "name": "author_id",
+                            "type": "attribute",
+                            "attribute": "data-author",
+                        },
+                    ],
+                    "fields": [
+                        {
+                            "name": "title",
+                            "selector": "h2.title a",
+                            "type": "text",
+                            "baseFields": [
+                                {
+                                    "name": "url",
+                                    "type": "attribute",
+                                    "attribute": "href",
+                                }
+                            ],
+                        },
+                        {
+                            "name": "author",
+                            "selector": "div.meta a.author",
+                            "type": "text",
+                            "baseFields": [
+                                {
+                                    "name": "profile_url",
+                                    "type": "attribute",
+                                    "attribute": "href",
+                                }
+                            ],
+                        },
+                        {"name": "date", "selector": "span.date", "type": "text"},
+                        {
+                            "name": "read_more",
+                            "selector": "a.read-more",
+                            "type": "nested",
+                            "fields": [
+                                {"name": "text", "type": "text"},
+                                {
+                                    "name": "url",
+                                    "type": "attribute",
+                                    "attribute": "href",
+                                },
+                            ],
+                        },
+                    ],
+                }
+            ],
+        }
+    )
+
+    # Demonstrate extraction from raw HTML
+    run_config = CrawlerRunConfig(
+        extraction_strategy=json_strategy, cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="raw:" + SAMPLE_HTML,  # Use raw: prefix for raw HTML
+            config=run_config,
+        )
+        print("Extracted Content:")
+        print(result.extracted_content)
+
+
+async def demo_input_formats():
+    """
+    Input Format Handling Demo
+    ----------------------
+
+    Demonstrates how LLM extraction can work with different input formats:
+    1. Markdown (default) - Good for simple text extraction
+    2. HTML - Better when you need structure and attributes
+
+    This example shows how HTML input can be beneficial when:
+    - You need to understand the DOM structure
+    - You want to extract both visible text and HTML attributes
+    - The content has complex layouts like tables or forms
+    """
+    print("\n4. Input Format Handling Demo")
+    print("---------------------------")
+
+    # Create a dummy HTML with rich structure
+    dummy_html = """
+    
    
+        
    
+            
    Senior AI/ML Engineer
    
+            
    
+                AI Research Division
+                San Francisco (Hybrid)
+            
    
+            
    
+                $150,000 - $220,000
+                per year
+            
    
+        
    
+        
+        
    
+            
    
+                
    Technical Requirements
    
+                
      
+                    
    • 
+                        5+ years experience in Machine Learning
+                    
      • 
+                    
    • 
+                        Proficiency in Python and PyTorch/TensorFlow
+                    
      • 
+                    
    • 
+                        Experience with distributed training systems
+                    
      • 
+                
    
+            
    
+            
+            
    
+                
    Professional Skills
    
+                
      
+                    
    • 
+                        Strong problem-solving abilities
+                    
      • 
+                    
    • 
+                        Experience leading technical teams
+                    
      • 
+                
    
+            
    
+        
    
+        
+        
    
+            
+        
    
+        
+        
    
+            
    
+                
    Hiring Manager
    
+                
    
+                    Dr. Sarah Chen
+                    Director of AI Research
+                    
+                
    
+            
    
+            
    
+                
    Join our team of 50+ researchers working on cutting-edge AI applications
    
+            
    
+        
    
+    
    
+    """
+
+    # Use raw:// prefix to pass HTML content directly
+    url = f"raw://{dummy_html}"
+
+    from pydantic import BaseModel, Field
+    from typing import List, Optional
+
+    # Define our schema using Pydantic
+    class JobRequirement(BaseModel):
+        category: str = Field(
+            description="Category of the requirement (e.g., Technical, Soft Skills)"
+        )
+        items: List[str] = Field(
+            description="List of specific requirements in this category"
+        )
+        priority: str = Field(
+            description="Priority level (Required/Preferred) based on the HTML class or context"
+        )
+
+    class JobPosting(BaseModel):
+        title: str = Field(description="Job title")
+        department: str = Field(description="Department or team")
+        location: str = Field(description="Job location, including remote options")
+        salary_range: Optional[str] = Field(description="Salary range if specified")
+        requirements: List[JobRequirement] = Field(
+            description="Categorized job requirements"
+        )
+        application_deadline: Optional[str] = Field(
+            description="Application deadline if specified"
+        )
+        contact_info: Optional[dict] = Field(
+            description="Contact information from footer or contact section"
+        )
+
+    # First try with markdown (default)
+    markdown_strategy = LLMExtractionStrategy(
+        provider="openai/gpt-4o",
+        api_token=os.getenv("OPENAI_API_KEY"),
+        schema=JobPosting.model_json_schema(),
+        extraction_type="schema",
+        instruction="""
+        Extract job posting details into structured data. Focus on the visible text content 
+        and organize requirements into categories.
+        """,
+        input_format="markdown",  # default
+    )
+
+    # Then with HTML for better structure understanding
+    html_strategy = LLMExtractionStrategy(
+        provider="openai/gpt-4",
+        api_token=os.getenv("OPENAI_API_KEY"),
+        schema=JobPosting.model_json_schema(),
+        extraction_type="schema",
+        instruction="""
+        Extract job posting details, using HTML structure to:
+        1. Identify requirement priorities from CSS classes (e.g., 'required' vs 'preferred')
+        2. Extract contact info from the page footer or dedicated contact section
+        3. Parse salary information from specially formatted elements
+        4. Determine application deadline from timestamp or date elements
+        
+        Use HTML attributes and classes to enhance extraction accuracy.
+        """,
+        input_format="html",  # explicitly use HTML
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        # Try with markdown first
+        markdown_config = CrawlerRunConfig(extraction_strategy=markdown_strategy)
+        markdown_result = await crawler.arun(url=url, config=markdown_config)
+        print("\nMarkdown-based Extraction Result:")
+        items = json.loads(markdown_result.extracted_content)
+        print(json.dumps(items, indent=2))
+
+        # Then with HTML for better structure understanding
+        html_config = CrawlerRunConfig(extraction_strategy=html_strategy)
+        html_result = await crawler.arun(url=url, config=html_config)
+        print("\nHTML-based Extraction Result:")
+        items = json.loads(html_result.extracted_content)
+        print(json.dumps(items, indent=2))
+
+
+# Main execution
+async def main():
+    print("Crawl4AI v0.4.24 Feature Walkthrough")
+    print("====================================")
+
+    # Run all demos
+    await demo_ssl_features()
+    await demo_content_filtering()
+    await demo_json_extraction()
+    # await demo_input_formats()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/releases_review/v0_4_3b2_features_demo.py b/docs/releases_review/v0_4_3b2_features_demo.py
new file mode 100644
index 0000000..3786278
--- /dev/null
+++ b/docs/releases_review/v0_4_3b2_features_demo.py
@@ -0,0 +1,349 @@
+"""
+Crawl4ai v0.4.3b2 Features Demo
+============================
+
+This demonstration showcases three major categories of new features in Crawl4ai v0.4.3:
+
+1. Efficiency & Speed:
+   - Memory-efficient dispatcher strategies
+   - New scraping algorithm
+   - Streaming support for batch crawling
+
+2. LLM Integration:
+   - Automatic schema generation
+   - LLM-powered content filtering
+   - Smart markdown generation
+
+3. Core Improvements:
+   - Robots.txt compliance
+   - Proxy rotation
+   - Enhanced URL handling
+   - Shared data among hooks
+   - add page routes
+
+Each demo function can be run independently or as part of the full suite.
+"""
+
+import asyncio
+import os
+import json
+import re
+import random
+from typing import Optional, Dict
+from dotenv import load_dotenv
+from crawl4ai import (
+    AsyncWebCrawler, 
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    DisplayMode,
+    MemoryAdaptiveDispatcher,
+    CrawlerMonitor,
+    DefaultMarkdownGenerator,
+    LXMLWebScrapingStrategy,
+    JsonCssExtractionStrategy,
+    LLMContentFilter
+)
+
+load_dotenv()
+
+async def demo_memory_dispatcher():
+    """Demonstrates the new memory-efficient dispatcher system.
+    
+    Key Features:
+    - Adaptive memory management
+    - Real-time performance monitoring
+    - Concurrent session control
+    """
+    print("\n=== Memory Dispatcher Demo ===")
+    
+    try:
+        # Configuration
+        browser_config = BrowserConfig(headless=True, verbose=False)
+        crawler_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator()
+        )
+
+        # Test URLs
+        urls = ["http://example.com", "http://example.org", "http://example.net"] * 3
+
+        print("\n📈 Initializing crawler with memory monitoring...")
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            monitor = CrawlerMonitor(
+                max_visible_rows=10,
+                display_mode=DisplayMode.DETAILED
+            )
+            
+            dispatcher = MemoryAdaptiveDispatcher(
+                memory_threshold_percent=80.0,
+                check_interval=0.5,
+                max_session_permit=5,
+                monitor=monitor
+            )
+            
+            print("\n🚀 Starting batch crawl...")
+            results = await crawler.arun_many(
+                urls=urls,
+                config=crawler_config,
+                dispatcher=dispatcher
+            )
+            print(f"\n✅ Completed {len(results)} URLs successfully")
+            
+    except Exception as e:
+        print(f"\n❌ Error in memory dispatcher demo: {str(e)}")
+
+async def demo_streaming_support():
+    """
+    2. Streaming Support Demo
+    ======================
+    Shows how to process URLs as they complete using streaming
+    """
+    print("\n=== 2. Streaming Support Demo ===")
+
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, stream=True)
+
+    # Test URLs
+    urls = ["http://example.com", "http://example.org", "http://example.net"] * 2
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Initialize dispatcher for streaming
+        dispatcher = MemoryAdaptiveDispatcher(max_session_permit=3, check_interval=0.5)
+
+        print("Starting streaming crawl...")
+        async for result in await crawler.arun_many(
+            urls=urls,
+            config=crawler_config,
+            dispatcher=dispatcher
+        ):
+            # Process each result as it arrives
+            print(
+                f"Received result for {result.url} - Success: {result.success}"
+            )
+            if result.success:
+                print(f"Content length: {len(result.markdown)}")
+
+async def demo_content_scraping():
+    """
+    3. Content Scraping Strategy Demo
+    ==============================
+    Demonstrates the new LXMLWebScrapingStrategy for faster content scraping.
+    """
+    print("\n=== 3. Content Scraping Strategy Demo ===")
+
+    crawler = AsyncWebCrawler()
+    url = "https://example.com/article"
+
+    # Configure with the new LXML strategy
+    config = CrawlerRunConfig(
+        scraping_strategy=LXMLWebScrapingStrategy(), 
+        verbose=True
+    )
+
+    print("Scraping content with LXML strategy...")
+    async with crawler:
+        result = await crawler.arun(url, config=config)
+        if result.success:
+            print("Successfully scraped content using LXML strategy")
+
+async def demo_llm_markdown():
+    """
+    4. LLM-Powered Markdown Generation Demo
+    ===================================
+    Shows how to use the new LLM-powered content filtering and markdown generation.
+    """
+    print("\n=== 4. LLM-Powered Markdown Generation Demo ===")
+
+    crawler = AsyncWebCrawler()
+    url = "https://docs.python.org/3/tutorial/classes.html"
+
+    content_filter = LLMContentFilter(
+        provider="openai/gpt-4o",
+        api_token=os.getenv("OPENAI_API_KEY"),
+        instruction="""
+        Focus on extracting the core educational content about Python classes.
+        Include:
+        - Key concepts and their explanations
+        - Important code examples
+        - Essential technical details
+        Exclude:
+        - Navigation elements
+        - Sidebars
+        - Footer content
+        - Version information
+        - Any non-essential UI elements
+        
+        Format the output as clean markdown with proper code blocks and headers.
+        """,
+        verbose=True,
+    )
+
+    # Configure LLM-powered markdown generation
+    config = CrawlerRunConfig(
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=content_filter
+        ), 
+        cache_mode = CacheMode.BYPASS,
+        verbose=True
+    )
+
+    print("Generating focused markdown with LLM...")
+    async with crawler:
+        result = await crawler.arun(url, config=config)
+        if result.success and result.markdown_v2:
+            print("Successfully generated LLM-filtered markdown")
+            print("First 500 chars of filtered content:")
+            print(result.markdown_v2.fit_markdown[:500])
+            print("Successfully generated LLM-filtered markdown")
+
+async def demo_robots_compliance():
+    """
+    5. Robots.txt Compliance Demo
+    ==========================
+    Demonstrates the new robots.txt compliance feature with SQLite caching.
+    """
+    print("\n=== 5. Robots.txt Compliance Demo ===")
+
+    crawler = AsyncWebCrawler()
+    urls = ["https://example.com", "https://facebook.com", "https://twitter.com"]
+
+    # Enable robots.txt checking
+    config = CrawlerRunConfig(check_robots_txt=True, verbose=True)
+
+    print("Crawling with robots.txt compliance...")
+    async with crawler:
+        results = await crawler.arun_many(urls, config=config)
+        for result in results:
+            if result.status_code == 403:
+                print(f"Access blocked by robots.txt: {result.url}")
+            elif result.success:
+                print(f"Successfully crawled: {result.url}")
+
+async def demo_json_schema_generation():
+    """
+    7. LLM-Powered Schema Generation Demo
+    =================================
+    Demonstrates automatic CSS and XPath schema generation using LLM models.
+    """
+    print("\n=== 7. LLM-Powered Schema Generation Demo ===")
+
+    # Example HTML content for a job listing
+    html_content = """
+    
    
+        
    Senior Software Engineer
    
+        
    
+            San Francisco, CA
+            $150,000 - $200,000
+            
    
+                
    Requirements
    
+                
      
+                    
    • 5+ years Python experience
      • 
+                    
    • Strong background in web crawling
      • 
+                
    
+            
    
+        
    
+    
    
+    """
+
+    print("Generating CSS selectors schema...")
+    # Generate CSS selectors with a specific query
+    css_schema = JsonCssExtractionStrategy.generate_schema(
+        html_content,
+        schema_type="CSS",
+        query="Extract job title, location, and salary information",
+        provider="openai/gpt-4o",  # or use other providers like "ollama"
+    )
+    print("\nGenerated CSS Schema:")
+    print(css_schema)
+
+    # Example of using the generated schema with crawler
+    crawler = AsyncWebCrawler()
+    url = "https://example.com/job-listing"
+
+    # Create an extraction strategy with the generated schema
+    extraction_strategy = JsonCssExtractionStrategy(schema=css_schema)
+
+    config = CrawlerRunConfig(extraction_strategy=extraction_strategy, verbose=True)
+
+    print("\nTesting generated schema with crawler...")
+    async with crawler:
+        result = await crawler.arun(url, config=config)
+        if result.success:
+            print(json.dumps(result.extracted_content, indent=2) if result.extracted_content else None)
+            print("Successfully used generated schema for crawling")
+
+async def demo_proxy_rotation():
+    """
+    8. Proxy Rotation Demo
+    ===================
+    Demonstrates how to rotate proxies for each request using Crawl4ai.
+    """
+    print("\n=== 8. Proxy Rotation Demo ===")
+
+    async def get_next_proxy(proxy_file: str = "proxies.txt") -> Optional[Dict]:
+        """Get next proxy from local file"""
+        try:
+            proxies = os.getenv("PROXIES", "").split(",")
+                
+            ip, port, username, password = random.choice(proxies).split(":")
+            return {
+                "server": f"http://{ip}:{port}",
+                "username": username,
+                "password": password,
+                "ip": ip  # Store original IP for verification
+            }
+        except Exception as e:
+            print(f"Error loading proxy: {e}")
+            return None    
+    
+    # Create 10 test requests to httpbin
+    urls = ["https://httpbin.org/ip"] * 2
+    
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        for url in urls:
+            proxy = await get_next_proxy()
+            if not proxy:
+                print("No proxy available, skipping...")
+                continue
+                
+            # Create new config with proxy
+            current_config = run_config.clone(proxy_config=proxy, user_agent="")
+            result = await crawler.arun(url=url, config=current_config)
+            
+            if result.success:
+                ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
+                print(f"Proxy {proxy['ip']} -> Response IP: {ip_match.group(0) if ip_match else 'Not found'}")
+                verified = ip_match.group(0) == proxy['ip']
+                if verified:
+                    print(f"✅ Proxy working! IP matches: {proxy['ip']}")
+                else:
+                    print("❌ Proxy failed or IP mismatch!")
+            else:
+                print(f"Failed with proxy {proxy['ip']}")
+
+async def main():
+    """Run all feature demonstrations."""
+    print("\n📊 Running Crawl4ai v0.4.3 Feature Demos\n")
+    
+    # Efficiency & Speed Demos
+    print("\n🚀 EFFICIENCY & SPEED DEMOS")
+    await demo_memory_dispatcher()
+    await demo_streaming_support()
+    await demo_content_scraping()
+    
+    # # LLM Integration Demos
+    print("\n🤖 LLM INTEGRATION DEMOS")
+    await demo_json_schema_generation()
+    await demo_llm_markdown()
+    
+    # # Core Improvements
+    print("\n🔧 CORE IMPROVEMENT DEMOS")
+    await demo_robots_compliance()
+    await demo_proxy_rotation()
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/docs/releases_review/v0_7_0_features_demo.py b/docs/releases_review/v0_7_0_features_demo.py
new file mode 100644
index 0000000..2f803a3
--- /dev/null
+++ b/docs/releases_review/v0_7_0_features_demo.py
@@ -0,0 +1,280 @@
+"""
+🚀 Crawl4AI v0.7.0 Feature Demo
+================================
+This file demonstrates the major features introduced in v0.7.0 with practical examples.
+"""
+
+import asyncio
+import json
+from pathlib import Path
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    BrowserConfig,
+    CacheMode,
+    # New imports for v0.7.0
+    VirtualScrollConfig,
+    LinkPreviewConfig,
+    AdaptiveCrawler,
+    AdaptiveConfig,
+    AsyncUrlSeeder,
+    SeedingConfig,
+    c4a_compile,
+)
+
+
+async def demo_link_preview():
+    """
+    Demo 1: Link Preview with 3-Layer Scoring
+    
+    Shows how to analyze links with intrinsic quality scores,
+    contextual relevance, and combined total scores.
+    """
+    print("\n" + "="*60)
+    print("🔗 DEMO 1: Link Preview & Intelligent Scoring")
+    print("="*60)
+    
+    # Configure link preview with contextual scoring
+    config = CrawlerRunConfig(
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,
+            include_external=False,
+            max_links=10,
+            concurrency=5,
+            query="machine learning tutorials",  # For contextual scoring
+            score_threshold=0.3,  # Minimum relevance
+            verbose=True
+        ),
+        score_links=True,  # Enable intrinsic scoring
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://scikit-learn.org/stable/", config=config)
+        
+        if result.success:
+            # Get scored links
+            internal_links = result.links.get("internal", [])
+            scored_links = [l for l in internal_links if l.get("total_score")]
+            scored_links.sort(key=lambda x: x.get("total_score", 0), reverse=True)
+            
+            print(f"\nTop 5 Most Relevant Links:")
+            for i, link in enumerate(scored_links[:5], 1):
+                print(f"\n{i}. {link.get('text', 'No text')[:50]}...")
+                print(f"   URL: {link['href']}")
+                print(f"   Intrinsic Score: {link.get('intrinsic_score', 0):.2f}/10")
+                print(f"   Contextual Score: {link.get('contextual_score', 0):.3f}")
+                print(f"   Total Score: {link.get('total_score', 0):.3f}")
+                
+                # Show metadata if available
+                if link.get('head_data'):
+                    title = link['head_data'].get('title', 'No title')
+                    print(f"   Title: {title[:60]}...")
+
+
+async def demo_adaptive_crawling():
+    """
+    Demo 2: Adaptive Crawling
+    
+    Shows intelligent crawling that stops when enough information
+    is gathered, with confidence tracking.
+    """
+    print("\n" + "="*60)
+    print("🎯 DEMO 2: Adaptive Crawling with Confidence Tracking")
+    print("="*60)
+    
+    # Configure adaptive crawler
+    config = AdaptiveConfig(
+        strategy="statistical",  # or "embedding" for semantic understanding
+        max_pages=10,
+        confidence_threshold=0.7,  # Stop at 70% confidence
+        top_k_links=3,  # Follow top 3 links per page
+        min_gain_threshold=0.05  # Need 5% information gain to continue
+    )
+    
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        adaptive = AdaptiveCrawler(crawler, config)
+        
+        print("Starting adaptive crawl about Python decorators...")
+        result = await adaptive.digest(
+            start_url="https://docs.python.org/3/glossary.html",
+            query="python decorators functions wrapping"
+        )
+        
+        print(f"\n✅ Crawling Complete!")
+        print(f"• Confidence Level: {adaptive.confidence:.0%}")
+        print(f"• Pages Crawled: {len(result.crawled_urls)}")
+        print(f"• Knowledge Base: {len(adaptive.state.knowledge_base)} documents")
+        
+        # Get most relevant content
+        relevant = adaptive.get_relevant_content(top_k=3)
+        print(f"\nMost Relevant Pages:")
+        for i, page in enumerate(relevant, 1):
+            print(f"{i}. {page['url']} (relevance: {page['score']:.2%})")
+
+
+async def demo_virtual_scroll():
+    """
+    Demo 3: Virtual Scroll for Modern Web Pages
+    
+    Shows how to capture content from pages with DOM recycling
+    (Twitter, Instagram, infinite scroll).
+    """
+    print("\n" + "="*60)
+    print("📜 DEMO 3: Virtual Scroll Support")
+    print("="*60)
+    
+    # Configure virtual scroll for a news site
+    virtual_config = VirtualScrollConfig(
+        container_selector="main, article, .content",  # Common containers
+        scroll_count=20,  # Scroll up to 20 times
+        scroll_by="container_height",  # Scroll by container height
+        wait_after_scroll=0.5  # Wait 500ms after each scroll
+    )
+    
+    config = CrawlerRunConfig(
+        virtual_scroll_config=virtual_config,
+        cache_mode=CacheMode.BYPASS,
+        wait_for="css:article"  # Wait for articles to load
+    )
+    
+    # Example with a real news site
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            "https://news.ycombinator.com/",
+            config=config
+        )
+        
+        if result.success:
+            # Count items captured
+            import re
+            items = len(re.findall(r'class="athing"', result.html))
+            print(f"\n✅ Captured {items} news items")
+            print(f"• HTML size: {len(result.html):,} bytes")
+            print(f"• Without virtual scroll, would only capture ~30 items")
+
+
+async def demo_url_seeder():
+    """
+    Demo 4: URL Seeder for Intelligent Discovery
+    
+    Shows how to discover and filter URLs before crawling,
+    with relevance scoring.
+    """
+    print("\n" + "="*60)
+    print("🌱 DEMO 4: URL Seeder - Smart URL Discovery")
+    print("="*60)
+    
+    async with AsyncUrlSeeder() as seeder:
+        # Discover Python tutorial URLs
+        config = SeedingConfig(
+            source="sitemap",  # Use sitemap
+            pattern="*python*",  # URL pattern filter
+            extract_head=True,  # Get metadata
+            query="python tutorial",  # For relevance scoring
+            scoring_method="bm25",
+            score_threshold=0.2,
+            max_urls=10
+        )
+        
+        print("Discovering Python async tutorial URLs...")
+        urls = await seeder.urls("https://www.geeksforgeeks.org/", config)
+        
+        print(f"\n✅ Found {len(urls)} relevant URLs:")
+        for i, url_info in enumerate(urls[:5], 1):
+            print(f"\n{i}. {url_info['url']}")
+            if url_info.get('relevance_score'):
+                print(f"   Relevance: {url_info['relevance_score']:.3f}")
+            if url_info.get('head_data', {}).get('title'):
+                print(f"   Title: {url_info['head_data']['title'][:60]}...")
+
+
+async def demo_c4a_script():
+    """
+    Demo 5: C4A Script Language
+    
+    Shows the domain-specific language for web automation
+    with JavaScript transpilation.
+    """
+    print("\n" + "="*60)
+    print("🎭 DEMO 5: C4A Script - Web Automation Language")
+    print("="*60)
+    
+    # Example C4A script
+    c4a_script = """
+# E-commerce automation script
+WAIT `body` 3
+
+# Handle cookie banner
+IF (EXISTS `.cookie-banner`) THEN CLICK `.accept-cookies`
+
+# Search for product
+CLICK `.search-box`
+TYPE "wireless headphones"
+PRESS Enter
+
+# Wait for results
+WAIT `.product-grid` 10
+
+# Load more products
+REPEAT (SCROLL DOWN 500, `document.querySelectorAll('.product').length < 50`)
+
+# Apply filter
+IF (EXISTS `.price-filter`) THEN CLICK `input[data-max-price="100"]`
+"""
+    
+    # Compile the script
+    print("Compiling C4A script...")
+    result = c4a_compile(c4a_script)
+    
+    if result.success:
+        print(f"✅ Successfully compiled to {len(result.js_code)} JavaScript statements!")
+        print("\nFirst 3 JS statements:")
+        for stmt in result.js_code[:3]:
+            print(f"  • {stmt}")
+        
+        # Use with crawler
+        config = CrawlerRunConfig(
+            c4a_script=c4a_script,  # Pass C4A script directly
+            cache_mode=CacheMode.BYPASS
+        )
+        
+        print("\n✅ Script ready for use with AsyncWebCrawler!")
+    else:
+        print(f"❌ Compilation error: {result.first_error.message}")
+
+
+async def main():
+    """Run all demos"""
+    print("\n🚀 Crawl4AI v0.7.0 Feature Demonstrations")
+    print("=" * 60)
+    
+    demos = [
+        ("Link Preview & Scoring", demo_link_preview),
+        ("Adaptive Crawling", demo_adaptive_crawling),
+        ("Virtual Scroll", demo_virtual_scroll),
+        ("URL Seeder", demo_url_seeder),
+        ("C4A Script", demo_c4a_script),
+    ]
+    
+    for name, demo_func in demos:
+        try:
+            await demo_func()
+        except Exception as e:
+            print(f"\n❌ Error in {name} demo: {str(e)}")
+        
+        # Pause between demos
+        await asyncio.sleep(1)
+    
+    print("\n" + "="*60)
+    print("✅ All demos completed!")
+    print("\nKey Takeaways:")
+    print("• Link Preview: 3-layer scoring for intelligent link analysis")
+    print("• Adaptive Crawling: Stop when you have enough information")
+    print("• Virtual Scroll: Capture all content from modern web pages")
+    print("• URL Seeder: Pre-discover and filter URLs efficiently")
+    print("• C4A Script: Simple language for complex automations")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/docs/security/GHSA-DRAFT-RCE-LFI.md b/docs/security/GHSA-DRAFT-RCE-LFI.md
new file mode 100644
index 0000000..e2cc409
--- /dev/null
+++ b/docs/security/GHSA-DRAFT-RCE-LFI.md
@@ -0,0 +1,171 @@
+# GitHub Security Advisory Draft
+
+> **Instructions**: Copy this content to create security advisories at:
+> https://github.com/unclecode/crawl4ai/security/advisories/new
+
+---
+
+## Advisory 1: Remote Code Execution via Hooks Parameter
+
+### Title
+Remote Code Execution in Docker API via Hooks Parameter
+
+### Severity
+Critical
+
+### CVSS Score
+10.0 (CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H)
+
+### CWE
+CWE-94 (Improper Control of Generation of Code)
+
+### Package
+crawl4ai (Docker API deployment)
+
+### Affected Versions
+< 0.8.0
+
+### Patched Versions
+0.8.0
+
+### Description
+
+A critical remote code execution vulnerability exists in the Crawl4AI Docker API deployment. The `/crawl` endpoint accepts a `hooks` parameter containing Python code that is executed using `exec()`. The `__import__` builtin was included in the allowed builtins, allowing attackers to import arbitrary modules and execute system commands.
+
+**Attack Vector:**
+```json
+POST /crawl
+{
+  "urls": ["https://example.com"],
+  "hooks": {
+    "code": {
+      "on_page_context_created": "async def hook(page, context, **kwargs):\n    __import__('os').system('malicious_command')\n    return page"
+    }
+  }
+}
+```
+
+### Impact
+
+An unauthenticated attacker can:
+- Execute arbitrary system commands
+- Read/write files on the server
+- Exfiltrate sensitive data (environment variables, API keys)
+- Pivot to internal network services
+- Completely compromise the server
+
+### Mitigation
+
+1. **Upgrade to v0.8.0** (recommended)
+2. If unable to upgrade immediately:
+   - Disable the Docker API
+   - Block `/crawl` endpoint at network level
+   - Add authentication to the API
+
+### Fix Details
+
+1. Removed `__import__` from `allowed_builtins` in `hook_manager.py`
+2. Hooks disabled by default (`CRAWL4AI_HOOKS_ENABLED=false`)
+3. Users must explicitly opt-in to enable hooks
+
+### Credits
+
+Discovered by Neo by ProjectDiscovery (https://projectdiscovery.io)
+
+### References
+
+- [Release Notes v0.8.0](https://github.com/unclecode/crawl4ai/blob/main/docs/RELEASE_NOTES_v0.8.0.md)
+- [Migration Guide](https://github.com/unclecode/crawl4ai/blob/main/docs/migration/v0.8.0-upgrade-guide.md)
+
+---
+
+## Advisory 2: Local File Inclusion via file:// URLs
+
+### Title
+Local File Inclusion in Docker API via file:// URLs
+
+### Severity
+High
+
+### CVSS Score
+8.6 (CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:N/A:N)
+
+### CWE
+CWE-22 (Improper Limitation of a Pathname to a Restricted Directory)
+
+### Package
+crawl4ai (Docker API deployment)
+
+### Affected Versions
+< 0.8.0
+
+### Patched Versions
+0.8.0
+
+### Description
+
+A local file inclusion vulnerability exists in the Crawl4AI Docker API. The `/execute_js`, `/screenshot`, `/pdf`, and `/html` endpoints accept `file://` URLs, allowing attackers to read arbitrary files from the server filesystem.
+
+**Attack Vector:**
+```json
+POST /execute_js
+{
+  "url": "file:///etc/passwd",
+  "scripts": ["document.body.innerText"]
+}
+```
+
+### Impact
+
+An unauthenticated attacker can:
+- Read sensitive files (`/etc/passwd`, `/etc/shadow`, application configs)
+- Access environment variables via `/proc/self/environ`
+- Discover internal application structure
+- Potentially read credentials and API keys
+
+### Mitigation
+
+1. **Upgrade to v0.8.0** (recommended)
+2. If unable to upgrade immediately:
+   - Disable the Docker API
+   - Add authentication to the API
+   - Use network-level filtering
+
+### Fix Details
+
+Added URL scheme validation to block:
+- `file://` URLs
+- `javascript:` URLs
+- `data:` URLs
+- Other non-HTTP schemes
+
+Only `http://`, `https://`, and `raw:` URLs are now allowed.
+
+### Credits
+
+Discovered by Neo by ProjectDiscovery (https://projectdiscovery.io)
+
+### References
+
+- [Release Notes v0.8.0](https://github.com/unclecode/crawl4ai/blob/main/docs/RELEASE_NOTES_v0.8.0.md)
+- [Migration Guide](https://github.com/unclecode/crawl4ai/blob/main/docs/migration/v0.8.0-upgrade-guide.md)
+
+---
+
+## Creating the Advisories on GitHub
+
+1. Go to: https://github.com/unclecode/crawl4ai/security/advisories/new
+
+2. Fill in the form for each advisory:
+   - **Ecosystem**: PyPI
+   - **Package name**: crawl4ai
+   - **Affected versions**: < 0.8.0
+   - **Patched versions**: 0.8.0
+   - **Severity**: Critical (for RCE), High (for LFI)
+
+3. After creating, GitHub will:
+   - Assign a GHSA ID
+   - Optionally request a CVE
+   - Notify users who have security alerts enabled
+
+4. Coordinate disclosure timing with the fix release
diff --git a/docs/snippets/deep_crawl/1.intro.py b/docs/snippets/deep_crawl/1.intro.py
new file mode 100644
index 0000000..d8fd2f9
--- /dev/null
+++ b/docs/snippets/deep_crawl/1.intro.py
@@ -0,0 +1,78 @@
+import asyncio
+from typing import List
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    BFSDeepCrawlStrategy,
+    CrawlResult,
+    FilterChain,
+    DomainFilter,
+    URLPatternFilter,
+)
+
+# Import necessary classes from crawl4ai library:
+# - AsyncWebCrawler: The main class for web crawling.
+# - CrawlerRunConfig: Configuration class for crawler behavior.
+# - BFSDeepCrawlStrategy: Breadth-First Search deep crawling strategy.
+# - CrawlResult: Data model for individual crawl results.
+# - FilterChain: Used to chain multiple URL filters.
+# - URLPatternFilter: Filter URLs based on patterns.
+# You had from crawl4ai.deep_crawling.filters import FilterChain, URLPatternFilter, which is also correct,
+# but for simplicity and consistency, we will use the direct import from crawl4ai in this example, as it is re-exported in __init__.py
+
+async def basic_deep_crawl():
+    """
+    Performs a basic deep crawl starting from a seed URL, demonstrating:
+    - Breadth-First Search (BFS) deep crawling strategy.
+    - Filtering URLs based on URL patterns.
+    - Accessing crawl results and metadata.
+    """
+
+    # 1. Define URL Filters:
+    # Create a URLPatternFilter to include only URLs containing "text".
+    # This filter will be used to restrict crawling to URLs that are likely to contain textual content.
+    url_filter = URLPatternFilter(
+        patterns=[
+            "*text*", # Include URLs that contain "text" in their path or URL
+        ]
+    )
+
+    # Create a DomainFilter to allow only URLs from the "groq.com" domain and block URLs from the "example.com" domain.
+    # This filter will be used to restrict crawling to URLs within the "groq.com" domain.
+    domain_filter = DomainFilter(
+        allowed_domains=["groq.com"],
+        blocked_domains=["example.com"],
+    )
+
+    # 2. Configure CrawlerRunConfig for Deep Crawling:
+    # Configure CrawlerRunConfig to use BFSDeepCrawlStrategy for deep crawling.
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=BFSDeepCrawlStrategy(
+            max_depth=2,  # Set the maximum depth of crawling to 2 levels from the start URL
+            max_pages=10, # Limit the total number of pages to crawl to 10, to prevent excessive crawling
+            include_external=False, # Set to False to only crawl URLs within the same domain as the start URL
+            filter_chain=FilterChain(filters=[url_filter, domain_filter]), # Apply the URLPatternFilter and DomainFilter to filter URLs during deep crawl
+        ),
+        verbose=True, # Enable verbose logging to see detailed output during crawling
+    )
+
+    # 3. Initialize and Run AsyncWebCrawler:
+    # Use AsyncWebCrawler as a context manager for automatic start and close.
+    async with AsyncWebCrawler() as crawler:
+        results: List[CrawlResult] = await crawler.arun(
+            # url="https://docs.crawl4ai.com", # Uncomment to use crawl4ai documentation as start URL
+            url="https://console.groq.com/docs", # Set the start URL for deep crawling to Groq documentation
+            config=config, # Pass the configured CrawlerRunConfig to arun method
+        )
+
+        # 4. Process and Print Crawl Results:
+        # Iterate through the list of CrawlResult objects returned by the deep crawl.
+        for result in results:
+            # Print the URL and its crawl depth from the metadata for each crawled URL.
+            print(f"URL: {result.url}, Depth: {result.metadata.get('depth', 0)}")
+
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(basic_deep_crawl())
diff --git a/docs/snippets/deep_crawl/2.filters.py b/docs/snippets/deep_crawl/2.filters.py
new file mode 100644
index 0000000..c50eae0
--- /dev/null
+++ b/docs/snippets/deep_crawl/2.filters.py
@@ -0,0 +1,162 @@
+import asyncio
+from typing import List
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    BFSDeepCrawlStrategy,
+    CrawlResult,
+    URLFilter, # Base class for filters, not directly used in examples but good to import for context
+    ContentTypeFilter,
+    DomainFilter,
+    FilterChain,
+    URLPatternFilter,
+    SEOFilter # Advanced filter, can be introduced later or as bonus
+)
+
+async def deep_crawl_filter_tutorial_part_2():
+    """
+    Tutorial demonstrating URL filters in Crawl4AI, focusing on isolated filter behavior
+    before integrating them into a deep crawl.
+
+    This tutorial covers:
+    - Testing individual filters with synthetic URLs.
+    - Understanding filter logic and behavior in isolation.
+    - Combining filters using FilterChain.
+    - Integrating filters into a deep crawling example.
+    """
+
+    # === Introduction: URL Filters in Isolation ===
+    print("\n" + "=" * 40)
+    print("=== Introduction: URL Filters in Isolation ===")
+    print("=" * 40 + "\n")
+    print("In this section, we will explore each filter individually using synthetic URLs.")
+    print("This allows us to understand exactly how each filter works before using them in a crawl.\n")
+
+
+    # === 2. ContentTypeFilter - Testing in Isolation ===
+    print("\n" + "=" * 40)
+    print("=== 2. ContentTypeFilter - Testing in Isolation ===")
+    print("=" * 40 + "\n")
+
+    # 2.1. Create ContentTypeFilter:
+    # Create a ContentTypeFilter to allow only 'text/html' and 'application/json' content types 
+    # BASED ON URL EXTENSIONS.
+    content_type_filter = ContentTypeFilter(allowed_types=["text/html", "application/json"])
+    print("ContentTypeFilter created, allowing types (by extension): ['text/html', 'application/json']")
+    print("Note: ContentTypeFilter in Crawl4ai works by checking URL file extensions, not HTTP headers.")
+
+
+    # 2.2. Synthetic URLs for Testing:
+    # ContentTypeFilter checks URL extensions. We provide URLs with different extensions to test.
+    test_urls_content_type = [
+        "https://example.com/page.html",       # Should pass: .html extension (text/html)
+        "https://example.com/data.json",       # Should pass: .json extension (application/json)
+        "https://example.com/image.png",       # Should reject: .png extension (not allowed type)
+        "https://example.com/document.pdf",    # Should reject: .pdf extension (not allowed type)
+        "https://example.com/page",            # Should pass: no extension (defaults to allow) - check default behaviour!
+        "https://example.com/page.xhtml",      # Should pass: .xhtml extension (text/html)
+    ]
+
+    # 2.3. Apply Filter and Show Results:
+    print("\n=== Testing ContentTypeFilter (URL Extension based) ===")
+    for url in test_urls_content_type:
+        passed = content_type_filter.apply(url)
+        result = "PASSED" if passed else "REJECTED"
+        extension = ContentTypeFilter._extract_extension(url) # Show extracted extension for clarity
+        print(f"- URL: {url} - {result} (Extension: '{extension or 'No Extension'}')")
+    print("=" * 40)
+
+    input("Press Enter to continue to DomainFilter example...")
+
+    # === 3. DomainFilter - Testing in Isolation ===
+    print("\n" + "=" * 40)
+    print("=== 3. DomainFilter - Testing in Isolation ===")
+    print("=" * 40 + "\n")
+
+    # 3.1. Create DomainFilter:
+    domain_filter = DomainFilter(allowed_domains=["crawl4ai.com", "example.com"])
+    print("DomainFilter created, allowing domains: ['crawl4ai.com', 'example.com']")
+
+    # 3.2. Synthetic URLs for Testing:
+    test_urls_domain = [
+        "https://docs.crawl4ai.com/api",
+        "https://example.com/products",
+        "https://another-website.org/blog",
+        "https://sub.example.com/about",
+        "https://crawl4ai.com.attacker.net", # Corrected example: now should be rejected
+    ]
+
+    # 3.3. Apply Filter and Show Results:
+    print("\n=== Testing DomainFilter ===")
+    for url in test_urls_domain:
+        passed = domain_filter.apply(url)
+        result = "PASSED" if passed else "REJECTED"
+        print(f"- URL: {url} - {result}")
+    print("=" * 40)
+
+    input("Press Enter to continue to FilterChain example...")
+
+    # === 4. FilterChain - Combining Filters ===
+    print("\n" + "=" * 40)
+    print("=== 4. FilterChain - Combining Filters ===")
+    print("=" * 40 + "\n")
+
+    combined_filter = FilterChain(
+        filters=[
+            URLPatternFilter(patterns=["*api*"]),
+            ContentTypeFilter(allowed_types=["text/html"]), # Still URL extension based
+            DomainFilter(allowed_domains=["docs.crawl4ai.com"]),
+        ]
+    )
+    print("FilterChain created, combining URLPatternFilter, ContentTypeFilter, and DomainFilter.")
+
+
+    test_urls_combined = [
+        "https://docs.crawl4ai.com/api/async-webcrawler",
+        "https://example.com/api/products",
+        "https://docs.crawl4ai.com/core/crawling",
+        "https://another-website.org/api/data",
+    ]
+
+    # 4.3. Apply FilterChain and Show Results
+    print("\n=== Testing FilterChain (URLPatternFilter + ContentTypeFilter + DomainFilter) ===")
+    for url in test_urls_combined:
+        passed = await combined_filter.apply(url)
+        result = "PASSED" if passed else "REJECTED"
+        print(f"- URL: {url} - {result}")
+    print("=" * 40)
+
+    input("Press Enter to continue to Deep Crawl with FilterChain example...")
+
+    # === 5. Deep Crawl with FilterChain ===
+    print("\n" + "=" * 40)
+    print("=== 5. Deep Crawl with FilterChain ===")
+    print("=" * 40 + "\n")
+    print("Finally, let's integrate the FilterChain into a deep crawl example.")
+
+    config_final_crawl = CrawlerRunConfig(
+        deep_crawl_strategy=BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=10,
+            include_external=False,
+            filter_chain=combined_filter
+        ),
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        results_final_crawl: List[CrawlResult] = await crawler.arun(
+            url="https://docs.crawl4ai.com", config=config_final_crawl
+        )
+
+        print("=== Crawled URLs (Deep Crawl with FilterChain) ===")
+        for result in results_final_crawl:
+            print(f"- {result.url}, Depth: {result.metadata.get('depth', 0)}")
+        print("=" * 40)
+
+    print("\nTutorial Completed! Review the output of each section to understand URL filters.")
+
+
+if __name__ == "__main__":
+    asyncio.run(deep_crawl_filter_tutorial_part_2())
\ No newline at end of file
diff --git a/docs/tutorials/coming_soon.md b/docs/tutorials/coming_soon.md
new file mode 100644
index 0000000..e69de29
diff --git a/mkdocs.yml b/mkdocs.yml
new file mode 100644
index 0000000..d821526
--- /dev/null
+++ b/mkdocs.yml
@@ -0,0 +1,129 @@
+site_name: Crawl4AI Documentation (v0.9.x)
+site_description:  🚀🤖 Crawl4AI, Open-source LLM-Friendly Web Crawler & Scraper
+site_url: https://docs.crawl4ai.com
+repo_url: https://github.com/unclecode/crawl4ai
+repo_name: unclecode/crawl4ai
+docs_dir: docs/md_v2
+
+nav:
+  - Home: 'index.md'
+  - "Growth": "stats.md"
+  - "Ask AI": "core/ask-ai.md"
+  - "Quick Start": "core/quickstart.md"
+  - "Code Examples": "core/examples.md"
+  - Apps:
+    - "Demo Apps": "apps/index.md"
+    - "C4A-Script Editor": "apps/c4a-script/index.html"
+    - "LLM Context Builder": "apps/llmtxt/index.html"
+    - "Marketplace": "marketplace/index.html"
+    - "Marketplace Admin": "marketplace/admin/index.html"
+  - Setup & Installation:
+    - "Installation": "core/installation.md"
+    - "Self-Hosting Guide": "core/self-hosting.md"
+  - "Blog & Changelog":
+    - "Blog Home": "blog/index.md"
+    - "Changelog": "https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md"
+  - Core:
+    - "Command Line Interface": "core/cli.md"
+    - "Simple Crawling": "core/simple-crawling.md"
+    - "Deep Crawling": "core/deep-crawling.md"
+    - "Adaptive Crawling": "core/adaptive-crawling.md"
+    - "URL Seeding": "core/url-seeding.md"
+    - "Domain Mapping": "core/domain-mapping.md"
+    - "C4A-Script": "core/c4a-script.md"
+    - "Crawler Result": "core/crawler-result.md"
+    - "Browser, Crawler & LLM Config": "core/browser-crawler-config.md"
+    - "Markdown Generation": "core/markdown-generation.md"
+    - "Fit Markdown": "core/fit-markdown.md"
+    - "Page Interaction": "core/page-interaction.md"
+    - "Content Selection": "core/content-selection.md"
+    - "Cache Modes": "core/cache-modes.md"
+    - "Local Files & Raw HTML": "core/local-files.md"
+    - "Link & Media": "core/link-media.md"
+  - Advanced:
+    - "Overview": "advanced/advanced-features.md"
+    - "Adaptive Strategies": "advanced/adaptive-strategies.md"
+    - "Virtual Scroll": "advanced/virtual-scroll.md"
+    - "File Downloading": "advanced/file-downloading.md"
+    - "Lazy Loading": "advanced/lazy-loading.md"
+    - "Hooks & Auth": "advanced/hooks-auth.md"
+    - "Proxy & Security": "advanced/proxy-security.md"
+    - "Anti-Bot & Fallback": "advanced/anti-bot-and-fallback.md"
+    - "Undetected Browser": "advanced/undetected-browser.md"
+    - "Session Management": "advanced/session-management.md"
+    - "Multi-URL Crawling": "advanced/multi-url-crawling.md"
+    - "Crawl Dispatcher": "advanced/crawl-dispatcher.md"
+    - "Identity Based Crawling": "advanced/identity-based-crawling.md"
+    - "SSL Certificate": "advanced/ssl-certificate.md"
+    - "Network & Console Capture": "advanced/network-console-capture.md"
+    - "PDF Parsing": "advanced/pdf-parsing.md"
+  - Extraction:
+    - "LLM-Free Strategies": "extraction/no-llm-strategies.md"
+    - "LLM Strategies": "extraction/llm-strategies.md"
+    - "Clustering Strategies": "extraction/clustring-strategies.md"
+    - "Chunking": "extraction/chunking.md"
+  - API Reference:
+    - "AsyncWebCrawler": "api/async-webcrawler.md"
+    - "arun()": "api/arun.md"
+    - "arun_many()": "api/arun_many.md"
+    - "Browser, Crawler & LLM Config": "api/parameters.md"
+    - "CrawlResult": "api/crawl-result.md"
+    - "Strategies": "api/strategies.md"
+    - "C4A-Script Reference": "api/c4a-script-reference.md"
+  - "Brand Book": "branding/index.md"
+  - Community:
+    - "Contributing Guide": "CONTRIBUTING.md"
+    - "Code of Conduct": "https://github.com/unclecode/crawl4ai/blob/main/CODE_OF_CONDUCT.md"
+    - "Contributors": "https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md"
+  - Legal:
+    - "Privacy Policy": "privacy.md"
+    - "Terms of Service": "terms.md"
+    - "Support": "support.md"
+
+theme:
+  name: 'terminal'
+  palette: 'dark'
+  favicon: favicon.ico
+  custom_dir: docs/md_v2/overrides
+  color_mode: 'dark'
+  icon:
+    repo: fontawesome/brands/github
+
+plugins:
+  - search
+
+markdown_extensions:
+  - pymdownx.highlight:
+      anchor_linenums: true
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+  - pymdownx.superfences
+  - admonition
+  - pymdownx.details
+  - attr_list
+  - tables
+
+extra:
+  version: !ENV [CRAWL4AI_VERSION, 'development']
+
+extra_css:
+  - assets/layout.css
+  - assets/styles.css
+  - assets/highlight.css
+  - assets/dmvendor.css
+  - assets/feedback-overrides.css
+  - assets/page_actions.css
+
+extra_javascript:
+  - https://www.googletagmanager.com/gtag/js?id=G-58W0K2ZQ25
+  - assets/gtag.js
+  - assets/highlight.min.js
+  - assets/highlight_init.js
+  - https://buttons.github.io/buttons.js
+  - assets/toc.js
+  - assets/github_stats.js
+  - assets/selection_ask_ai.js
+  - assets/copy_code.js
+  - assets/floating_ask_ai_button.js
+  - assets/mobile_menu.js
+  - assets/page_actions.js?v=20251006
\ No newline at end of file
diff --git a/prompts/prompt_net_requests.md b/prompts/prompt_net_requests.md
new file mode 100644
index 0000000..d033591
--- /dev/null
+++ b/prompts/prompt_net_requests.md
@@ -0,0 +1,489 @@
+I want to enhance the `AsyncPlaywrightCrawlerStrategy` to optionally capture network requests and console messages during a crawl, storing them in the final `CrawlResult`.
+
+Here's a breakdown of the proposed changes across the relevant files:
+
+**1. Configuration (`crawl4ai/async_configs.py`)**
+
+*   **Goal:** Add flags to `CrawlerRunConfig` to enable/disable capturing.
+*   **Changes:**
+    *   Add two new boolean attributes to `CrawlerRunConfig`:
+        *   `capture_network_requests: bool = False`
+        *   `capture_console_messages: bool = False`
+    *   Update `__init__`, `from_kwargs`, `to_dict`, and implicitly `clone`/`dump`/`load` to include these new attributes.
+
+```python
+# ==== File: crawl4ai/async_configs.py ====
+# ... (imports) ...
+
+class CrawlerRunConfig():
+    # ... (existing attributes) ...
+
+    # NEW: Network and Console Capturing Parameters
+    capture_network_requests: bool = False
+    capture_console_messages: bool = False
+
+    # Experimental Parameters
+    experimental: Dict[str, Any] = None,
+
+    def __init__(
+        self,
+        # ... (existing parameters) ...
+
+        # NEW: Network and Console Capturing Parameters
+        capture_network_requests: bool = False,
+        capture_console_messages: bool = False,
+
+        # Experimental Parameters
+        experimental: Dict[str, Any] = None,
+    ):
+        # ... (existing assignments) ...
+
+        # NEW: Assign new parameters
+        self.capture_network_requests = capture_network_requests
+        self.capture_console_messages = capture_console_messages
+
+        # Experimental Parameters
+        self.experimental = experimental or {}
+
+        # ... (rest of __init__) ...
+
+    @staticmethod
+    def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
+        return CrawlerRunConfig(
+            # ... (existing kwargs gets) ...
+
+            # NEW: Get new parameters
+            capture_network_requests=kwargs.get("capture_network_requests", False),
+            capture_console_messages=kwargs.get("capture_console_messages", False),
+
+            # Experimental Parameters
+            experimental=kwargs.get("experimental"),
+        )
+
+    def to_dict(self):
+        return {
+            # ... (existing dict entries) ...
+
+            # NEW: Add new parameters to dict
+            "capture_network_requests": self.capture_network_requests,
+            "capture_console_messages": self.capture_console_messages,
+
+            "experimental": self.experimental,
+        }
+
+    # clone(), dump(), load() should work automatically if they rely on to_dict() and from_kwargs()
+    # or the serialization logic correctly handles all attributes.
+```
+
+**2. Data Models (`crawl4ai/models.py`)**
+
+*   **Goal:** Add fields to store the captured data in the response/result objects.
+*   **Changes:**
+    *   Add `network_requests: Optional[List[Dict[str, Any]]] = None` and `console_messages: Optional[List[Dict[str, Any]]] = None` to `AsyncCrawlResponse`.
+    *   Add the same fields to `CrawlResult`.
+
+```python
+# ==== File: crawl4ai/models.py ====
+# ... (imports) ...
+
+# ... (Existing dataclasses/models) ...
+
+class AsyncCrawlResponse(BaseModel):
+    html: str
+    response_headers: Dict[str, str]
+    js_execution_result: Optional[Dict[str, Any]] = None
+    status_code: int
+    screenshot: Optional[str] = None
+    pdf_data: Optional[bytes] = None
+    get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
+    downloaded_files: Optional[List[str]] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+    redirected_url: Optional[str] = None
+    # NEW: Fields for captured data
+    network_requests: Optional[List[Dict[str, Any]]] = None
+    console_messages: Optional[List[Dict[str, Any]]] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+
+# ... (Existing models like MediaItem, Link, etc.) ...
+
+class CrawlResult(BaseModel):
+    url: str
+    html: str
+    success: bool
+    cleaned_html: Optional[str] = None
+    media: Dict[str, List[Dict]] = {}
+    links: Dict[str, List[Dict]] = {}
+    downloaded_files: Optional[List[str]] = None
+    js_execution_result: Optional[Dict[str, Any]] = None
+    screenshot: Optional[str] = None
+    pdf: Optional[bytes] = None
+    mhtml: Optional[str] = None # Added mhtml based on the provided models.py
+    _markdown: Optional[MarkdownGenerationResult] = PrivateAttr(default=None)
+    extracted_content: Optional[str] = None
+    metadata: Optional[dict] = None
+    error_message: Optional[str] = None
+    session_id: Optional[str] = None
+    response_headers: Optional[dict] = None
+    status_code: Optional[int] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+    dispatch_result: Optional[DispatchResult] = None
+    redirected_url: Optional[str] = None
+    # NEW: Fields for captured data
+    network_requests: Optional[List[Dict[str, Any]]] = None
+    console_messages: Optional[List[Dict[str, Any]]] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    # ... (Existing __init__, properties, model_dump for markdown compatibility) ...
+
+# ... (Rest of the models) ...
+```
+
+**3. Crawler Strategy (`crawl4ai/async_crawler_strategy.py`)**
+
+*   **Goal:** Implement the actual capturing logic within `AsyncPlaywrightCrawlerStrategy._crawl_web`.
+*   **Changes:**
+    *   Inside `_crawl_web`, initialize empty lists `captured_requests = []` and `captured_console = []`.
+    *   Conditionally attach Playwright event listeners (`page.on(...)`) based on the `config.capture_network_requests` and `config.capture_console_messages` flags.
+    *   Define handler functions for these listeners to extract relevant data and append it to the respective lists. Include timestamps.
+    *   Pass the captured lists to the `AsyncCrawlResponse` constructor at the end of the method.
+
+```python
+# ==== File: crawl4ai/async_crawler_strategy.py ====
+# ... (imports) ...
+import time # Make sure time is imported
+
+class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
+    # ... (existing methods like __init__, start, close, etc.) ...
+
+    async def _crawl_web(
+        self, url: str, config: CrawlerRunConfig
+    ) -> AsyncCrawlResponse:
+        """
+        Internal method to crawl web URLs with the specified configuration.
+        Includes optional network and console capturing. # MODIFIED DOCSTRING
+        """
+        config.url = url
+        response_headers = {}
+        execution_result = None
+        status_code = None
+        redirected_url = url
+
+        # Reset downloaded files list for new crawl
+        self._downloaded_files = []
+
+        # Initialize capture lists - IMPORTANT: Reset per crawl
+        captured_requests: List[Dict[str, Any]] = []
+        captured_console: List[Dict[str, Any]] = []
+
+        # Handle user agent ... (existing code) ...
+
+        # Get page for session
+        page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
+
+        # ... (existing code for cookies, navigator overrides, hooks) ...
+
+        # --- Setup Capturing Listeners ---
+        # NOTE: These listeners are attached *before* page.goto()
+
+        # Network Request Capturing
+        if config.capture_network_requests:
+            async def handle_request_capture(request):
+                try:
+                    post_data_str = None
+                    try:
+                        # Be cautious with large post data
+                        post_data = request.post_data_buffer
+                        if post_data:
+                             # Attempt to decode, fallback to base64 or size indication
+                             try:
+                                 post_data_str = post_data.decode('utf-8', errors='replace')
+                             except UnicodeDecodeError:
+                                 post_data_str = f"[Binary data: {len(post_data)} bytes]"
+                    except Exception:
+                        post_data_str = "[Error retrieving post data]"
+
+                    captured_requests.append({
+                        "event_type": "request",
+                        "url": request.url,
+                        "method": request.method,
+                        "headers": dict(request.headers), # Convert Header dict
+                        "post_data": post_data_str,
+                        "resource_type": request.resource_type,
+                        "is_navigation_request": request.is_navigation_request(),
+                        "timestamp": time.time()
+                    })
+                except Exception as e:
+                    self.logger.warning(f"Error capturing request details for {request.url}: {e}", tag="CAPTURE")
+                    captured_requests.append({"event_type": "request_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()})
+
+            async def handle_response_capture(response):
+                try:
+                    # Avoid capturing full response body by default due to size/security
+                    # security_details = await response.security_details() # Optional: More SSL info
+                    captured_requests.append({
+                        "event_type": "response",
+                        "url": response.url,
+                        "status": response.status,
+                        "status_text": response.status_text,
+                        "headers": dict(response.headers), # Convert Header dict
+                        "from_service_worker": response.from_service_worker,
+                        # "security_details": security_details, # Uncomment if needed
+                        "request_timing": response.request.timing, # Detailed timing info
+                        "timestamp": time.time()
+                    })
+                except Exception as e:
+                    self.logger.warning(f"Error capturing response details for {response.url}: {e}", tag="CAPTURE")
+                    captured_requests.append({"event_type": "response_capture_error", "url": response.url, "error": str(e), "timestamp": time.time()})
+
+            async def handle_request_failed_capture(request):
+                 try:
+                    captured_requests.append({
+                        "event_type": "request_failed",
+                        "url": request.url,
+                        "method": request.method,
+                        "resource_type": request.resource_type,
+                        "failure_text": request.failure.error_text if request.failure else "Unknown failure",
+                        "timestamp": time.time()
+                    })
+                 except Exception as e:
+                    self.logger.warning(f"Error capturing request failed details for {request.url}: {e}", tag="CAPTURE")
+                    captured_requests.append({"event_type": "request_failed_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()})
+
+            page.on("request", handle_request_capture)
+            page.on("response", handle_response_capture)
+            page.on("requestfailed", handle_request_failed_capture)
+
+        # Console Message Capturing
+        if config.capture_console_messages:
+            def handle_console_capture(msg):
+                 try:
+                    location = msg.location()
+                    # Attempt to resolve JSHandle args to primitive values
+                    resolved_args = []
+                    try:
+                        for arg in msg.args:
+                            resolved_args.append(arg.json_value()) # May fail for complex objects
+                    except Exception:
+                         resolved_args.append("[Could not resolve JSHandle args]")
+
+                    captured_console.append({
+                        "type": msg.type(), # e.g., 'log', 'error', 'warning'
+                        "text": msg.text(),
+                        "args": resolved_args, # Captured arguments
+                        "location": f"{location['url']}:{location['lineNumber']}:{location['columnNumber']}" if location else "N/A",
+                        "timestamp": time.time()
+                    })
+                 except Exception as e:
+                    self.logger.warning(f"Error capturing console message: {e}", tag="CAPTURE")
+                    captured_console.append({"type": "console_capture_error", "error": str(e), "timestamp": time.time()})
+
+            def handle_pageerror_capture(err):
+                 try:
+                    captured_console.append({
+                        "type": "error", # Consistent type for page errors
+                        "text": err.message,
+                        "stack": err.stack,
+                        "timestamp": time.time()
+                    })
+                 except Exception as e:
+                    self.logger.warning(f"Error capturing page error: {e}", tag="CAPTURE")
+                    captured_console.append({"type": "pageerror_capture_error", "error": str(e), "timestamp": time.time()})
+
+            page.on("console", handle_console_capture)
+            page.on("pageerror", handle_pageerror_capture)
+        # --- End Setup Capturing Listeners ---
+
+
+        # Set up console logging if requested (Keep original logging logic separate or merge carefully)
+        if config.log_console:
+            # ... (original log_console setup using page.on(...) remains here) ...
+            # This allows logging to screen *and* capturing to the list if both flags are True
+            def log_consol(msg, console_log_type="debug"):
+                # ... existing implementation ...
+                pass # Placeholder for existing code
+
+            page.on("console", lambda msg: log_consol(msg, "debug"))
+            page.on("pageerror", lambda e: log_consol(e, "error"))
+
+
+        try:
+            # ... (existing code for SSL, downloads, goto, waits, JS execution, etc.) ...
+
+            # Get final HTML content
+            # ... (existing code for selector logic or page.content()) ...
+            if config.css_selector:
+                # ... existing selector logic ...
+                html = f"
    \n" + "\n".join(html_parts) + "\n
    "
+            else:
+                html = await page.content()
+
+            await self.execute_hook(
+                "before_return_html", page=page, html=html, context=context, config=config
+            )
+
+            # Handle PDF and screenshot generation
+            # ... (existing code) ...
+
+            # Define delayed content getter
+            # ... (existing code) ...
+
+            # Return complete response - ADD CAPTURED DATA HERE
+            return AsyncCrawlResponse(
+                html=html,
+                response_headers=response_headers,
+                js_execution_result=execution_result,
+                status_code=status_code,
+                screenshot=screenshot_data,
+                pdf_data=pdf_data,
+                get_delayed_content=get_delayed_content,
+                ssl_certificate=ssl_cert,
+                downloaded_files=(
+                    self._downloaded_files if self._downloaded_files else None
+                ),
+                redirected_url=redirected_url,
+                # NEW: Pass captured data conditionally
+                network_requests=captured_requests if config.capture_network_requests else None,
+                console_messages=captured_console if config.capture_console_messages else None,
+            )
+
+        except Exception as e:
+            raise e # Re-raise the original exception
+
+        finally:
+            # If no session_id is given we should close the page
+            if not config.session_id:
+                # Detach listeners before closing to prevent potential errors during close
+                if config.capture_network_requests:
+                    page.remove_listener("request", handle_request_capture)
+                    page.remove_listener("response", handle_response_capture)
+                    page.remove_listener("requestfailed", handle_request_failed_capture)
+                if config.capture_console_messages:
+                    page.remove_listener("console", handle_console_capture)
+                    page.remove_listener("pageerror", handle_pageerror_capture)
+                # Also remove logging listeners if they were attached
+                if config.log_console:
+                    # Need to figure out how to remove the lambdas if necessary,
+                    # or ensure they don't cause issues on close. Often, it's fine.
+                    pass
+
+                await page.close()
+
+    # ... (rest of AsyncPlaywrightCrawlerStrategy methods) ...
+
+```
+
+**4. Core Crawler (`crawl4ai/async_webcrawler.py`)**
+
+*   **Goal:** Ensure the captured data from `AsyncCrawlResponse` is transferred to the final `CrawlResult`.
+*   **Changes:**
+    *   In `arun`, when processing a non-cached result (inside the `if not cached_result or not html:` block), after receiving `async_response` and calling `aprocess_html` to get `crawl_result`, copy the `network_requests` and `console_messages` from `async_response` to `crawl_result`.
+
+```python
+# ==== File: crawl4ai/async_webcrawler.py ====
+# ... (imports) ...
+
+class AsyncWebCrawler:
+    # ... (existing methods) ...
+
+    async def arun(
+        self,
+        url: str,
+        config: CrawlerRunConfig = None,
+        **kwargs,
+    ) -> RunManyReturn:
+        # ... (existing setup, cache check) ...
+
+        async with self._lock or self.nullcontext():
+            try:
+                # ... (existing logging, cache context setup) ...
+
+                if cached_result:
+                    # ... (existing cache handling logic) ...
+                    # Note: Captured network/console usually not useful from cache
+                    # Ensure they are None or empty if read from cache, unless stored explicitly
+                    cached_result.network_requests = cached_result.network_requests or None
+                    cached_result.console_messages = cached_result.console_messages or None
+                    # ... (rest of cache logic) ...
+
+                # Fetch fresh content if needed
+                if not cached_result or not html:
+                    t1 = time.perf_counter()
+
+                    # ... (existing user agent update, robots.txt check) ...
+
+                    ##############################
+                    # Call CrawlerStrategy.crawl #
+                    ##############################
+                    async_response = await self.crawler_strategy.crawl(
+                        url,
+                        config=config,
+                    )
+
+                    # ... (existing assignment of html, screenshot, pdf, js_result from async_response) ...
+
+                    t2 = time.perf_counter()
+                    # ... (existing logging) ...
+
+                    ###############################################################
+                    # Process the HTML content, Call CrawlerStrategy.process_html #
+                    ###############################################################
+                    crawl_result: CrawlResult = await self.aprocess_html(
+                        # ... (existing args) ...
+                    )
+
+                    # --- Transfer data from AsyncCrawlResponse to CrawlResult ---
+                    crawl_result.status_code = async_response.status_code
+                    crawl_result.redirected_url = async_response.redirected_url or url
+                    crawl_result.response_headers = async_response.response_headers
+                    crawl_result.downloaded_files = async_response.downloaded_files
+                    crawl_result.js_execution_result = js_execution_result
+                    crawl_result.ssl_certificate = async_response.ssl_certificate
+                    # NEW: Copy captured data
+                    crawl_result.network_requests = async_response.network_requests
+                    crawl_result.console_messages = async_response.console_messages
+                    # ------------------------------------------------------------
+
+                    crawl_result.success = bool(html)
+                    crawl_result.session_id = getattr(config, "session_id", None)
+
+                    # ... (existing logging) ...
+
+                    # Update cache if appropriate
+                    if cache_context.should_write() and not bool(cached_result):
+                        # crawl_result now includes network/console data if captured
+                        await async_db_manager.acache_url(crawl_result)
+
+                    return CrawlResultContainer(crawl_result)
+
+                else: # Cached result was used
+                     # ... (existing logging for cache hit) ...
+                    cached_result.success = bool(html)
+                    cached_result.session_id = getattr(config, "session_id", None)
+                    cached_result.redirected_url = cached_result.redirected_url or url
+                    return CrawlResultContainer(cached_result)
+
+            except Exception as e:
+                # ... (existing error handling) ...
+                return CrawlResultContainer(
+                    CrawlResult(
+                        url=url, html="", success=False, error_message=error_message
+                    )
+                )
+
+    # ... (aprocess_html remains unchanged regarding capture) ...
+
+    # ... (arun_many remains unchanged regarding capture) ...
+```
+
+**Summary of Changes:**
+
+1.  **Configuration:** Added `capture_network_requests` and `capture_console_messages` flags to `CrawlerRunConfig`.
+2.  **Models:** Added corresponding `network_requests` and `console_messages` fields (List of Dicts) to `AsyncCrawlResponse` and `CrawlResult`.
+3.  **Strategy:** Implemented conditional event listeners in `AsyncPlaywrightCrawlerStrategy._crawl_web` to capture data into lists when flags are true. Populated these fields in the returned `AsyncCrawlResponse`. Added basic error handling within capture handlers. Added timestamps.
+4.  **Crawler:** Modified `AsyncWebCrawler.arun` to copy the captured data from `AsyncCrawlResponse` into the final `CrawlResult` for non-cached fetches.
+
+This approach keeps the capturing logic contained within the Playwright strategy, uses clear configuration flags, and integrates the results into the existing data flow. The data format (list of dictionaries) is flexible for storing varied information from requests/responses/console messages.
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..f93d106
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,100 @@
+[build-system]
+requires = ["setuptools>=64.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "Crawl4AI"
+dynamic = ["version"]
+description = "🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [
+    {name = "Unclecode", email = "unclecode@kidocode.com"}
+]
+dependencies = [
+    "aiofiles>=24.1.0",
+    "aiohttp>=3.11.11",
+    "aiosqlite~=0.20",
+    "anyio>=4.0.0",
+    "lxml>=5.3,<7",
+    "unclecode-litellm==1.81.13",
+    "numpy>=1.26.0,<3",
+    "pillow>=10.4",
+    "playwright>=1.49.0",
+    "patchright>=1.49.0",
+    "python-dotenv~=1.0",
+    "requests~=2.26",
+    "beautifulsoup4~=4.12",
+    "playwright-stealth>=2.0.0",
+    "xxhash~=3.4",
+    "rank-bm25~=0.2",
+    "snowballstemmer~=2.2",
+    "pydantic>=2.10",
+    "pyOpenSSL>=25.3.0",
+    "psutil>=6.1.1",
+    "PyYAML>=6.0",
+    "nltk>=3.9.1",
+    "rich>=13.9.4",
+    "cssselect>=1.2.0",
+    "httpx>=0.27.2",
+    "httpx[http2]>=0.27.2",
+    "fake-useragent>=2.0.3",
+    "click>=8.1.7",
+    "chardet>=5.2.0",
+    "brotli>=1.1.0",
+    "humanize>=4.10.0",
+    "lark>=1.2.2",
+    "alphashape>=1.3.1",
+    "shapely>=2.0.0"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+
+[project.optional-dependencies]
+pdf = ["pypdf"]  
+torch = ["torch", "nltk", "scikit-learn"]
+transformer = ["transformers", "tokenizers", "sentence-transformers"]
+cosine = ["torch", "transformers", "nltk", "sentence-transformers"]
+sync = ["selenium"]
+all = [
+    "pypdf",
+    "torch",
+    "nltk",
+    "scikit-learn",
+    "transformers",
+    "tokenizers",
+    "sentence-transformers",
+    "selenium"
+]
+
+[project.scripts]
+crawl4ai-download-models = "crawl4ai.model_loader:main"
+crawl4ai-migrate = "crawl4ai.migrations:main"
+crawl4ai-setup = "crawl4ai.install:post_install"
+crawl4ai-doctor = "crawl4ai.install:doctor"
+crwl = "crawl4ai.cli:main"
+
+[tool.setuptools]
+packages = {find = {where = ["."], include = ["crawl4ai*"]}}
+
+[tool.setuptools.package-data]
+crawl4ai = ["js_snippet/*.js", "crawlers/google_search/*.js"]
+
+[tool.setuptools.dynamic]
+version = {attr = "crawl4ai.__version__.__version__"}
+
+[tool.uv.sources]
+crawl4ai = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "crawl4ai",
+]
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..524a3d0
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,36 @@
+# Note: These requirements are also specified in pyproject.toml
+# This file is kept for development environment setup and compatibility
+aiofiles>=24.1.0
+aiohttp>=3.11.11
+aiosqlite~=0.20
+anyio>=4.0.0
+lxml>=5.3,<7
+unclecode-litellm==1.81.13
+numpy>=1.26.0,<3
+pillow>=10.4
+playwright>=1.49.0
+patchright>=1.49.0
+python-dotenv~=1.0
+requests~=2.26
+beautifulsoup4~=4.12
+playwright-stealth>=2.0.0
+xxhash~=3.4
+rank-bm25~=0.2
+colorama~=0.4
+snowballstemmer~=2.2
+pydantic>=2.10
+pyOpenSSL>=25.3.0
+psutil>=6.1.1
+PyYAML>=6.0
+nltk>=3.9.1
+rich>=13.9.4
+cssselect>=1.2.0
+chardet>=5.2.0
+brotli>=1.1.0
+httpx[http2]>=0.27.2
+alphashape>=1.3.1
+shapely>=2.0.0
+
+fake-useragent>=2.2.0
+pdf2image>=1.17.0
+pypdf>=6.0.0
\ No newline at end of file
diff --git a/sbom/README.md b/sbom/README.md
new file mode 100644
index 0000000..9ee3873
--- /dev/null
+++ b/sbom/README.md
@@ -0,0 +1,13 @@
+# Software Bill of Materials (SBOM)
+
+This directory contains the CycloneDX SBOM for the project.
+
+## Disclaimer
+
+This SBOM is generated on a best-effort basis from project metadata and reflects dependencies at the time of generation. It is not a guarantee of completeness or accuracy.
+
+## Regenerating
+
+```bash
+./scripts/gen-sbom.sh
+```
diff --git a/sbom/sbom.cdx.json b/sbom/sbom.cdx.json
new file mode 100644
index 0000000..1e703fd
--- /dev/null
+++ b/sbom/sbom.cdx.json
@@ -0,0 +1 @@
+{"$schema":"http://cyclonedx.org/schema/bom-1.6.schema.json","bomFormat":"CycloneDX","specVersion":"1.6","serialNumber":"urn:uuid:03585831-4a90-4206-8589-7c4d89f6d58c","version":1,"metadata":{"timestamp":"2026-01-27T01:43:08Z","tools":{"components":[{"type":"application","author":"anchore","name":"syft","version":"1.40.1"}]},"component":{"bom-ref":"af63bd4c8601b7f1","type":"file","name":"."}},"components":[{"bom-ref":"pkg:github/ilshidur/action-discord@master?package-id=c3a62187caee5473","type":"library","name":"Ilshidur/action-discord","version":"master","cpe":"cpe:2.3:a:Ilshidur\\/action-discord:Ilshidur\\/action-discord:master:*:*:*:*:*:*:*","purl":"pkg:github/Ilshidur/action-discord@master","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:package:metadataType","value":"github-actions-use-statement"},{"name":"syft:cpe23","value":"cpe:2.3:a:Ilshidur\\/action-discord:Ilshidur\\/action_discord:master:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:Ilshidur\\/action_discord:Ilshidur\\/action-discord:master:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:Ilshidur\\/action_discord:Ilshidur\\/action_discord:master:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:Ilshidur\\/action:Ilshidur\\/action-discord:master:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:Ilshidur\\/action:Ilshidur\\/action_discord:master:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/main.yml"}]},{"bom-ref":"7a1423cc652b4afe","type":"application","name":"Simple Launcher","version":"1.1.0.14","cpe":"cpe:2.3:a:Simple_Launcher:Simple_Launcher:1.1.0.14:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pip/_vendor/distlib/t32.exe"}]},{"bom-ref":"6c56712f01b96651","type":"application","name":"Simple Launcher","version":"1.1.0.14","cpe":"cpe:2.3:a:Simple_Launcher:Simple_Launcher:1.1.0.14:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pip/_vendor/distlib/t64-arm.exe"}]},{"bom-ref":"29cb7ccd8d7b477f","type":"application","name":"Simple Launcher","version":"1.1.0.14","cpe":"cpe:2.3:a:Simple_Launcher:Simple_Launcher:1.1.0.14:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pip/_vendor/distlib/t64.exe"}]},{"bom-ref":"213d4a6800241b59","type":"application","name":"Simple Launcher","version":"1.1.0.14","cpe":"cpe:2.3:a:Simple_Launcher:Simple_Launcher:1.1.0.14:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pip/_vendor/distlib/w32.exe"}]},{"bom-ref":"a312964eec3cb9bd","type":"application","name":"Simple Launcher","version":"1.1.0.14","cpe":"cpe:2.3:a:Simple_Launcher:Simple_Launcher:1.1.0.14:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pip/_vendor/distlib/w64-arm.exe"}]},{"bom-ref":"d1f7f47d7476217e","type":"application","name":"Simple Launcher","version":"1.1.0.14","cpe":"cpe:2.3:a:Simple_Launcher:Simple_Launcher:1.1.0.14:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pip/_vendor/distlib/w64.exe"}]},{"bom-ref":"pkg:github/actions/checkout@v4?package-id=67e3327395f5aefa","type":"library","name":"actions/checkout","version":"v4","cpe":"cpe:2.3:a:actions\\/checkout:actions\\/checkout:v4:*:*:*:*:*:*:*","purl":"pkg:github/actions/checkout@v4","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:package:metadataType","value":"github-actions-use-statement"},{"name":"syft:location:0:path","value":"/.github/workflows/docker-release.yml"}]},{"bom-ref":"pkg:github/actions/checkout@v4?package-id=9eb4338500f07f9e","type":"library","name":"actions/checkout","version":"v4","cpe":"cpe:2.3:a:actions\\/checkout:actions\\/checkout:v4:*:*:*:*:*:*:*","purl":"pkg:github/actions/checkout@v4","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:package:metadataType","value":"github-actions-use-statement"},{"name":"syft:location:0:path","value":"/.github/workflows/release.yml"}]},{"bom-ref":"pkg:github/actions/setup-python@v5?package-id=c1420dcfcd7b84e1","type":"library","name":"actions/setup-python","version":"v5","cpe":"cpe:2.3:a:actions\\/setup-python:actions\\/setup-python:v5:*:*:*:*:*:*:*","purl":"pkg:github/actions/setup-python@v5","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:package:metadataType","value":"github-actions-use-statement"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup-python:actions\\/setup_python:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup_python:actions\\/setup-python:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup_python:actions\\/setup_python:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup:actions\\/setup-python:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:actions\\/setup:actions\\/setup_python:v5:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/release.yml"}]},{"bom-ref":"pkg:pypi/aiofiles@24.1.0?package-id=87d1c86c885117b6","type":"library","name":"aiofiles","version":"24.1.0","cpe":"cpe:2.3:a:python-aiofiles:python-aiofiles:24.1.0:*:*:*:*:*:*:*","purl":"pkg:pypi/aiofiles@24.1.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiofiles:python_aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiofiles:python-aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiofiles:python_aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiofiles:python-aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiofiles:python_aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiofiles:aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiofiles:aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiofiles:aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:aiofiles:24.1.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/aiofiles@25.1.0?package-id=1e64442043f26b0e","type":"library","author":"Tin Tvrtkovic ","name":"aiofiles","version":"25.1.0","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:tin_tvrtkovic_\\","name":"aiohappyeyeballs","version":"2.6.1","licenses":[{"license":{"id":"PSF-2.0"}}],"cpe":"cpe:2.3:a:python-aiohappyeyeballs:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*","purl":"pkg:pypi/aiohappyeyeballs@2.6.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiohappyeyeballs:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohappyeyeballs:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohappyeyeballs:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_koston_project:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_koston_project:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_kostonproject:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_kostonproject:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohappyeyeballs:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohappyeyeballs:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiohappyeyeballs:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohappyeyeballs:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_koston_project:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_koston:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_koston:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_kostonproject:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nick_project:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nick_project:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nickproject:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nickproject:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohappyeyeballs:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:j__nick_koston:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nick_project:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nick:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nick:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nickproject:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nick:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/aiohappyeyeballs-2.6.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/aiohappyeyeballs-2.6.1.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/aiohappyeyeballs@2.6.1?package-id=6f7476d9472e6d64","type":"library","name":"aiohappyeyeballs","version":"2.6.1","cpe":"cpe:2.3:a:python-aiohappyeyeballs:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*","purl":"pkg:pypi/aiohappyeyeballs@2.6.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiohappyeyeballs:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohappyeyeballs:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohappyeyeballs:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohappyeyeballs:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohappyeyeballs:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiohappyeyeballs:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohappyeyeballs:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohappyeyeballs:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:aiohappyeyeballs:2.6.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/aiohttp@3.12.13?package-id=7fea866e72d6c9bb","type":"library","name":"aiohttp","version":"3.12.13","cpe":"cpe:2.3:a:python-aiohttp:python-aiohttp:3.12.13:*:*:*:*:*:*:*","purl":"pkg:pypi/aiohttp@3.12.13","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiohttp:python_aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohttp:python-aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohttp:python_aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohttp:python-aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohttp:python_aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiohttp:aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohttp:aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohttp:aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:aiohttp:3.12.13:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/aiohttp@3.13.3?package-id=0eeb93580cf1583c","type":"library","name":"aiohttp","version":"3.13.3","licenses":[{"expression":"Apache-2.0 AND MIT"}],"cpe":"cpe:2.3:a:python-aiohttp:python-aiohttp:3.13.3:*:*:*:*:*:*:*","purl":"pkg:pypi/aiohttp@3.13.3","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiohttp:python_aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohttp:python-aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohttp:python_aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohttp:python-aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohttp:python_aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiohttp:aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiohttp:aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiohttp:aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:aiohttp:3.13.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/aiohttp-3.13.3.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/aiohttp-3.13.3.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/aiohttp-3.13.3.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/aiosignal@1.4.0?package-id=39c5b9811fe44208","type":"library","name":"aiosignal","version":"1.4.0","licenses":[{"license":{"name":"Apache 2.0"}}],"cpe":"cpe:2.3:a:python-aiosignal:python-aiosignal:1.4.0:*:*:*:*:*:*:*","purl":"pkg:pypi/aiosignal@1.4.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiosignal:python_aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosignal:python-aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosignal:python_aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosignal:python-aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosignal:python_aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiosignal:aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosignal:aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosignal:aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/aiosignal-1.4.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/aiosignal-1.4.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/aiosignal-1.4.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/aiosignal@1.4.0?package-id=d5a5ed1c60484452","type":"library","name":"aiosignal","version":"1.4.0","cpe":"cpe:2.3:a:python-aiosignal:python-aiosignal:1.4.0:*:*:*:*:*:*:*","purl":"pkg:pypi/aiosignal@1.4.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiosignal:python_aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosignal:python-aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosignal:python_aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosignal:python-aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosignal:python_aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiosignal:aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosignal:aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosignal:aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:aiosignal:1.4.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/aiosqlite@0.21.0?package-id=bbae5f9dfee6288b","type":"library","name":"aiosqlite","version":"0.21.0","cpe":"cpe:2.3:a:python-aiosqlite:python-aiosqlite:0.21.0:*:*:*:*:*:*:*","purl":"pkg:pypi/aiosqlite@0.21.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiosqlite:python_aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosqlite:python-aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosqlite:python_aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosqlite:python-aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosqlite:python_aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-aiosqlite:aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_aiosqlite:aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:aiosqlite:aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:aiosqlite:0.21.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/aiosqlite@0.22.1?package-id=bcdd3c4946a9db72","type":"library","author":"Amethyst Reese ","name":"aiosqlite","version":"0.22.1","cpe":"cpe:2.3:a:amethyst_reese_\\","name":"alphashape","version":"1.3.1","licenses":[{"license":{"name":"MIT license"}}],"cpe":"cpe:2.3:a:kenneth_e__bellock_project:python-alphashape:1.3.1:*:*:*:*:*:*:*","purl":"pkg:pypi/alphashape@1.3.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenneth_e__bellock_project:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenneth_e__bellockproject:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenneth_e__bellockproject:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenneth_e__bellock_project:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenneth_e__bellock:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenneth_e__bellock:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenneth_e__bellockproject:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-alphashape:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-alphashape:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_alphashape:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_alphashape:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ken_project:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ken_project:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenneth_e__bellock:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alphashape:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alphashape:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenproject:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenproject:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-alphashape:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_alphashape:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ken_project:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alphashape:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ken:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ken:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kenproject:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ken:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/alphashape-1.3.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/alphashape-1.3.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/alphashape-1.3.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/alphashape@1.3.1?package-id=8e8c1ca2202bb145","type":"library","name":"alphashape","version":"1.3.1","cpe":"cpe:2.3:a:python-alphashape:python-alphashape:1.3.1:*:*:*:*:*:*:*","purl":"pkg:pypi/alphashape@1.3.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-alphashape:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_alphashape:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_alphashape:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alphashape:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alphashape:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-alphashape:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_alphashape:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alphashape:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:alphashape:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/annotated-types@0.7.0?package-id=9705bf5db3cf47aa","type":"library","author":"Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>, Samuel Colvin , Zac Hatfield-Dodds ","name":"annotated-types","version":"0.7.0","cpe":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangb_project:python-annotated-types:0.7.0:*:*:*:*:*:*:*","purl":"pkg:pypi/annotated-types@0.7.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangb_project:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangbproject:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangbproject:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangb_project:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangb_project:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian-garcia-badaracco-\\<1755071\\+adriangb:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian-garcia-badaracco-\\<1755071\\+adriangb:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangb:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangb:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangbproject:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangbproject:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian-garcia-badaracco-\\<1755071\\+adriangb:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian-garcia-badaracco-\\<1755071\\+adriangb:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangb:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:adrian_garcia_badaracco_\\<1755071\\+adriangb:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated-types:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated-types:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated_types:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated_types:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated-types:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated-types:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated_types:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated_types:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated-types:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated-types:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated_types:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated_types:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated-types:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated-types:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated_types:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated_types:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/annotated_types-0.7.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/annotated_types-0.7.0.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/annotated-types@0.7.0?package-id=7fc5ac93eed59ada","type":"library","name":"annotated-types","version":"0.7.0","cpe":"cpe:2.3:a:python-annotated-types:python-annotated-types:0.7.0:*:*:*:*:*:*:*","purl":"pkg:pypi/annotated-types@0.7.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated-types:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated_types:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated_types:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated-types:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated-types:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated_types:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated_types:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated-types:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated-types:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated_types:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated_types:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-annotated:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_annotated:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated-types:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated-types:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated_types:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated_types:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:annotated:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:annotated-types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:annotated_types:0.7.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/anyio@4.12.1?package-id=52065d8c6127896a","type":"library","author":"Alex Grönholm ","name":"anyio","version":"4.12.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:python-anyio:python-anyio:4.12.1:*:*:*:*:*:*:*","purl":"pkg:pypi/anyio@4.12.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-anyio:python_anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:python-anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:python_anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:python-anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:python_anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-anyio:anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:anyio:4.12.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/anyio-4.12.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/anyio-4.12.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/anyio-4.12.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/anyio@4.9.0?package-id=4f3de70a0dd7f289","type":"library","name":"anyio","version":"4.9.0","cpe":"cpe:2.3:a:python-anyio:python-anyio:4.9.0:*:*:*:*:*:*:*","purl":"pkg:pypi/anyio@4.9.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-pip-requirements-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-anyio:python_anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:python-anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:python_anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:python-anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:python_anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-anyio:anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/deploy/docker/requirements.txt"}]},{"bom-ref":"pkg:pypi/anyio@4.9.0?package-id=d89d7370232e1a2f","type":"library","name":"anyio","version":"4.9.0","cpe":"cpe:2.3:a:python-anyio:python-anyio:4.9.0:*:*:*:*:*:*:*","purl":"pkg:pypi/anyio@4.9.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-anyio:python_anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:python-anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:python_anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:python-anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:python_anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-anyio:anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_anyio:anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anyio:anyio:4.9.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/async-timeout@5.0.1?package-id=39b040f5f8ff04af","type":"library","author":"Andrew Svetlov  ","name":"async-timeout","version":"5.0.1","licenses":[{"license":{"name":"Apache 2"}}],"cpe":"cpe:2.3:a:andrew_svetlov_\\_project:python-async-timeout:5.0.1:*:*:*:*:*:*:*","purl":"pkg:pypi/async-timeout@5.0.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\_project:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\project:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\project:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\_project:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\_project:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\project:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\project:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_\\:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async-timeout:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async-timeout:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async_timeout:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async_timeout:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async-timeout:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async-timeout:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async_timeout:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async_timeout:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async-timeout:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async-timeout:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async_timeout:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async_timeout:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async-timeout:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async-timeout:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async_timeout:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async_timeout:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/async_timeout-5.0.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/async_timeout-5.0.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/async_timeout-5.0.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/async-timeout@5.0.1?package-id=14cb75fb52aaef4c","type":"library","name":"async-timeout","version":"5.0.1","cpe":"cpe:2.3:a:python-async-timeout:python-async-timeout:5.0.1:*:*:*:*:*:*:*","purl":"pkg:pypi/async-timeout@5.0.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async-timeout:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async_timeout:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async_timeout:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async-timeout:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async-timeout:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async_timeout:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async_timeout:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async-timeout:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async-timeout:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async_timeout:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async_timeout:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async-timeout:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async-timeout:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async_timeout:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async_timeout:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async:python-async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async:python_async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-async:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_async:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async:async-timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:async:async_timeout:5.0.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/attrs@25.3.0?package-id=c2f6025c826e3e79","type":"library","name":"attrs","version":"25.3.0","cpe":"cpe:2.3:a:python-attrs:python-attrs:25.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/attrs@25.3.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-attrs:python_attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_attrs:python-attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_attrs:python_attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:attrs:python-attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:attrs:python_attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-attrs:attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_attrs:attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:attrs:attrs:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/attrs@25.4.0?package-id=1953b486d1e53d50","type":"library","author":"Hynek Schlawack ","name":"attrs","version":"25.4.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:hynek_schlawack_\\","name":"backports-asyncio-runner","version":"1.2.0","licenses":[{"license":{"id":"PSF-2.0"}}],"cpe":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100_project:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/backports-asyncio-runner@1.2.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100_project:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100project:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100project:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports-asyncio-runner:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports-asyncio-runner:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports_asyncio_runner:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports_asyncio_runner:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100_project:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100_project:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100-\\<3933065\\+samypr100:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100-\\<3933065\\+samypr100:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100project:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100project:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports-asyncio-runner:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports-asyncio-runner:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports_asyncio_runner:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports_asyncio_runner:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports-asyncio-runner:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports-asyncio-runner:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports-asyncio:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports-asyncio:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports_asyncio:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports_asyncio:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports_asyncio_runner:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports_asyncio_runner:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100-\\<3933065\\+samypr100:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100-\\<3933065\\+samypr100:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:samypr100_\\<3933065\\+samypr100:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports-asyncio-runner:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports-asyncio-runner:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports-asyncio:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports-asyncio:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports_asyncio:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports_asyncio:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports_asyncio_runner:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports_asyncio_runner:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports-asyncio:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports-asyncio:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports_asyncio:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports_asyncio:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports-asyncio:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports-asyncio:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports_asyncio:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports_asyncio:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-backports:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_backports:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:backports:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:backports-asyncio-runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:backports_asyncio_runner:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/backports_asyncio_runner-1.2.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/backports_asyncio_runner-1.2.0.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/beautifulsoup4@4.13.4?package-id=86e1999d0a39798f","type":"library","name":"beautifulsoup4","version":"4.13.4","cpe":"cpe:2.3:a:python-beautifulsoup4:python-beautifulsoup4:4.13.4:*:*:*:*:*:*:*","purl":"pkg:pypi/beautifulsoup4@4.13.4","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-beautifulsoup4:python_beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_beautifulsoup4:python-beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_beautifulsoup4:python_beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:beautifulsoup4:python-beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:beautifulsoup4:python_beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-beautifulsoup4:beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_beautifulsoup4:beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:beautifulsoup4:beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:beautifulsoup4:4.13.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/beautifulsoup4@4.14.3?package-id=c146856852c9cbfb","type":"library","author":"Leonard Richardson ","name":"beautifulsoup4","version":"4.14.3","licenses":[{"license":{"name":"MIT License"}}],"cpe":"cpe:2.3:a:leonard_richardson_\\","name":"certifi","version":"2026.1.4","licenses":[{"license":{"id":"MPL-2.0"}}],"cpe":"cpe:2.3:a:certifi:certifi:2026.1.4:*:*:*:*:python:*:*","purl":"pkg:pypi/certifi@2026.1.4","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/certifi-2026.1.4.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/certifi-2026.1.4.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/certifi-2026.1.4.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/cffi@2.0.0?package-id=59bb4aa36789c67d","type":"library","author":"Armin Rigo, Maciej Fijalkowski","name":"cffi","version":"2.0.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowski_project:python-cffi:2.0.0:*:*:*:*:*:*:*","purl":"pkg:pypi/cffi@2.0.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowski_project:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowskiproject:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowskiproject:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowski_project:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowski:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowski:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowskiproject:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:armin_rigo\\,_maciej_fijalkowski:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cffi:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cffi:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cffi:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cffi:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cffi:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cffi:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cffi:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cffi:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cffi:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/cffi-2.0.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/cffi-2.0.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/cffi-2.0.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/cffi@2.0.0?package-id=d611818916a9c440","type":"library","name":"cffi","version":"2.0.0","cpe":"cpe:2.3:a:python-cffi:python-cffi:2.0.0:*:*:*:*:*:*:*","purl":"pkg:pypi/cffi@2.0.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cffi:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cffi:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cffi:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cffi:python-cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cffi:python_cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cffi:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cffi:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cffi:cffi:2.0.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/chardet@5.2.0?package-id=8124df40818b4c75","type":"library","author":"Mark Pilgrim ","name":"chardet","version":"5.2.0","licenses":[{"license":{"name":"LGPL"}}],"cpe":"cpe:2.3:a:mark_pilgrim_project:python-chardet:5.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/chardet@5.2.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_pilgrim_project:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_pilgrimproject:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_pilgrimproject:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-chardet:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-chardet:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_chardet:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_chardet:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_pilgrim_project:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_pilgrim:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_pilgrim:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_pilgrimproject:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_project:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_project:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markproject:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markproject:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:chardet:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:chardet:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-chardet:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_chardet:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_pilgrim:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_project:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markproject:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:chardet:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/chardet-5.2.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/chardet-5.2.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/chardet-5.2.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/chardet@5.2.0?package-id=e0feccb55f16982f","type":"library","name":"chardet","version":"5.2.0","cpe":"cpe:2.3:a:python-chardet:python-chardet:5.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/chardet@5.2.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-chardet:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_chardet:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_chardet:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:chardet:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:chardet:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-chardet:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_chardet:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:chardet:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:chardet:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/charset-normalizer@3.4.2?package-id=6d979f112ca1f834","type":"library","name":"charset-normalizer","version":"3.4.2","cpe":"cpe:2.3:a:python-charset-normalizer:python-charset-normalizer:3.4.2:*:*:*:*:*:*:*","purl":"pkg:pypi/charset-normalizer@3.4.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-charset-normalizer:python_charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_charset_normalizer:python-charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_charset_normalizer:python_charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset-normalizer:python-charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset-normalizer:python_charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset_normalizer:python-charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset_normalizer:python_charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-charset-normalizer:charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-charset-normalizer:charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_charset_normalizer:charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_charset_normalizer:charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-charset:python-charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-charset:python_charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_charset:python-charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_charset:python_charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset-normalizer:charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset-normalizer:charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset_normalizer:charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset_normalizer:charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset:python-charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset:python_charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-charset:charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-charset:charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_charset:charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_charset:charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset:charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:charset:charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:charset-normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:charset_normalizer:3.4.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/charset-normalizer@3.4.4?package-id=2b55b66df7178d61","type":"library","author":"\"Ahmed R. TAHRI\" ","name":"charset-normalizer","version":"3.4.4","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:\\\"ahmed_r__tahri\\\"_\\","name":"click-log","version":"0.4.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:markus_unterwaditzer_project:python-click-log:0.4.0:*:*:*:*:*:*:*","purl":"pkg:pypi/click-log@0.4.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzer_project:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzerproject:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzerproject:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzer_project:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzer_project:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzer:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzer:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzerproject:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzerproject:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click-log:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click-log:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click_log:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click_log:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_project:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_project:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzer:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_unterwaditzer:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markusproject:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markusproject:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click-log:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click-log:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click_log:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click_log:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click-log:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click-log:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click_log:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click_log:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_project:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus_project:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markusproject:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markusproject:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click-log:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click-log:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click_log:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click_log:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markus:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/click_log-0.4.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/click_log-0.4.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/click_log-0.4.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/click-log@0.4.0?package-id=6d5d4b4f955ec7c1","type":"library","name":"click-log","version":"0.4.0","cpe":"cpe:2.3:a:python-click-log:python-click-log:0.4.0:*:*:*:*:*:*:*","purl":"pkg:pypi/click-log@0.4.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click-log:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click_log:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click_log:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click-log:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click-log:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click_log:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click_log:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click-log:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click-log:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click_log:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click_log:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click:python-click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click:python_click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-click:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_click:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click-log:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click-log:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click_log:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click_log:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click:click-log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:click:click_log:0.4.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/colorama@0.4.6?package-id=998a6b2b64fbdb5b","type":"library","name":"colorama","version":"0.4.6","cpe":"cpe:2.3:a:python-colorama:python-colorama:0.4.6:*:*:*:*:*:*:*","purl":"pkg:pypi/colorama@0.4.6","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-colorama:python_colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_colorama:python-colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_colorama:python_colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:colorama:python-colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:colorama:python_colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-colorama:colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_colorama:colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:colorama:colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:colorama:0.4.6:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/crawl4ai@0.7.8?package-id=e23d1711f25830c1","type":"library","author":"Unclecode >","name":"crawl4ai","version":"0.7.8","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:unclecode_\\>","name":"crawl4ai","version":"0.7.8","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:unclecode_\\","name":"cryptography","version":"46.0.3","licenses":[{"expression":"Apache-2.0 OR BSD-3-Clause"}],"cpe":"cpe:2.3:a:cryptography.io:cryptography:46.0.3:*:*:*:*:python:*:*","purl":"pkg:pypi/cryptography@46.0.3","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography.io:cryptography:46.0.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/cryptography-46.0.3.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/cryptography-46.0.3.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/cryptography@46.0.3?package-id=dc29b162c0d67a6b","type":"library","name":"cryptography","version":"46.0.3","cpe":"cpe:2.3:a:cryptography.io:cryptography:46.0.3:*:*:*:*:python:*:*","purl":"pkg:pypi/cryptography@46.0.3","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography.io:cryptography:46.0.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/cssselect@1.3.0?package-id=489e617efabbb144","type":"library","author":"Ian Bicking ","name":"cssselect","version":"1.3.0","licenses":[{"license":{"name":"BSD"}}],"cpe":"cpe:2.3:a:ian_bicking_project:python-cssselect:1.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/cssselect@1.3.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:ian_bicking_project:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ian_bickingproject:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ian_bickingproject:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cssselect:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cssselect:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cssselect:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cssselect:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ian_bicking_project:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianb_project:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianb_project:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ian_bicking:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ian_bicking:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ian_bickingproject:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianbproject:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianbproject:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cssselect:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cssselect:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cssselect:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cssselect:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianb_project:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ian_bicking:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianb:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianb:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianbproject:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cssselect:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ianb:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/cssselect-1.3.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/cssselect-1.3.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/cssselect-1.3.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/cssselect@1.3.0?package-id=c04fa70d6c55720e","type":"library","name":"cssselect","version":"1.3.0","cpe":"cpe:2.3:a:python-cssselect:python-cssselect:1.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/cssselect@1.3.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cssselect:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cssselect:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cssselect:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cssselect:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cssselect:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-cssselect:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_cssselect:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cssselect:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:cssselect:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/distro@1.9.0?package-id=d253d3c673ba1dc1","type":"library","author":"Nir Cohen ","name":"distro","version":"1.9.0","licenses":[{"license":{"name":"Apache License, Version 2.0"}}],"cpe":"cpe:2.3:a:nir_cohen_project:python-distro:1.9.0:*:*:*:*:*:*:*","purl":"pkg:pypi/distro@1.9.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir_cohen_project:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir_cohenproject:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir_cohenproject:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36g_project:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36g_project:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36gproject:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36gproject:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-distro:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-distro:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_distro:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_distro:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir_cohen_project:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir_cohen:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir_cohen:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir_cohenproject:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36g_project:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:distro:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:distro:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36g:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36g:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36gproject:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-distro:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_distro:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir_cohen:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:distro:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nir36g:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/distro-1.9.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/distro-1.9.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/distro-1.9.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/distro@1.9.0?package-id=cd7f2e57b478f342","type":"library","name":"distro","version":"1.9.0","cpe":"cpe:2.3:a:python-distro:python-distro:1.9.0:*:*:*:*:*:*:*","purl":"pkg:pypi/distro@1.9.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-distro:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_distro:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_distro:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:distro:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:distro:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-distro:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_distro:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:distro:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:distro:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:github/docker/build-push-action@v5?package-id=d8b6fab7a5d50874","type":"library","name":"docker/build-push-action","version":"v5","cpe":"cpe:2.3:a:docker\\/build-push-action:docker\\/build-push-action:v5:*:*:*:*:*:*:*","purl":"pkg:github/docker/build-push-action@v5","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:package:metadataType","value":"github-actions-use-statement"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build-push-action:docker\\/build_push_action:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build_push_action:docker\\/build-push-action:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build_push_action:docker\\/build_push_action:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build-push:docker\\/build-push-action:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build-push:docker\\/build_push_action:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build_push:docker\\/build-push-action:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build_push:docker\\/build_push_action:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build:docker\\/build-push-action:v5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/build:docker\\/build_push_action:v5:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/docker-release.yml"}]},{"bom-ref":"pkg:github/docker/login-action@v3?package-id=b2eb49bc9c24511c","type":"library","name":"docker/login-action","version":"v3","cpe":"cpe:2.3:a:docker\\/login-action:docker\\/login-action:v3:*:*:*:*:*:*:*","purl":"pkg:github/docker/login-action@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:package:metadataType","value":"github-actions-use-statement"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/login-action:docker\\/login_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/login_action:docker\\/login-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/login_action:docker\\/login_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/login:docker\\/login-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/login:docker\\/login_action:v3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/docker-release.yml"}]},{"bom-ref":"pkg:github/docker/setup-buildx-action@v3?package-id=3a12327f79cbbb38","type":"library","name":"docker/setup-buildx-action","version":"v3","cpe":"cpe:2.3:a:docker\\/setup-buildx-action:docker\\/setup-buildx-action:v3:*:*:*:*:*:*:*","purl":"pkg:github/docker/setup-buildx-action@v3","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:package:metadataType","value":"github-actions-use-statement"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup-buildx-action:docker\\/setup_buildx_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup_buildx_action:docker\\/setup-buildx-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup_buildx_action:docker\\/setup_buildx_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup-buildx:docker\\/setup-buildx-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup-buildx:docker\\/setup_buildx_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup_buildx:docker\\/setup-buildx-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup_buildx:docker\\/setup_buildx_action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup:docker\\/setup-buildx-action:v3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:docker\\/setup:docker\\/setup_buildx_action:v3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/docker-release.yml"}]},{"bom-ref":"pkg:pypi/email-validator@2.2.0?package-id=5d83fd52feba2015","type":"library","name":"email-validator","version":"2.2.0","cpe":"cpe:2.3:a:python-email-validator:python-email-validator:2.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/email-validator@2.2.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-pip-requirements-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-email-validator:python_email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_email_validator:python-email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_email_validator:python_email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email-validator:python-email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email-validator:python_email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email_validator:python-email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email_validator:python_email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-email-validator:email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-email-validator:email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_email_validator:email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_email_validator:email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-email:python-email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-email:python_email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_email:python-email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_email:python_email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email-validator:email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email-validator:email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email_validator:email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email_validator:email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email:python-email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email:python_email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-email:email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-email:email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_email:email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_email:email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email:email-validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:email:email_validator:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/deploy/docker/requirements.txt"}]},{"bom-ref":"pkg:pypi/exceptiongroup@1.3.0?package-id=54b9dd9a5ea38f09","type":"library","name":"exceptiongroup","version":"1.3.0","cpe":"cpe:2.3:a:python-exceptiongroup:python-exceptiongroup:1.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/exceptiongroup@1.3.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-exceptiongroup:python_exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_exceptiongroup:python-exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_exceptiongroup:python_exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:exceptiongroup:python-exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:exceptiongroup:python_exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-exceptiongroup:exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_exceptiongroup:exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:exceptiongroup:exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:exceptiongroup:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/exceptiongroup@1.3.1?package-id=d458176bf0f02c89","type":"library","author":"Alex Grönholm ","name":"exceptiongroup","version":"1.3.1","cpe":"cpe:2.3:a:python-exceptiongroup:python-exceptiongroup:1.3.1:*:*:*:*:*:*:*","purl":"pkg:pypi/exceptiongroup@1.3.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-exceptiongroup:python_exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_exceptiongroup:python-exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_exceptiongroup:python_exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:exceptiongroup:python-exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:exceptiongroup:python_exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-exceptiongroup:exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_exceptiongroup:exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:exceptiongroup:exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:exceptiongroup:1.3.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/exceptiongroup-1.3.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/exceptiongroup-1.3.1.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/fake-http-header@0.3.5?package-id=2bde71aa1630306c","type":"library","author":"Michael Tatarski ","name":"fake-http-header","version":"0.3.5","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:michael_tatarski_project:python-fake-http-header:0.3.5:*:*:*:*:*:*:*","purl":"pkg:pypi/fake-http-header@0.3.5","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarski_project:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarskiproject:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarskiproject:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarski_project:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarski_project:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http-header:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http-header:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http_header:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http_header:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarskiproject:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarskiproject:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarski_project:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarski_project:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http-header:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http-header:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http_header:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http_header:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarski:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarski:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarskiproject:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarskiproject:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarski_project:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarski_project:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http-header:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http-header:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http_header:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http_header:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarski:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarski:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarskiproject:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarskiproject:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http-header:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http-header:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http_header:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http_header:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarski:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michael_tatarski:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarski:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:michaeltatarski:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/fake_http_header-0.3.5.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/fake_http_header-0.3.5.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/fake_http_header-0.3.5.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/fake-http-header@0.3.5?package-id=75b3f3578786f229","type":"library","name":"fake-http-header","version":"0.3.5","cpe":"cpe:2.3:a:python-fake-http-header:python-fake-http-header:0.3.5:*:*:*:*:*:*:*","purl":"pkg:pypi/fake-http-header@0.3.5","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http-header:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http_header:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http_header:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http-header:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http-header:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http_header:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http_header:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http-header:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http-header:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http_header:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http_header:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http-header:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http-header:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http_header:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http_header:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake-http:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake_http:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake:python-fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake:python_fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-fake:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_fake:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake-http:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake_http:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake:fake-http-header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fake:fake_http_header:0.3.5:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/fake-useragent@2.2.0?package-id=b21d5b517dd2ff55","type":"library","author":"Melroy van den Berg , Victor Kovtun ","name":"fake-useragent","version":"2.2.0","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:melroy_van_den_berg_\\","name":"greenlet","version":"3.3.0","licenses":[{"expression":"MIT AND Python-2.0"}],"cpe":"cpe:2.3:a:alexey_borzenkov_project:python-greenlet:3.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/greenlet@3.3.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:alexey_borzenkov_project:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alexey_borzenkovproject:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alexey_borzenkovproject:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alexey_borzenkov_project:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alexey_borzenkov:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alexey_borzenkov:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alexey_borzenkovproject:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-greenlet:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-greenlet:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_greenlet:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_greenlet:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snaury_project:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snaury_project:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snauryproject:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snauryproject:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:alexey_borzenkov:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:greenlet:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:greenlet:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-greenlet:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_greenlet:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snaury_project:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snaury:python-greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snaury:python_greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snauryproject:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:greenlet:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snaury:greenlet:3.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/greenlet-3.3.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/greenlet-3.3.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/greenlet-3.3.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/grpcio@1.76.0?package-id=0b46c3cc7c4ef0a4","type":"library","author":"The gRPC Authors ","name":"grpcio","version":"1.76.0","licenses":[{"license":{"name":"Apache License 2.0"}}],"cpe":"cpe:2.3:a:grpc_authors_project:python-grpcio:1.76.0:*:*:*:*:*:*:*","purl":"pkg:pypi/grpcio@1.76.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_authors_project:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_authorsproject:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_authorsproject:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_io_project:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_io_project:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_ioproject:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_ioproject:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_authors_project:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-grpcio:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-grpcio:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_grpcio:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_grpcio:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_authors:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_authors:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_authorsproject:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_io_project:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc-io:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc-io:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_io:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_io:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_ioproject:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpcio:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpcio:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-grpcio:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_grpcio:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_authors:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc-io:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpc_io:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grpcio:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:grpcio:1.76.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/grpcio-1.76.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/grpcio-1.76.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/grpcio-1.76.0.dist-info/top_level.txt"}]},{"bom-ref":"432cdc4fc73f0246","type":"application","name":"gui","version":"UNKNOWN","cpe":"cpe:2.3:a:gui:gui:*:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/setuptools/gui.exe"}]},{"bom-ref":"b4fd8c63ee46844b","type":"application","name":"gui-32","version":"UNKNOWN","cpe":"cpe:2.3:a:gui-32:gui-32:*:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-32:gui_32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_32:gui-32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_32:gui_32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-32:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-32:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-:gui-32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-:gui_32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_32:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_32:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_:gui-32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_:gui_32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui-32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui_32:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/setuptools/gui-32.exe"}]},{"bom-ref":"2179238356db6e05","type":"application","name":"gui-64","version":"UNKNOWN","cpe":"cpe:2.3:a:gui-64:gui-64:*:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-64:gui_64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_64:gui-64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_64:gui_64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-64:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-64:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-:gui-64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-:gui_64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_64:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_64:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_:gui-64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_:gui_64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui-64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui_64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui-:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui_:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/setuptools/gui-64.exe"}]},{"bom-ref":"ee96561f47d69cd5","type":"application","name":"gui-arm64","version":"UNKNOWN","cpe":"cpe:2.3:a:gui-arm64:gui-arm64:*:*:*:*:*:*:*:*","properties":[{"name":"syft:package:foundBy","value":"pe-binary-package-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"pe-binary"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-arm64:gui_arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_arm64:gui-arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_arm64:gui_arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-arm64:gui-arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-arm64:gui_arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-arm:gui-arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-arm:gui_arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_arm64:gui-arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_arm64:gui_arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_arm:gui-arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_arm:gui_arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-arm:gui-arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui-arm:gui_arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_arm:gui-arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui_arm:gui_arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui-arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui_arm64:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui-arm:*:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:gui:gui_arm:*:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/setuptools/gui-arm64.exe"}]},{"bom-ref":"pkg:pypi/h11@0.16.0?package-id=0185242b33830fbf","type":"library","author":"Nathaniel J. Smith ","name":"h11","version":"0.16.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:nathaniel_j__smith_project:python-h11:0.16.0:*:*:*:*:*:*:*","purl":"pkg:pypi/h11@0.16.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nathaniel_j__smith_project:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nathaniel_j__smithproject:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nathaniel_j__smithproject:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nathaniel_j__smith_project:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nathaniel_j__smith:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nathaniel_j__smith:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nathaniel_j__smithproject:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nathaniel_j__smith:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njs_project:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njs_project:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njsproject:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njsproject:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-h11:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-h11:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h11:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h11:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njs_project:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h11:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h11:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njs:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njs:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njsproject:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-h11:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h11:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h11:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:njs:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/h11-0.16.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/h11-0.16.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/h11-0.16.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/h11@0.16.0?package-id=648440d8bad0175e","type":"library","name":"h11","version":"0.16.0","cpe":"cpe:2.3:a:python-h11:python-h11:0.16.0:*:*:*:*:*:*:*","purl":"pkg:pypi/h11@0.16.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-h11:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h11:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h11:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h11:python-h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h11:python_h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-h11:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h11:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h11:h11:0.16.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/h2@4.2.0?package-id=45c4b858b553293f","type":"library","name":"h2","version":"4.2.0","cpe":"cpe:2.3:a:python-h2:python-h2:4.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/h2@4.2.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-h2:python_h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h2:python-h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h2:python_h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h2:python-h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h2:python_h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-h2:h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_h2:h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:h2:h2:4.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/h2@4.3.0?package-id=805e14a246b074cf","type":"library","author":"Cory Benfield ","name":"h2","version":"4.3.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:cory_benfield_\\","name":"hpack","version":"4.1.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:cory_benfield_\\","name":"httpcore","version":"1.0.9","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:tom_christie_\\","name":"httpx","version":"0.28.1","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:encode:httpx:0.28.1:*:*:*:*:python:*:*","purl":"pkg:pypi/httpx@0.28.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/httpx-0.28.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/httpx-0.28.1.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/httpx@0.28.1?package-id=0e3e980ce70c21ff","type":"library","name":"httpx","version":"0.28.1","cpe":"cpe:2.3:a:encode:httpx:0.28.1:*:*:*:*:python:*:*","purl":"pkg:pypi/httpx@0.28.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/huggingface-hub@0.33.2?package-id=5cce22e40aa79bfa","type":"library","name":"huggingface-hub","version":"0.33.2","cpe":"cpe:2.3:a:python-huggingface-hub:python-huggingface-hub:0.33.2:*:*:*:*:*:*:*","purl":"pkg:pypi/huggingface-hub@0.33.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface-hub:python_huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface_hub:python-huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface_hub:python_huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface:python-huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface:python_huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface:python-huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface:python_huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface-hub:python-huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface-hub:python_huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface_hub:python-huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface_hub:python_huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface-hub:huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface-hub:huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface_hub:huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface_hub:huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface:python-huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface:python_huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface:huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface:huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface:huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface:huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface-hub:huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface-hub:huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface_hub:huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface_hub:huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface:huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface:huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:huggingface-hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:huggingface_hub:0.33.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/huggingface-hub@0.36.0?package-id=fd0eff543e96fc27","type":"library","author":"Hugging Face, Inc. ","name":"huggingface-hub","version":"0.36.0","licenses":[{"license":{"name":"Apache"}}],"cpe":"cpe:2.3:a:hugging_face\\,_inc__project:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*","purl":"pkg:pypi/huggingface-hub@0.36.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc__project:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc_project:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc_project:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface-hub:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface-hub:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface_hub:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface_hub:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc__project:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc__project:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc_:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc_:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc_project:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc_project:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface-hub:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface-hub:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface_hub:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface_hub:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface-hub:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface-hub:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface_hub:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface_hub:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julien_project:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julien_project:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julienproject:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julienproject:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc_:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face\\,_inc_:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-huggingface:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_huggingface:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface-hub:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface-hub:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface_hub:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface_hub:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julien_project:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julien_project:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julien:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julien:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julienproject:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julienproject:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:huggingface:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julien:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:julien:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:huggingface-hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:huggingface_hub:0.36.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/huggingface_hub-0.36.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/huggingface_hub-0.36.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/huggingface_hub-0.36.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/humanize@4.12.3?package-id=8100b70a76fc1df9","type":"library","name":"humanize","version":"4.12.3","cpe":"cpe:2.3:a:python-humanize:python-humanize:4.12.3:*:*:*:*:*:*:*","purl":"pkg:pypi/humanize@4.12.3","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-humanize:python_humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_humanize:python-humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_humanize:python_humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:humanize:python-humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:humanize:python_humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-humanize:humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_humanize:humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:humanize:humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:humanize:4.12.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/humanize@4.15.0?package-id=5e4190482b3f392b","type":"library","author":"Jason Moiron ","name":"humanize","version":"4.15.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:jason_moiron_\\","name":"hyperframe","version":"6.1.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:cory_benfield_\\","name":"idna","version":"3.11","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:kim_davies_\\","name":"importlib-metadata","version":"8.7.1","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:\\\"jason_r__coombs\\\"_\\, Holger Krekel ","name":"iniconfig","version":"2.3.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:ronny_pfannschmidt_\\","name":"jiter","version":"0.12.0","cpe":"cpe:2.3:a:samuel_colvin_\\","name":"joblib","version":"1.5.3","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:gael_varoquaux_\\","name":"jsonschema","version":"4.26.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:julian_berman_\\","name":"jsonschema-specifications","version":"2025.9.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:julian_berman_\\","name":"lark","version":"1.3.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:erez_shinan_\\","name":"lxml","version":"5.4.0","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:lxml:lxml:5.4.0:*:*:*:*:*:*:*","purl":"pkg:pypi/lxml@5.4.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/lxml-5.4.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/lxml-5.4.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/lxml-5.4.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/lxml@5.4.0?package-id=743df38347bf0bf8","type":"library","name":"lxml","version":"5.4.0","cpe":"cpe:2.3:a:lxml:lxml:5.4.0:*:*:*:*:*:*:*","purl":"pkg:pypi/lxml@5.4.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/markdown-it-py@3.0.0?package-id=e34ba28c39b83d00","type":"library","name":"markdown-it-py","version":"3.0.0","cpe":"cpe:2.3:a:python-markdown-it-py:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*","purl":"pkg:pypi/markdown-it-py@3.0.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown-it-py:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown_it_py:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown_it_py:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown-it:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown-it:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown_it:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown_it:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown-it-py:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown-it-py:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown_it_py:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown_it_py:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown-it-py:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown-it-py:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown_it_py:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown_it_py:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown-it:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown-it:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown_it:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown_it:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown-it:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown-it:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown_it:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown_it:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-markdown:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_markdown:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown-it-py:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown-it-py:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown_it_py:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown_it_py:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown-it:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown-it:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown_it:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown_it:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:markdown:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:markdown-it-py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:markdown_it_py:3.0.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/markdown-it-py@4.0.0?package-id=976e5d7c2ec4cdaf","type":"library","author":"Chris Sewell ","name":"markdown-it-py","version":"4.0.0","cpe":"cpe:2.3:a:chris_sewell_\\","name":"mdurl","version":"0.1.2","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:taneli_hukkinen_\\","name":"mpmath","version":"1.3.0","licenses":[{"license":{"name":"BSD"}}],"cpe":"cpe:2.3:a:fredrik_johansson_project:python-mpmath:1.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/mpmath@1.3.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik_johansson_project:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik_johanssonproject:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik_johanssonproject:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik_johansson_project:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik-johansson:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik-johansson:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik_johansson:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik_johansson:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik_johanssonproject:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-mpmath:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-mpmath:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_mpmath:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_mpmath:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik-johansson:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:fredrik_johansson:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mpmath:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mpmath:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-mpmath:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_mpmath:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mpmath:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/mpmath-1.3.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/mpmath-1.3.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/mpmath-1.3.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/mpmath@1.3.0?package-id=0f1054733df92f07","type":"library","name":"mpmath","version":"1.3.0","cpe":"cpe:2.3:a:python-mpmath:python-mpmath:1.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/mpmath@1.3.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-mpmath:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_mpmath:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_mpmath:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mpmath:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mpmath:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-mpmath:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_mpmath:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mpmath:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:mpmath:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/multidict@6.6.3?package-id=3d925171559344bf","type":"library","name":"multidict","version":"6.6.3","cpe":"cpe:2.3:a:python-multidict:python-multidict:6.6.3:*:*:*:*:*:*:*","purl":"pkg:pypi/multidict@6.6.3","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-multidict:python_multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_multidict:python-multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_multidict:python_multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:multidict:python-multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:multidict:python_multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-multidict:multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_multidict:multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:multidict:multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:multidict:6.6.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/multidict@6.7.0?package-id=77f67584dc6f5cf8","type":"library","author":"Andrew Svetlov ","name":"multidict","version":"6.7.0","licenses":[{"license":{"name":"Apache License 2.0"}}],"cpe":"cpe:2.3:a:andrew_svetlov_project:python-multidict:6.7.0:*:*:*:*:*:*:*","purl":"pkg:pypi/multidict@6.7.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:python_multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:python-multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:python_multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-multidict:python-multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-multidict:python_multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_multidict:python-multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_multidict:python_multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:python-multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:python_multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:python-multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:python_multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:multidict:python-multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:multidict:python_multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-multidict:multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_multidict:multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:multidict:multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:multidict:6.7.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/multidict-6.7.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/multidict-6.7.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/multidict-6.7.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/networkx@3.2.1?package-id=0b5406feb7fa859d","type":"library","name":"networkx","version":"3.2.1","cpe":"cpe:2.3:a:python:networkx:3.2.1:*:*:*:*:*:*:*","purl":"pkg:pypi/networkx@3.2.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/networkx@3.4.2?package-id=66685219e6944e23","type":"library","author":"Aric Hagberg ","name":"networkx","version":"3.4.2","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:python:networkx:3.4.2:*:*:*:*:*:*:*","purl":"pkg:pypi/networkx@3.4.2","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/networkx-3.4.2.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/networkx-3.4.2.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/networkx-3.4.2.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nltk@3.9.1?package-id=f60c12b1bcd7c95f","type":"library","name":"nltk","version":"3.9.1","cpe":"cpe:2.3:a:python-nltk:python-nltk:3.9.1:*:*:*:*:*:*:*","purl":"pkg:pypi/nltk@3.9.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nltk:python_nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nltk:python-nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nltk:python_nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk:python-nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk:python_nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nltk:nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nltk:nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk:nltk:3.9.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nltk@3.9.2?package-id=2de459b98d0f601d","type":"library","author":"NLTK Team ","name":"nltk","version":"3.9.2","licenses":[{"license":{"name":"Apache License, Version 2.0"}}],"cpe":"cpe:2.3:a:nltk_team_project:python-nltk:3.9.2:*:*:*:*:*:*:*","purl":"pkg:pypi/nltk@3.9.2","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk_team_project:python_nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk_teamproject:python-nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk_teamproject:python_nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nltk:python-nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nltk:python_nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nltk:python-nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nltk:python_nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk_team_project:nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk-team:python-nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk-team:python_nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk_team:python-nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk_team:python_nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk_teamproject:nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk:python-nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk:python_nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nltk:nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nltk:nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk-team:nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk_team:nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nltk:nltk:3.9.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nltk-3.9.2.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nltk-3.9.2.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nltk-3.9.2.dist-info/top_level.txt"}]},{"bom-ref":"pkg:generic/node@24.11.1?package-id=3008818bb687b088","type":"application","name":"node","version":"24.11.1","cpe":"cpe:2.3:a:nodejs:node.js:24.11.1:*:*:*:*:*:*:*","purl":"pkg:generic/node@24.11.1","properties":[{"name":"syft:package:foundBy","value":"binary-classifier-cataloger"},{"name":"syft:package:type","value":"binary"},{"name":"syft:package:metadataType","value":"binary-signature"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/patchright/driver/node"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/playwright/driver/node"}]},{"bom-ref":"pkg:pypi/numpy@2.0.2?package-id=5c4185ec9e439822","type":"library","name":"numpy","version":"2.0.2","cpe":"cpe:2.3:a:python-numpy:python-numpy:2.0.2:*:*:*:*:*:*:*","purl":"pkg:pypi/numpy@2.0.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-numpy:python_numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_numpy:python-numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_numpy:python_numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:numpy:python-numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:numpy:python_numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-numpy:numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_numpy:numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:numpy:numpy:2.0.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/numpy@2.2.6?package-id=eb3d0a3424ea172b","type":"library","author":"Travis E. Oliphant et al.","name":"numpy","version":"2.2.6","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:travis_e__oliphant_et_al__project:python-numpy:2.2.6:*:*:*:*:*:*:*","purl":"pkg:pypi/numpy@2.2.6","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:travis_e__oliphant_et_al__project:python_numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:travis_e__oliphant_et_al_project:python-numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:travis_e__oliphant_et_al_project:python_numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:travis_e__oliphant_et_al__project:numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:travis_e__oliphant_et_al_:python-numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:travis_e__oliphant_et_al_:python_numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:travis_e__oliphant_et_al_project:numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:travis_e__oliphant_et_al_:numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-numpy:python-numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-numpy:python_numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_numpy:python-numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_numpy:python_numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:numpy:python-numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:numpy:python_numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-numpy:numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_numpy:numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:numpy:numpy:2.2.6:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/numpy-2.2.6.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/numpy-2.2.6.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/nvidia-cublas-cu12@12.6.4.1?package-id=f0be6763601c5e92","type":"library","name":"nvidia-cublas-cu12","version":"12.6.4.1","cpe":"cpe:2.3:a:python-nvidia-cublas-cu12:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cublas-cu12@12.6.4.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas-cu12:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas_cu12:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas_cu12:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas-cu12:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas-cu12:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas_cu12:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas_cu12:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas-cu12:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas-cu12:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas_cu12:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas_cu12:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas-cu12:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas-cu12:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas_cu12:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas_cu12:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cublas-cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cublas_cu12:12.6.4.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cublas-cu12@12.8.4.1?package-id=0114f17d95ba21da","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cublas-cu12","version":"12.8.4.1","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cublas-cu12@12.8.4.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas-cu12:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas-cu12:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas_cu12:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas_cu12:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas-cu12:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas-cu12:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas_cu12:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas_cu12:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas-cu12:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas-cu12:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas_cu12:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas_cu12:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cublas:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cublas:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas-cu12:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas-cu12:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas_cu12:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas_cu12:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cublas:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cublas:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cublas-cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cublas_cu12:12.8.4.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cublas_cu12-12.8.4.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cublas_cu12-12.8.4.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cublas_cu12-12.8.4.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cuda-cupti-cu12@12.6.80?package-id=082532b3fc4bbaf0","type":"library","name":"nvidia-cuda-cupti-cu12","version":"12.6.80","cpe":"cpe:2.3:a:python-nvidia-cuda-cupti-cu12:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cuda-cupti-cu12@12.6.80","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti-cu12:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti_cu12:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti_cu12:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti-cu12:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti-cu12:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti_cu12:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti_cu12:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti-cu12:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti-cu12:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti_cu12:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti_cu12:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti-cu12:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti-cu12:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti_cu12:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti_cu12:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cuda-cupti-cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cuda_cupti_cu12:12.6.80:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cuda-cupti-cu12@12.8.90?package-id=7a0a2260e31e859b","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cuda-cupti-cu12","version":"12.8.90","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cuda-cupti-cu12@12.8.90","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti-cu12:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti-cu12:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti_cu12:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti_cu12:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti-cu12:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti-cu12:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti_cu12:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti_cu12:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti-cu12:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti-cu12:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti_cu12:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti_cu12:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-cupti:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_cupti:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti-cu12:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti-cu12:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti_cu12:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti_cu12:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-cupti:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_cupti:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cuda-cupti-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cuda_cupti_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_cupti_cu12-12.8.90.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_cupti_cu12-12.8.90.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_cupti_cu12-12.8.90.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cuda-nvrtc-cu12@12.6.77?package-id=fbccf85b04731e47","type":"library","name":"nvidia-cuda-nvrtc-cu12","version":"12.6.77","cpe":"cpe:2.3:a:python-nvidia-cuda-nvrtc-cu12:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cuda-nvrtc-cu12@12.6.77","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc-cu12:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc_cu12:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc_cu12:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc-cu12:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc-cu12:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc_cu12:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc_cu12:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc-cu12:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc-cu12:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc_cu12:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc_cu12:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc-cu12:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc-cu12:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc_cu12:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc_cu12:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cuda-nvrtc-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cuda_nvrtc_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cuda-nvrtc-cu12@12.8.93?package-id=ec503355c3eb09ef","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cuda-nvrtc-cu12","version":"12.8.93","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cuda-nvrtc-cu12@12.8.93","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc-cu12:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc-cu12:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc_cu12:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc_cu12:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc-cu12:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc-cu12:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc_cu12:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc_cu12:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc-cu12:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc-cu12:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc_cu12:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc_cu12:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-nvrtc:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_nvrtc:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc-cu12:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc-cu12:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc_cu12:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc_cu12:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-nvrtc:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_nvrtc:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cuda-nvrtc-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cuda_nvrtc_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_nvrtc_cu12-12.8.93.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_nvrtc_cu12-12.8.93.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_nvrtc_cu12-12.8.93.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cuda-runtime-cu12@12.6.77?package-id=3def8c4288c1d8d3","type":"library","name":"nvidia-cuda-runtime-cu12","version":"12.6.77","cpe":"cpe:2.3:a:python-nvidia-cuda-runtime-cu12:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cuda-runtime-cu12@12.6.77","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime-cu12:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime_cu12:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime_cu12:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime-cu12:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime-cu12:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime_cu12:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime_cu12:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime-cu12:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime-cu12:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime_cu12:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime_cu12:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime-cu12:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime-cu12:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime_cu12:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime_cu12:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cuda-runtime-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cuda_runtime_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cuda-runtime-cu12@12.8.90?package-id=974be3b4dd9b0360","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cuda-runtime-cu12","version":"12.8.90","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cuda-runtime-cu12@12.8.90","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime-cu12:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime-cu12:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime_cu12:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime_cu12:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime-cu12:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime-cu12:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime_cu12:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime_cu12:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime-cu12:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime-cu12:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime_cu12:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime_cu12:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda-runtime:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda_runtime:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime-cu12:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime-cu12:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime_cu12:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime_cu12:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda-runtime:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_runtime:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cuda:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cuda:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cuda:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cuda-runtime-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cuda_runtime_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_runtime_cu12-12.8.90.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_runtime_cu12-12.8.90.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cuda_runtime_cu12-12.8.90.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cudnn-cu12@9.10.2.21?package-id=e5c00b000228c32f","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cudnn-cu12","version":"9.10.2.21","licenses":[{"expression":"LicenseRef-NVIDIA-Proprietary"}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cudnn-cu12@9.10.2.21","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn-cu12:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn-cu12:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn_cu12:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn_cu12:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn-cu12:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn-cu12:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn_cu12:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn_cu12:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn-cu12:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn-cu12:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn_cu12:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn_cu12:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn-cu12:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn-cu12:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn_cu12:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn_cu12:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cudnn-cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cudnn_cu12:9.10.2.21:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cudnn_cu12-9.10.2.21.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cudnn_cu12-9.10.2.21.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cudnn_cu12-9.10.2.21.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cudnn-cu12@9.5.1.17?package-id=b902163efb60efe3","type":"library","name":"nvidia-cudnn-cu12","version":"9.5.1.17","cpe":"cpe:2.3:a:python-nvidia-cudnn-cu12:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cudnn-cu12@9.5.1.17","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn-cu12:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn_cu12:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn_cu12:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn-cu12:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn-cu12:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn_cu12:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn_cu12:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn-cu12:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn-cu12:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn_cu12:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn_cu12:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cudnn:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cudnn:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn-cu12:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn-cu12:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn_cu12:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn_cu12:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cudnn:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cudnn:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cudnn-cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cudnn_cu12:9.5.1.17:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cufft-cu12@11.3.0.4?package-id=83312b4626297aad","type":"library","name":"nvidia-cufft-cu12","version":"11.3.0.4","cpe":"cpe:2.3:a:python-nvidia-cufft-cu12:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cufft-cu12@11.3.0.4","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft-cu12:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft_cu12:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft_cu12:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft-cu12:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft-cu12:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft_cu12:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft_cu12:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft-cu12:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft-cu12:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft_cu12:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft_cu12:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft-cu12:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft-cu12:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft_cu12:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft_cu12:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cufft-cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cufft_cu12:11.3.0.4:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cufft-cu12@11.3.3.83?package-id=9f573e9dc504d39f","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cufft-cu12","version":"11.3.3.83","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cufft-cu12@11.3.3.83","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft-cu12:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft-cu12:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft_cu12:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft_cu12:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft-cu12:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft-cu12:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft_cu12:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft_cu12:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft-cu12:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft-cu12:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft_cu12:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft_cu12:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufft:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufft:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft-cu12:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft-cu12:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft_cu12:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft_cu12:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufft:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufft:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cufft-cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cufft_cu12:11.3.3.83:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cufft_cu12-11.3.3.83.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cufft_cu12-11.3.3.83.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cufft_cu12-11.3.3.83.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cufile-cu12@1.11.1.6?package-id=721d7f21000b26cf","type":"library","name":"nvidia-cufile-cu12","version":"1.11.1.6","cpe":"cpe:2.3:a:python-nvidia-cufile-cu12:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cufile-cu12@1.11.1.6","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile-cu12:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile_cu12:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile_cu12:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile-cu12:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile-cu12:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile_cu12:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile_cu12:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile-cu12:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile-cu12:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile_cu12:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile_cu12:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile-cu12:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile-cu12:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile_cu12:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile_cu12:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cufile-cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cufile_cu12:1.11.1.6:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cufile-cu12@1.13.1.3?package-id=22d558488e79d694","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cufile-cu12","version":"1.13.1.3","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cufile-cu12@1.13.1.3","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile-cu12:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile-cu12:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile_cu12:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile_cu12:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile-cu12:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile-cu12:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile_cu12:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile_cu12:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile-cu12:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile-cu12:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile_cu12:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile_cu12:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cufile:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cufile:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile-cu12:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile-cu12:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile_cu12:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile_cu12:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cufile:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cufile:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cufile-cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cufile_cu12:1.13.1.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cufile_cu12-1.13.1.3.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cufile_cu12-1.13.1.3.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cufile_cu12-1.13.1.3.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-curand-cu12@10.3.7.77?package-id=b33123af8e2e2136","type":"library","name":"nvidia-curand-cu12","version":"10.3.7.77","cpe":"cpe:2.3:a:python-nvidia-curand-cu12:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-curand-cu12@10.3.7.77","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand-cu12:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand_cu12:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand_cu12:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand-cu12:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand-cu12:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand_cu12:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand_cu12:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand-cu12:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand-cu12:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand_cu12:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand_cu12:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand-cu12:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand-cu12:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand_cu12:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand_cu12:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-curand-cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_curand_cu12:10.3.7.77:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-curand-cu12@10.3.9.90?package-id=8ce74724bb5020ce","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-curand-cu12","version":"10.3.9.90","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-curand-cu12@10.3.9.90","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand-cu12:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand-cu12:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand_cu12:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand_cu12:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand-cu12:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand-cu12:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand_cu12:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand_cu12:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand-cu12:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand-cu12:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand_cu12:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand_cu12:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-curand:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_curand:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand-cu12:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand-cu12:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand_cu12:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand_cu12:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-curand:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_curand:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-curand-cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_curand_cu12:10.3.9.90:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_curand_cu12-10.3.9.90.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_curand_cu12-10.3.9.90.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_curand_cu12-10.3.9.90.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cusolver-cu12@11.7.1.2?package-id=8e42d2eaaf57b22f","type":"library","name":"nvidia-cusolver-cu12","version":"11.7.1.2","cpe":"cpe:2.3:a:python-nvidia-cusolver-cu12:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cusolver-cu12@11.7.1.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver-cu12:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver_cu12:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver_cu12:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver-cu12:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver-cu12:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver_cu12:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver_cu12:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver-cu12:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver-cu12:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver_cu12:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver_cu12:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver-cu12:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver-cu12:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver_cu12:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver_cu12:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cusolver-cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cusolver_cu12:11.7.1.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cusolver-cu12@11.7.3.90?package-id=ee8021203c6faded","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cusolver-cu12","version":"11.7.3.90","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cusolver-cu12@11.7.3.90","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver-cu12:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver-cu12:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver_cu12:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver_cu12:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver-cu12:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver-cu12:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver_cu12:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver_cu12:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver-cu12:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver-cu12:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver_cu12:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver_cu12:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusolver:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusolver:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver-cu12:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver-cu12:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver_cu12:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver_cu12:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusolver:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusolver:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cusolver-cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cusolver_cu12:11.7.3.90:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusolver_cu12-11.7.3.90.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusolver_cu12-11.7.3.90.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusolver_cu12-11.7.3.90.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cusparse-cu12@12.5.4.2?package-id=aa4eea95c61e361d","type":"library","name":"nvidia-cusparse-cu12","version":"12.5.4.2","cpe":"cpe:2.3:a:python-nvidia-cusparse-cu12:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cusparse-cu12@12.5.4.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse-cu12:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse_cu12:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse_cu12:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse-cu12:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse-cu12:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse_cu12:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse_cu12:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse-cu12:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse-cu12:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse_cu12:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse_cu12:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse-cu12:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse-cu12:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse_cu12:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse_cu12:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cusparse-cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cusparse_cu12:12.5.4.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cusparse-cu12@12.5.8.93?package-id=4e97b7bd05fcfc89","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-cusparse-cu12","version":"12.5.8.93","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cusparse-cu12@12.5.8.93","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse-cu12:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse-cu12:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse_cu12:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse_cu12:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse-cu12:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse-cu12:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse_cu12:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse_cu12:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse-cu12:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse-cu12:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse_cu12:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse_cu12:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparse:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparse:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse-cu12:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse-cu12:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse_cu12:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse_cu12:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparse:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparse:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cusparse-cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cusparse_cu12:12.5.8.93:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusparse_cu12-12.5.8.93.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusparse_cu12-12.5.8.93.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusparse_cu12-12.5.8.93.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-cusparselt-cu12@0.6.3?package-id=5316e924766bd4c9","type":"library","name":"nvidia-cusparselt-cu12","version":"0.6.3","cpe":"cpe:2.3:a:python-nvidia-cusparselt-cu12:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cusparselt-cu12@0.6.3","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt-cu12:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt_cu12:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt_cu12:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt-cu12:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt-cu12:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt_cu12:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt_cu12:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt-cu12:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt-cu12:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt_cu12:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt_cu12:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt-cu12:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt-cu12:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt_cu12:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt_cu12:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cusparselt-cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cusparselt_cu12:0.6.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-cusparselt-cu12@0.7.1?package-id=a6f3a81747fe8aea","type":"library","author":"NVIDIA Corporation ","name":"nvidia-cusparselt-cu12","version":"0.7.1","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:python-nvidia-cusparselt-cu12:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-cusparselt-cu12@0.7.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt-cu12:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt_cu12:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt_cu12:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporation_project:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporation_project:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporationproject:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporationproject:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installer_project:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installer_project:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt-cu12:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt-cu12:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt_cu12:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt_cu12:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt-cu12:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt-cu12:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt_cu12:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt_cu12:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installerproject:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installerproject:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporation_project:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporation_project:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporation:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporation:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporationproject:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporationproject:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-cusparselt:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_cusparselt:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installer_project:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installer_project:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt-cu12:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt-cu12:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt_cu12:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt_cu12:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda-installer:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda-installer:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installer:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installer:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installerproject:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installerproject:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporation:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_corporation:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-cusparselt:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cusparselt:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda-installer:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda-installer:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installer:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cuda_installer:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-cusparselt-cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_cusparselt_cu12:0.7.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusparselt_cu12-0.7.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusparselt_cu12-0.7.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_cusparselt_cu12-0.7.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-nccl-cu12@2.26.2?package-id=8010e2ca156a20b8","type":"library","name":"nvidia-nccl-cu12","version":"2.26.2","cpe":"cpe:2.3:a:python-nvidia-nccl-cu12:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-nccl-cu12@2.26.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl-cu12:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl_cu12:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl_cu12:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl-cu12:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl-cu12:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl_cu12:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl_cu12:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl-cu12:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl-cu12:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl_cu12:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl_cu12:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl-cu12:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl-cu12:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl_cu12:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl_cu12:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-nccl-cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_nccl_cu12:2.26.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-nccl-cu12@2.27.5?package-id=4ea1eb7607842808","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-nccl-cu12","version":"2.27.5","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-nccl-cu12@2.27.5","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl-cu12:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl-cu12:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl_cu12:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl_cu12:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl-cu12:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl-cu12:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl_cu12:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl_cu12:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl-cu12:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl-cu12:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl_cu12:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl_cu12:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nccl:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nccl:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl-cu12:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl-cu12:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl_cu12:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl_cu12:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nccl:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nccl:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-nccl-cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_nccl_cu12:2.27.5:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nccl_cu12-2.27.5.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nccl_cu12-2.27.5.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nccl_cu12-2.27.5.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-nvjitlink-cu12@12.6.85?package-id=d682cf614113efe9","type":"library","name":"nvidia-nvjitlink-cu12","version":"12.6.85","cpe":"cpe:2.3:a:python-nvidia-nvjitlink-cu12:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-nvjitlink-cu12@12.6.85","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink-cu12:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink_cu12:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink_cu12:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink-cu12:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink-cu12:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink_cu12:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink_cu12:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink-cu12:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink-cu12:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink_cu12:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink_cu12:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink-cu12:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink-cu12:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink_cu12:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink_cu12:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-nvjitlink-cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_nvjitlink_cu12:12.6.85:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-nvjitlink-cu12@12.8.93?package-id=7703624d1b3f44e2","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-nvjitlink-cu12","version":"12.8.93","licenses":[{"license":{"name":"NVIDIA Proprietary Software"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-nvjitlink-cu12@12.8.93","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink-cu12:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink-cu12:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink_cu12:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink_cu12:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink-cu12:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink-cu12:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink_cu12:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink_cu12:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink-cu12:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink-cu12:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink_cu12:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink_cu12:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvjitlink:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvjitlink:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink-cu12:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink-cu12:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink_cu12:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink_cu12:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvjitlink:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvjitlink:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-nvjitlink-cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_nvjitlink_cu12:12.8.93:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvjitlink_cu12-12.8.93.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvjitlink_cu12-12.8.93.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvjitlink_cu12-12.8.93.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-nvshmem-cu12@3.3.20?package-id=c9c416d31c00c641","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-nvshmem-cu12","version":"3.3.20","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-nvshmem-cu12@3.3.20","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvshmem-cu12:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvshmem-cu12:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvshmem_cu12:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvshmem_cu12:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvshmem:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvshmem:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvshmem:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvshmem:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvshmem-cu12:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvshmem-cu12:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvshmem_cu12:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvshmem_cu12:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvshmem-cu12:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvshmem-cu12:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvshmem_cu12:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvshmem_cu12:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvshmem:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvshmem:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvshmem:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvshmem:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvshmem:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvshmem:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvshmem:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvshmem:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvshmem-cu12:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvshmem-cu12:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvshmem_cu12:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvshmem_cu12:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvshmem:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvshmem:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvshmem:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvshmem:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-nvshmem-cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_nvshmem_cu12:3.3.20:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvshmem_cu12-3.3.20.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvshmem_cu12-3.3.20.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvshmem_cu12-3.3.20.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/nvidia-nvtx-cu12@12.6.77?package-id=6d801c7cc05e57f0","type":"library","name":"nvidia-nvtx-cu12","version":"12.6.77","cpe":"cpe:2.3:a:python-nvidia-nvtx-cu12:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-nvtx-cu12@12.6.77","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx-cu12:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx_cu12:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx_cu12:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx-cu12:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx-cu12:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx_cu12:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx_cu12:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx-cu12:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx-cu12:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx_cu12:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx_cu12:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx-cu12:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx-cu12:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx_cu12:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx_cu12:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-nvtx-cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_nvtx_cu12:12.6.77:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/nvidia-nvtx-cu12@12.8.90?package-id=d4fd0767c9d928fe","type":"library","author":"Nvidia CUDA Installer Team ","name":"nvidia-nvtx-cu12","version":"12.8.90","licenses":[{"license":{"name":"Apache 2.0"}}],"cpe":"cpe:2.3:a:nvidia_cuda_installer_team_project:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*","purl":"pkg:pypi/nvidia-nvtx-cu12@12.8.90","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team_project:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_teamproject:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx-cu12:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx-cu12:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx_cu12:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx_cu12:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_cuda_installer_team:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer_project:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installerproject:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx-cu12:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx-cu12:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx_cu12:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx_cu12:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx-cu12:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx-cu12:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx_cu12:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx_cu12:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia-nvtx:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia_nvtx:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute-installer:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:compute_installer:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx-cu12:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx-cu12:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx_cu12:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx_cu12:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-nvidia:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_nvidia:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia-nvtx:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia_nvtx:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nvidia:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia-nvtx-cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:nvidia_nvtx_cu12:12.8.90:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvtx_cu12-12.8.90.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvtx_cu12-12.8.90.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/nvidia_nvtx_cu12-12.8.90.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/openai@1.93.3?package-id=a273677e5e984b0d","type":"library","name":"openai","version":"1.93.3","cpe":"cpe:2.3:a:python-openai:python-openai:1.93.3:*:*:*:*:*:*:*","purl":"pkg:pypi/openai@1.93.3","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-openai:python_openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_openai:python-openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_openai:python_openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:openai:python-openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:openai:python_openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-openai:openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_openai:openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:openai:openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:openai:1.93.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/openai@2.15.0?package-id=e48ce808d95cf415","type":"library","author":"OpenAI ","name":"openai","version":"2.15.0","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:openai_\\","name":"outcome","version":"1.3.0.post0","licenses":[{"expression":"MIT OR Apache-2.0"}],"cpe":"cpe:2.3:a:frazer_mclean_project:python-outcome:1.3.0.post0:*:*:*:*:*:*:*","purl":"pkg:pypi/outcome@1.3.0.post0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_mclean_project:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_mcleanproject:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_mcleanproject:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_mclean_project:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_project:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_project:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-outcome:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-outcome:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_outcome:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_outcome:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_mclean:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_mclean:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_mcleanproject:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazerproject:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazerproject:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_project:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:outcome:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:outcome:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-outcome:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_outcome:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer_mclean:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazerproject:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:outcome:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:frazer:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/outcome-1.3.0.post0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/outcome-1.3.0.post0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/outcome-1.3.0.post0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/outcome@1.3.0.post0?package-id=85f7f4936ff9906d","type":"library","name":"outcome","version":"1.3.0.post0","cpe":"cpe:2.3:a:python-outcome:python-outcome:1.3.0.post0:*:*:*:*:*:*:*","purl":"pkg:pypi/outcome@1.3.0.post0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-outcome:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_outcome:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_outcome:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:outcome:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:outcome:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-outcome:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_outcome:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:outcome:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:outcome:1.3.0.post0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/packaging@25.0?package-id=6e259a8838d0bc42","type":"library","author":"Donald Stufft ","name":"packaging","version":"25.0","cpe":"cpe:2.3:a:donald_stufft_\\","name":"pillow","version":"12.1.0","licenses":[{"license":{"id":"MIT-CMU"}}],"cpe":"cpe:2.3:a:\\\"jeffrey_a__clark\\\"_\\","name":"pip","version":"25.3","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:pip_developers_\\","name":"pluggy","version":"1.6.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:holger_krekel_\\","name":"propcache","version":"0.4.1","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:andrew_svetlov_project:python-propcache:0.4.1:*:*:*:*:*:*:*","purl":"pkg:pypi/propcache@0.4.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:python_propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:python-propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:python_propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-propcache:python-propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-propcache:python_propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_propcache:python-propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_propcache:python_propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:python-propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:python_propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:python-propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:python_propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:propcache:python-propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:propcache:python_propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-propcache:propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_propcache:propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:propcache:propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:propcache:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/propcache-0.4.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/propcache-0.4.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/propcache-0.4.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/psutil@7.0.0?package-id=e409ef092e0c2dda","type":"library","name":"psutil","version":"7.0.0","cpe":"cpe:2.3:a:python-psutil:python-psutil:7.0.0:*:*:*:*:*:*:*","purl":"pkg:pypi/psutil@7.0.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-psutil:python_psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_psutil:python-psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_psutil:python_psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:psutil:python-psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:psutil:python_psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-psutil:psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_psutil:psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:psutil:psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:psutil:7.0.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/psutil@7.2.1?package-id=fc6cf7a098b75095","type":"library","author":"Giampaolo Rodola ","name":"psutil","version":"7.2.1","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:giampaolo_rodola_project:python-psutil:7.2.1:*:*:*:*:*:*:*","purl":"pkg:pypi/psutil@7.2.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:giampaolo_rodola_project:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:giampaolo_rodolaproject:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:giampaolo_rodolaproject:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:giampaolo_rodola_project:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodola_project:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodola_project:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:giampaolo_rodola:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:giampaolo_rodola:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:giampaolo_rodolaproject:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodolaproject:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodolaproject:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-psutil:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-psutil:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_psutil:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_psutil:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodola_project:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:giampaolo_rodola:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g-rodola:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g-rodola:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodola:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodola:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodolaproject:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:psutil:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:psutil:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-psutil:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_psutil:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g-rodola:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:g_rodola:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:psutil:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:psutil:7.2.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/psutil-7.2.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/psutil-7.2.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/psutil-7.2.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/pycparser@2.22?package-id=129f7964fa1f0734","type":"library","name":"pycparser","version":"2.22","cpe":"cpe:2.3:a:python-pycparser:python-pycparser:2.22:*:*:*:*:*:*:*","purl":"pkg:pypi/pycparser@2.22","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pycparser:python_pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pycparser:python-pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pycparser:python_pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pycparser:python-pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pycparser:python_pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pycparser:pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pycparser:pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pycparser:pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pycparser:2.22:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/pycparser@2.23?package-id=1fada86463d1c38a","type":"library","author":"Eli Bendersky ","name":"pycparser","version":"2.23","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:eli_bendersky_project:python-pycparser:2.23:*:*:*:*:*:*:*","purl":"pkg:pypi/pycparser@2.23","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:eli_bendersky_project:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eli_benderskyproject:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eli_benderskyproject:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pycparser:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pycparser:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pycparser:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pycparser:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eli_bendersky_project:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eliben_project:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eliben_project:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eli_bendersky:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eli_bendersky:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eli_benderskyproject:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:elibenproject:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:elibenproject:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pycparser:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pycparser:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pycparser:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pycparser:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eliben_project:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eli_bendersky:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eliben:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eliben:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:elibenproject:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pycparser:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:eliben:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pycparser:2.23:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pycparser-2.23.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/pycparser-2.23.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/pycparser-2.23.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/pydantic@2.11.7?package-id=b6519580f96ae747","type":"library","name":"pydantic","version":"2.11.7","cpe":"cpe:2.3:a:python-pydantic:python-pydantic:2.11.7:*:*:*:*:*:*:*","purl":"pkg:pypi/pydantic@2.11.7","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pydantic:python_pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pydantic:python-pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pydantic:python_pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pydantic:python-pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pydantic:python_pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pydantic:pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pydantic:pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pydantic:pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pydantic:2.11.7:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/pydantic@2.12.5?package-id=d4834e9598ae66c4","type":"library","author":"Samuel Colvin , Eric Jolibois , Hasan Ramezani , Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>, Terrence Dorsey , David Montague , Serge Matveenko , Marcelo Trylesinski , Sydney Runkle , David Hewitt , Alex Hall , Victorien Plot , Douwe Maan ","name":"pydantic","version":"2.12.5","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:samuel_colvin_\\, Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>, David Montague , David Hewitt , Sydney Runkle , Victorien Plot ","name":"pydantic-core","version":"2.41.5","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:samuel_colvin_\\","name":"pyee","version":"13.0.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:josh_holbrook_\\","name":"pygments","version":"2.19.2","licenses":[{"license":{"id":"BSD-2-Clause"}}],"cpe":"cpe:2.3:a:georg_brandl_\\","name":"pyopenssl","version":"25.3.0","licenses":[{"license":{"name":"Apache License, Version 2.0"}}],"cpe":"cpe:2.3:a:pyopenssl_developers_project:python-pyopenssl:25.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/pyopenssl@25.3.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl_developers_project:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl_developersproject:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl_developersproject:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_dev_project:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_dev_project:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_devproject:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_devproject:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl_developers_project:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl_developers:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl_developers:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl_developersproject:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_dev_project:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography-dev:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography-dev:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_dev:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_dev:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_devproject:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pyopenssl:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pyopenssl:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyopenssl:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyopenssl:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl_developers:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography-dev:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:cryptography_dev:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pyopenssl:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyopenssl:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pyopenssl-25.3.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/pyopenssl-25.3.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/pyopenssl-25.3.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/pyopenssl@25.3.0?package-id=218fa1d6544a119f","type":"library","name":"pyopenssl","version":"25.3.0","cpe":"cpe:2.3:a:python-pyopenssl:python-pyopenssl:25.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/pyopenssl@25.3.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pyopenssl:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyopenssl:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyopenssl:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pyopenssl:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyopenssl:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyopenssl:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pyopenssl:25.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/pypdf@6.4.1?package-id=ab6c3274076edf8c","type":"library","name":"pypdf","version":"6.4.1","cpe":"cpe:2.3:a:pypdf_project:pypdf:6.4.1:*:*:*:*:*:*:*","purl":"pkg:pypi/pypdf@6.4.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/pypdf@6.6.0?package-id=584901e7d93ee52f","type":"library","author":"Mathieu Fenniak ","name":"pypdf","version":"6.6.0","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:pypdf_project:pypdf:6.6.0:*:*:*:*:*:*:*","purl":"pkg:pypi/pypdf@6.6.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pypdf-6.6.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/pypdf-6.6.0.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/pysocks@1.7.1?package-id=85422de4ee67fc97","type":"library","author":"Anorov ","name":"pysocks","version":"1.7.1","licenses":[{"license":{"name":"BSD"}}],"cpe":"cpe:2.3:a:anorov_vorona_project:python-pysocks:1.7.1:*:*:*:*:*:*:*","purl":"pkg:pypi/pysocks@1.7.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_vorona_project:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_voronaproject:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_voronaproject:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_project:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_project:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_vorona_project:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pysocks:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pysocks:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pysocks:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pysocks:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov-vorona:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov-vorona:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_vorona:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_vorona:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_voronaproject:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorovproject:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorovproject:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_project:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pysocks:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pysocks:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pysocks:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pysocks:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov-vorona:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov_vorona:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorovproject:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pysocks:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:anorov:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/PySocks-1.7.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/PySocks-1.7.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/PySocks-1.7.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/pysocks@1.7.1?package-id=447d2c71a2b6e55a","type":"library","name":"pysocks","version":"1.7.1","cpe":"cpe:2.3:a:python-pysocks:python-pysocks:1.7.1:*:*:*:*:*:*:*","purl":"pkg:pypi/pysocks@1.7.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pysocks:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pysocks:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pysocks:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pysocks:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pysocks:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pysocks:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pysocks:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pysocks:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pysocks:1.7.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/pytest@9.0.2?package-id=e5b6148ee0b1d7ce","type":"library","author":"Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt, Floris Bruynooghe, Brianna Laugher, Florian Bruhin, Others (See AUTHORS)","name":"pytest","version":"9.0.2","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\)_project:python-pytest:9.0.2:*:*:*:*:*:*:*","purl":"pkg:pypi/pytest@9.0.2","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\)_project:python_pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\)project:python-pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\)project:python_pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\)_project:pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\):python-pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\):python_pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\)project:pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:holger_krekel\\,_bruno_oliveira\\,_ronny_pfannschmidt\\,_floris_bruynooghe\\,_brianna_laugher\\,_florian_bruhin\\,_others_\\(see_authors\\):pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest:python-pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest:python_pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest:python-pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest:python_pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest:python-pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest:python_pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest:pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest:pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest:pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pytest:9.0.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pytest-9.0.2.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/pytest-9.0.2.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/pytest-9.0.2.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/pytest-asyncio@1.3.0?package-id=f8634b667ec65d05","type":"library","author":"Tin Tvrtković ","name":"pytest-asyncio","version":"1.3.0","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:python-pytest-asyncio:python-pytest-asyncio:1.3.0:*:*:*:*:*:*:*","purl":"pkg:pypi/pytest-asyncio@1.3.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest-asyncio:python_pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest_asyncio:python-pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest_asyncio:python_pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest-asyncio:python-pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest-asyncio:python_pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest_asyncio:python-pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest_asyncio:python_pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest-asyncio:pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest-asyncio:pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest_asyncio:pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest_asyncio:pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest:python-pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest:python_pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest:python-pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest:python_pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest-asyncio:pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest-asyncio:pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest_asyncio:pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest_asyncio:pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest:python-pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest:python_pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest:pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pytest:pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest:pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pytest:pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest:pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pytest:pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pytest-asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pytest_asyncio:1.3.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pytest_asyncio-1.3.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/pytest_asyncio-1.3.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/pytest_asyncio-1.3.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/python-dotenv@1.1.1?package-id=b21407c23adbcfff","type":"library","name":"python-dotenv","version":"1.1.1","cpe":"cpe:2.3:a:python-dotenv:python-dotenv:1.1.1:*:*:*:*:*:*:*","purl":"pkg:pypi/python-dotenv@1.1.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-dotenv:python_dotenv:1.1.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_dotenv:python-dotenv:1.1.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_dotenv:python_dotenv:1.1.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-dotenv:1.1.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_dotenv:1.1.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/python-dotenv@1.2.1?package-id=53627f7d50301d4a","type":"library","author":"Saurabh Kumar ","name":"python-dotenv","version":"1.2.1","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:saurabh_kumar_\\","name":"pyyaml","version":"6.0.3","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:kirill_simonov_project:python-pyyaml:6.0.3:*:*:*:*:*:*:*","purl":"pkg:pypi/pyyaml@6.0.3","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:kirill_simonov_project:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kirill_simonovproject:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kirill_simonovproject:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kirill_simonov_project:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kirill_simonov:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kirill_simonov:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kirill_simonovproject:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pyyaml:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pyyaml:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyyaml:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyyaml:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xi_project:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xi_project:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xiproject:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xiproject:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:kirill_simonov:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-pyyaml:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_pyyaml:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyyaml:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyyaml:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xi_project:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xi:python-pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xi:python_pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xiproject:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:pyyaml:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xi:pyyaml:6.0.3:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/pyyaml-6.0.3.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/pyyaml-6.0.3.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/pyyaml-6.0.3.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/rank-bm25@0.2.2?package-id=bfeb787ac3a9532a","type":"library","author":"D. Brown ","name":"rank-bm25","version":"0.2.2","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:dorianstuartbrown_project:python-rank-bm25:0.2.2:*:*:*:*:*:*:*","purl":"pkg:pypi/rank-bm25@0.2.2","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrown_project:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrownproject:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrownproject:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrown_project:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrown_project:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrown:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrown:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrownproject:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrownproject:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brown_project:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brown_project:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brownproject:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brownproject:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrown:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:dorianstuartbrown:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brown_project:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brown_project:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brown:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brown:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brownproject:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brownproject:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brown:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:d__brown:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/rank_bm25-0.2.2.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/rank_bm25-0.2.2.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/rank_bm25-0.2.2.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/rank-bm25@0.2.2?package-id=7b3c902e1f7f4c31","type":"library","name":"rank-bm25","version":"0.2.2","cpe":"cpe:2.3:a:python-rank-bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*","purl":"pkg:pypi/rank-bm25@0.2.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-pip-requirements-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/deploy/docker/requirements.txt"}]},{"bom-ref":"pkg:pypi/rank-bm25@0.2.2?package-id=1d33b3c8c8b8a4c9","type":"library","name":"rank-bm25","version":"0.2.2","cpe":"cpe:2.3:a:python-rank-bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*","purl":"pkg:pypi/rank-bm25@0.2.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank-bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank_bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:python-rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:python_rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank-bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank_bm25:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:rank-bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rank:rank_bm25:0.2.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/referencing@0.36.2?package-id=dd1f6096e0b10c43","type":"library","name":"referencing","version":"0.36.2","cpe":"cpe:2.3:a:python-referencing:python-referencing:0.36.2:*:*:*:*:*:*:*","purl":"pkg:pypi/referencing@0.36.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-referencing:python_referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_referencing:python-referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_referencing:python_referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-referencing:referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_referencing:referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:referencing:python-referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:referencing:python_referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:referencing:referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:referencing:0.36.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/referencing@0.37.0?package-id=728e8978d0f65973","type":"library","author":"Julian Berman ","name":"referencing","version":"0.37.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:julian_berman_\\","name":"regex","version":"2026.1.15","licenses":[{"expression":"Apache-2.0 AND CNRI-Python"}],"cpe":"cpe:2.3:a:matthew_barnett_\\","name":"requests","version":"2.32.5","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:python:requests:2.32.5:*:*:*:*:*:*:*","purl":"pkg:pypi/requests@2.32.5","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/requests-2.32.5.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/requests-2.32.5.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/requests-2.32.5.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/rich@14.0.0?package-id=9aac75b499ad9a98","type":"library","name":"rich","version":"14.0.0","cpe":"cpe:2.3:a:python-rich:python-rich:14.0.0:*:*:*:*:*:*:*","purl":"pkg:pypi/rich@14.0.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rich:python_rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rich:python-rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rich:python_rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rich:rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rich:rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rich:python-rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rich:python_rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rich:rich:14.0.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/rich@14.2.0?package-id=304a0e26ab360c83","type":"library","author":"Will McGugan ","name":"rich","version":"14.2.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:will_mcgugan_project:python-rich:14.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/rich@14.2.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:will_mcgugan_project:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:will_mcguganproject:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:will_mcguganproject:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcgugan_project:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcgugan_project:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcguganproject:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcguganproject:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:will_mcgugan_project:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:will_mcgugan:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:will_mcgugan:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:will_mcguganproject:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcgugan_project:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rich:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rich:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rich:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rich:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcgugan:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcgugan:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcguganproject:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:will_mcgugan:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rich:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rich:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rich:python-rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rich:python_rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:willmcgugan:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rich:rich:14.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/rich-14.2.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/rich-14.2.0.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/rpds-py@0.26.0?package-id=6a92caf3553687cd","type":"library","name":"rpds-py","version":"0.26.0","cpe":"cpe:2.3:a:python-rpds-py:python-rpds-py:0.26.0:*:*:*:*:*:*:*","purl":"pkg:pypi/rpds-py@0.26.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rpds-py:python_rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rpds_py:python-rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rpds_py:python_rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rpds:python-rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rpds:python_rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rpds:python-rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rpds:python_rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rpds-py:rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rpds-py:rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rpds_py:rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rpds_py:rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds-py:python-rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds-py:python_rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds_py:python-rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds_py:python_rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rpds:rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-rpds:rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rpds:rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_rpds:rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds:python-rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds:python_rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds-py:rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds-py:rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds_py:rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds_py:rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds:rpds-py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:rpds:rpds_py:0.26.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/rpds-py@0.30.0?package-id=1febbff9028f1939","type":"library","author":"Julian Berman ","name":"rpds-py","version":"0.30.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:julian_berman_\\","name":"rtree","version":"1.4.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:sean_gillies_\\","name":"safetensors","version":"0.7.0","cpe":"cpe:2.3:a:nicolas_patry_\\, Tom Aarsen ","name":"sentence-transformers","version":"5.2.0","licenses":[{"license":{"name":"Apache 2.0"}}],"cpe":"cpe:2.3:a:python-sentence-transformers:python-sentence-transformers:5.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/sentence-transformers@5.2.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sentence-transformers:python_sentence_transformers:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sentence_transformers:python-sentence-transformers:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sentence_transformers:python_sentence_transformers:5.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:nils_reimers_\\","name":"setuptools","version":"59.6.0","licenses":[{"license":{"name":"UNKNOWN"}}],"cpe":"cpe:2.3:a:python:setuptools:59.6.0:*:*:*:*:*:*:*","purl":"pkg:pypi/setuptools@59.6.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/setuptools-59.6.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/setuptools-59.6.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/setuptools-59.6.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/setuptools@80.9.0?package-id=4b720961b56196fc","type":"library","name":"setuptools","version":"80.9.0","cpe":"cpe:2.3:a:python:setuptools:80.9.0:*:*:*:*:*:*:*","purl":"pkg:pypi/setuptools@80.9.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/shapely@2.0.7?package-id=196260620b436564","type":"library","name":"shapely","version":"2.0.7","cpe":"cpe:2.3:a:python-shapely:python-shapely:2.0.7:*:*:*:*:*:*:*","purl":"pkg:pypi/shapely@2.0.7","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-shapely:python_shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_shapely:python-shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_shapely:python_shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-shapely:shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_shapely:shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shapely:python-shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shapely:python_shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shapely:shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:shapely:2.0.7:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/shapely@2.1.2?package-id=c2bef081f2f1a90d","type":"library","author":"Sean Gillies","name":"shapely","version":"2.1.2","licenses":[{"license":{"name":"BSD 3-Clause"}}],"cpe":"cpe:2.3:a:sean_gillies_project:python-shapely:2.1.2:*:*:*:*:*:*:*","purl":"pkg:pypi/shapely@2.1.2","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:sean_gillies_project:python_shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sean_gilliesproject:python-shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sean_gilliesproject:python_shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-shapely:python-shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-shapely:python_shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_shapely:python-shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_shapely:python_shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sean_gillies_project:shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sean_gillies:python-shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sean_gillies:python_shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sean_gilliesproject:shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-shapely:shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_shapely:shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shapely:python-shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shapely:python_shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sean_gillies:shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shapely:shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:shapely:2.1.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/shapely-2.1.2.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/shapely-2.1.2.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/shapely-2.1.2.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/slowapi@0.1.9?package-id=ee54c06d3880b2c6","type":"library","name":"slowapi","version":"0.1.9","cpe":"cpe:2.3:a:python-slowapi:python-slowapi:0.1.9:*:*:*:*:*:*:*","purl":"pkg:pypi/slowapi@0.1.9","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-pip-requirements-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-slowapi:python_slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_slowapi:python-slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_slowapi:python_slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-slowapi:slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_slowapi:slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slowapi:python-slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slowapi:python_slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:slowapi:slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:slowapi:0.1.9:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/deploy/docker/requirements.txt"}]},{"bom-ref":"pkg:pypi/sniffio@1.3.1?package-id=f20794e746c24715","type":"library","author":"\"Nathaniel J. Smith\" ","name":"sniffio","version":"1.3.1","licenses":[{"expression":"MIT OR Apache-2.0"}],"cpe":"cpe:2.3:a:\\\"nathaniel_j__smith\\\"_\\","name":"snowballstemmer","version":"2.2.0","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:snowball_developers_project:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/snowballstemmer@2.2.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_developers_project:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_developersproject:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_developersproject:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discuss_project:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discuss_project:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discussproject:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discussproject:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-snowballstemmer:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-snowballstemmer:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_snowballstemmer:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_snowballstemmer:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_developers_project:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_developers:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_developers:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_developersproject:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discuss_project:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball-discuss:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball-discuss:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discuss:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discuss:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discussproject:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-snowballstemmer:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_snowballstemmer:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowballstemmer:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowballstemmer:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_developers:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball-discuss:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowball_discuss:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowballstemmer:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/snowballstemmer-2.2.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/snowballstemmer-2.2.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/snowballstemmer-2.2.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/snowballstemmer@2.2.0?package-id=58aed13f7dd380be","type":"library","name":"snowballstemmer","version":"2.2.0","cpe":"cpe:2.3:a:python-snowballstemmer:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/snowballstemmer@2.2.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-snowballstemmer:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_snowballstemmer:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_snowballstemmer:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-snowballstemmer:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_snowballstemmer:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowballstemmer:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowballstemmer:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:snowballstemmer:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:snowballstemmer:2.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:github/softprops/action-gh-release@v2?package-id=854017c06a3f14e3","type":"library","name":"softprops/action-gh-release","version":"v2","cpe":"cpe:2.3:a:softprops\\/action-gh-release:softprops\\/action-gh-release:v2:*:*:*:*:*:*:*","purl":"pkg:github/softprops/action-gh-release@v2","properties":[{"name":"syft:package:foundBy","value":"github-actions-usage-cataloger"},{"name":"syft:package:type","value":"github-action"},{"name":"syft:package:metadataType","value":"github-actions-use-statement"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action-gh-release:softprops\\/action_gh_release:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action_gh_release:softprops\\/action-gh-release:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action_gh_release:softprops\\/action_gh_release:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action-gh:softprops\\/action-gh-release:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action-gh:softprops\\/action_gh_release:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action_gh:softprops\\/action-gh-release:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action_gh:softprops\\/action_gh_release:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action:softprops\\/action-gh-release:v2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:softprops\\/action:softprops\\/action_gh_release:v2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.github/workflows/release.yml"}]},{"bom-ref":"pkg:pypi/sortedcontainers@2.4.0?package-id=3e7defbd2eaf35e1","type":"library","author":"Grant Jenks ","name":"sortedcontainers","version":"2.4.0","licenses":[{"license":{"name":"Apache 2.0"}}],"cpe":"cpe:2.3:a:python-sortedcontainers:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*","purl":"pkg:pypi/sortedcontainers@2.4.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sortedcontainers:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sortedcontainers:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sortedcontainers:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenks_project:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenks_project:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenksproject:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenksproject:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sortedcontainers:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sortedcontainers:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sortedcontainers:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sortedcontainers:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contact_project:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contact_project:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contactproject:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contactproject:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenks_project:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenks:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenks:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenksproject:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sortedcontainers:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contact_project:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contact:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contact:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contactproject:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:grant_jenks:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:contact:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/sortedcontainers-2.4.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/sortedcontainers-2.4.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/sortedcontainers-2.4.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/sortedcontainers@2.4.0?package-id=0551501277b7862e","type":"library","name":"sortedcontainers","version":"2.4.0","cpe":"cpe:2.3:a:python-sortedcontainers:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*","purl":"pkg:pypi/sortedcontainers@2.4.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sortedcontainers:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sortedcontainers:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sortedcontainers:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sortedcontainers:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sortedcontainers:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sortedcontainers:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sortedcontainers:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sortedcontainers:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:sortedcontainers:2.4.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/soupsieve@2.7?package-id=9747dec32d4557c8","type":"library","name":"soupsieve","version":"2.7","cpe":"cpe:2.3:a:python-soupsieve:python-soupsieve:2.7:*:*:*:*:*:*:*","purl":"pkg:pypi/soupsieve@2.7","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-soupsieve:python_soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_soupsieve:python-soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_soupsieve:python_soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-soupsieve:soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_soupsieve:soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:soupsieve:python-soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:soupsieve:python_soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:soupsieve:soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:soupsieve:2.7:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/soupsieve@2.8.1?package-id=93131a6c866ee6d8","type":"library","author":"Isaac Muse ","name":"soupsieve","version":"2.8.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:isaac_muse_\\","name":"sympy","version":"1.14.0","licenses":[{"license":{"name":"BSD"}}],"cpe":"cpe:2.3:a:sympy_development_team_project:python-sympy:1.14.0:*:*:*:*:*:*:*","purl":"pkg:pypi/sympy@1.14.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_development_team_project:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_development_teamproject:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_development_teamproject:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_development_team_project:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_development_team:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_development_team:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_development_teamproject:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_development_team:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_project:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_project:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sympy:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sympy:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sympy:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sympy:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympyproject:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympyproject:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy_project:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sympy:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sympy:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympyproject:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/sympy-1.14.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/sympy-1.14.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/sympy-1.14.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/sympy@1.14.0?package-id=6a70f05a3911505b","type":"library","name":"sympy","version":"1.14.0","cpe":"cpe:2.3:a:python-sympy:python-sympy:1.14.0:*:*:*:*:*:*:*","purl":"pkg:pypi/sympy@1.14.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sympy:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sympy:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sympy:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-sympy:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_sympy:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy:python-sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy:python_sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:sympy:sympy:1.14.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/tf-playwright-stealth@1.2.0?package-id=0ebcacbc134091cc","type":"library","name":"tf-playwright-stealth","version":"1.2.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:python-tf-playwright-stealth:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/tf-playwright-stealth@1.2.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright-stealth:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright_stealth:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright_stealth:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright-stealth:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright-stealth:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright_stealth:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright_stealth:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright-stealth:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright-stealth:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright_stealth:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright_stealth:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright-stealth:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright-stealth:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright_stealth:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright_stealth:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/tf_playwright_stealth-1.2.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/tf_playwright_stealth-1.2.0.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/tf-playwright-stealth@1.2.0?package-id=e217cde24f15c522","type":"library","name":"tf-playwright-stealth","version":"1.2.0","cpe":"cpe:2.3:a:python-tf-playwright-stealth:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/tf-playwright-stealth@1.2.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright-stealth:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright_stealth:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright_stealth:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright-stealth:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright-stealth:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright_stealth:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright_stealth:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright-stealth:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright-stealth:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright_stealth:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright_stealth:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright-stealth:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright-stealth:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright_stealth:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright_stealth:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf-playwright:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf_playwright:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf-playwright:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf_playwright:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tf:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tf:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf:python-tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf:python_tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf:tf-playwright-stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tf:tf_playwright_stealth:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/threadpoolctl@3.6.0?package-id=08346af784693976","type":"library","author":"Thomas Moreau ","name":"threadpoolctl","version":"3.6.0","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:thomas_moreau_2010_project:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*","purl":"pkg:pypi/threadpoolctl@3.6.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_2010_project:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_2010project:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_2010project:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_project:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_project:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-threadpoolctl:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-threadpoolctl:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_threadpoolctl:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_threadpoolctl:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreauproject:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreauproject:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_2010_project:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas-moreau-2010:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas-moreau-2010:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_2010:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_2010:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_2010project:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_project:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-threadpoolctl:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_threadpoolctl:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreauproject:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:threadpoolctl:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:threadpoolctl:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas-moreau-2010:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau_2010:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:thomas_moreau:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:threadpoolctl:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/threadpoolctl-3.6.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/threadpoolctl-3.6.0.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/threadpoolctl@3.6.0?package-id=057db710f6843309","type":"library","name":"threadpoolctl","version":"3.6.0","cpe":"cpe:2.3:a:python-threadpoolctl:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*","purl":"pkg:pypi/threadpoolctl@3.6.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-threadpoolctl:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_threadpoolctl:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_threadpoolctl:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-threadpoolctl:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_threadpoolctl:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:threadpoolctl:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:threadpoolctl:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:threadpoolctl:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:threadpoolctl:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/tiktoken@0.12.0?package-id=dbd6180a61a20943","type":"library","author":"Shantanu Jain ","name":"tiktoken","version":"0.12.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:shantanu_jain_project:python-tiktoken:0.12.0:*:*:*:*:*:*:*","purl":"pkg:pypi/tiktoken@0.12.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_jain_project:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_jainproject:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_jainproject:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_project:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_project:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tiktoken:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tiktoken:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tiktoken:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tiktoken:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanuproject:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanuproject:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_jain_project:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_jain:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_jain:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_jainproject:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_project:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tiktoken:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tiktoken:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanuproject:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tiktoken:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tiktoken:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu_jain:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:shantanu:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tiktoken:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:tiktoken:0.12.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/tiktoken-0.12.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/tiktoken-0.12.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/tiktoken-0.12.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/tiktoken@0.9.0?package-id=93b03a6d5469ce83","type":"library","name":"tiktoken","version":"0.9.0","cpe":"cpe:2.3:a:python-tiktoken:python-tiktoken:0.9.0:*:*:*:*:*:*:*","purl":"pkg:pypi/tiktoken@0.9.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tiktoken:python_tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tiktoken:python-tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tiktoken:python_tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tiktoken:tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tiktoken:tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tiktoken:python-tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tiktoken:python_tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tiktoken:tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:tiktoken:0.9.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/tokenizers@0.21.2?package-id=2c631dae4556d52e","type":"library","name":"tokenizers","version":"0.21.2","cpe":"cpe:2.3:a:python-tokenizers:python-tokenizers:0.21.2:*:*:*:*:*:*:*","purl":"pkg:pypi/tokenizers@0.21.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tokenizers:python_tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tokenizers:python-tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tokenizers:python_tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-tokenizers:tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_tokenizers:tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tokenizers:python-tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tokenizers:python_tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:tokenizers:tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:tokenizers:0.21.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/tokenizers@0.22.2?package-id=b60d8d378fc9737c","type":"library","author":"Nicolas Patry , Anthony Moi ","name":"tokenizers","version":"0.22.2","cpe":"cpe:2.3:a:nicolas_patry_\\","name":"tomli","version":"2.4.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:taneli_hukkinen_\\","name":"torch","version":"2.9.1","licenses":[{"license":{"id":"BSD-3-Clause"}}],"cpe":"cpe:2.3:a:pytorch_team_\\","name":"transformers","version":"4.57.5","licenses":[{"license":{"name":"Apache 2.0 License"}}],"cpe":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\)_project:python-transformers:4.57.5:*:*:*:*:*:*:*","purl":"pkg:pypi/transformers@4.57.5","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\)_project:python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\)project:python-transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\)project:python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\)_project:transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\):python-transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\):python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\)project:transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:hugging_face_team_\\(past_and_future\\)_with_the_help_of_all_our_contributors_\\(https\\:\\/\\/github_com\\/huggingface\\/transformers\\/graphs\\/contributors\\):transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformers_project:python-transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformers_project:python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-transformers:python-transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-transformers:python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_transformers:python-transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_transformers:python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformersproject:python-transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformersproject:python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformers_project:transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-transformers:transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_transformers:transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformers:python-transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformers:python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformersproject:transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:transformers:transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:transformers:4.57.5:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/transformers-4.57.5.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/transformers-4.57.5.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/transformers-4.57.5.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/trimesh@4.11.0?package-id=e272a6b5e409db24","type":"library","author":"Michael Dawson-Haggerty ","name":"trimesh","version":"4.11.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:michael_dawson_haggerty_\\","name":"trio","version":"0.32.0","licenses":[{"expression":"MIT OR Apache-2.0"}],"cpe":"cpe:2.3:a:\\\"nathaniel_j__smith\\\"_\\","name":"trio-websocket","version":"0.12.2","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:mark_e__haase_project:python-trio-websocket:0.12.2:*:*:*:*:*:*:*","purl":"pkg:pypi/trio-websocket@0.12.2","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haase_project:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio-websocket:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio-websocket:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio_websocket:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio_websocket:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haaseproject:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haaseproject:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaase_project:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaase_project:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haase_project:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haase_project:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaaseproject:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaaseproject:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio-websocket:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio-websocket:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio_websocket:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio_websocket:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio-websocket:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio-websocket:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio_websocket:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio_websocket:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haase:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haase:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haaseproject:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haaseproject:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaase_project:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaase_project:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaase:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaase:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaaseproject:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaaseproject:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio-websocket:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio-websocket:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio_websocket:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio_websocket:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haase:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mark_e__haase:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaase:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:mehaase:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/trio_websocket-0.12.2.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/trio_websocket-0.12.2.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/trio_websocket-0.12.2.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/trio-websocket@0.12.2?package-id=2a8c55fd137991b6","type":"library","name":"trio-websocket","version":"0.12.2","cpe":"cpe:2.3:a:python-trio-websocket:python-trio-websocket:0.12.2:*:*:*:*:*:*:*","purl":"pkg:pypi/trio-websocket@0.12.2","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio-websocket:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio_websocket:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio_websocket:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio-websocket:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio-websocket:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio_websocket:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio_websocket:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio-websocket:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio-websocket:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio_websocket:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio_websocket:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio-websocket:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio-websocket:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio_websocket:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio_websocket:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-trio:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_trio:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio:python-trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio:python_trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio:trio-websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:trio:trio_websocket:0.12.2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/triton@3.3.1?package-id=40afd7e3e861f418","type":"library","name":"triton","version":"3.3.1","cpe":"cpe:2.3:a:python-triton:python-triton:3.3.1:*:*:*:*:*:*:*","purl":"pkg:pypi/triton@3.3.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-triton:python_triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_triton:python-triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_triton:python_triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-triton:triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_triton:triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:triton:python-triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:triton:python_triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:triton:triton:3.3.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/triton@3.5.1?package-id=f47a97f1f4e82b65","type":"library","author":"Philippe Tillet ","name":"triton","version":"3.5.1","cpe":"cpe:2.3:a:philippe_tillet_project:python-triton:3.5.1:*:*:*:*:*:*:*","purl":"pkg:pypi/triton@3.5.1","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:philippe_tillet_project:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philippe_tilletproject:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philippe_tilletproject:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philippe_tillet_project:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philippe_tillet:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philippe_tillet:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philippe_tilletproject:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-triton:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-triton:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_triton:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_triton:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:phil_project:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:phil_project:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philproject:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philproject:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philippe_tillet:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-triton:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_triton:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:triton:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:triton:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:phil_project:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:phil:python-triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:phil:python_triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:philproject:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:triton:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:phil:triton:3.5.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/triton-3.5.1.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/triton-3.5.1.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/triton-3.5.1.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/typing-extensions@4.14.1?package-id=b12a3e8cd6e5114b","type":"library","name":"typing-extensions","version":"4.14.1","cpe":"cpe:2.3:a:python-typing-extensions:python-typing-extensions:4.14.1:*:*:*:*:*:*:*","purl":"pkg:pypi/typing-extensions@4.14.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-extensions:python_typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_extensions:python-typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_extensions:python_typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-extensions:typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-extensions:typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_extensions:typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_extensions:typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-extensions:python-typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-extensions:python_typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_extensions:python-typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_extensions:python_typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:python-typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:python_typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:python-typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:python_typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-extensions:typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-extensions:typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_extensions:typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_extensions:typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:python-typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:python_typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:typing-extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:typing_extensions:4.14.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/typing-extensions@4.15.0?package-id=40ce501b97eb7c03","type":"library","author":"\"Guido van Rossum, Jukka Lehtosalo, Łukasz Langa, Michael Lee\" ","name":"typing-extensions","version":"4.15.0","licenses":[{"license":{"id":"PSF-2.0"}}],"cpe":"cpe:2.3:a:python-typing-extensions:python-typing-extensions:4.15.0:*:*:*:*:*:*:*","purl":"pkg:pypi/typing-extensions@4.15.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-extensions:python_typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_extensions:python-typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_extensions:python_typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-extensions:typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-extensions:typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_extensions:typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_extensions:typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-extensions:python-typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-extensions:python_typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_extensions:python-typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_extensions:python_typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:python-typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:python_typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:python-typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:python_typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-extensions:typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-extensions:typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_extensions:typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_extensions:typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:python-typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:python_typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:typing-extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:typing_extensions:4.15.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/typing_extensions-4.15.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/typing_extensions-4.15.0.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/typing-inspection@0.4.1?package-id=4c595a8326e8f88d","type":"library","name":"typing-inspection","version":"0.4.1","cpe":"cpe:2.3:a:python-typing-inspection:python-typing-inspection:0.4.1:*:*:*:*:*:*:*","purl":"pkg:pypi/typing-inspection@0.4.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-inspection:python_typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_inspection:python-typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_inspection:python_typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-inspection:typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing-inspection:typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_inspection:typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing_inspection:typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-inspection:python-typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-inspection:python_typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_inspection:python-typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_inspection:python_typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:python-typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:python_typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:python-typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:python_typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-inspection:typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing-inspection:typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_inspection:typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing_inspection:typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-typing:typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_typing:typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:python-typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:python_typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:typing-inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:typing:typing_inspection:0.4.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/typing-inspection@0.4.2?package-id=841e326e011da9f4","type":"library","author":"Victorien Plot ","name":"typing-inspection","version":"0.4.2","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:victorien_plot_\\","name":"urllib3","version":"2.6.3","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:python:urllib3:2.6.3:*:*:*:*:*:*:*","purl":"pkg:pypi/urllib3@2.6.3","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/urllib3-2.6.3.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/urllib3-2.6.3.dist-info/RECORD"}]},{"bom-ref":"pkg:pypi/websocket-client@1.8.0?package-id=482f5b4b5f703c89","type":"library","name":"websocket-client","version":"1.8.0","cpe":"cpe:2.3:a:python-websocket-client:python-websocket-client:1.8.0:*:*:*:*:*:*:*","purl":"pkg:pypi/websocket-client@1.8.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket-client:python_websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket_client:python-websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket_client:python_websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket-client:websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket-client:websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket:python-websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket:python_websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket:python-websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket:python_websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket_client:websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket_client:websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket-client:python-websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket-client:python_websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket_client:python-websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket_client:python_websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket:websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket:websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket:websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket:websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket-client:websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket-client:websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket:python-websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket:python_websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket_client:websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket_client:websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket:websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket:websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:websocket-client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:websocket_client:1.8.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/websocket-client@1.9.0?package-id=f8c0dc7d2b3cb42a","type":"library","author":"liris ","name":"websocket-client","version":"1.9.0","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:python-websocket-client:python-websocket-client:1.9.0:*:*:*:*:*:*:*","purl":"pkg:pypi/websocket-client@1.9.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket-client:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket_client:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket_client:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_pp_project:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_pp_project:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket-client:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket-client:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket_client:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket_client:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket-client:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket-client:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket_client:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket_client:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_ppproject:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_ppproject:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_project:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_project:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:lirisproject:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:lirisproject:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_pp_project:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_pp_project:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-websocket:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_websocket:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket-client:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket-client:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket_client:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket_client:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris-pp:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris-pp:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_pp:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_pp:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_ppproject:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_ppproject:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_project:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_project:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris:python-websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris:python_websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:lirisproject:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:lirisproject:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:websocket:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris-pp:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris-pp:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_pp:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris_pp:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris:websocket-client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:liris:websocket_client:1.9.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/websocket_client-1.9.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/websocket_client-1.9.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/websocket_client-1.9.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/wsproto@1.2.0?package-id=37a4edf82e795291","type":"library","name":"wsproto","version":"1.2.0","cpe":"cpe:2.3:a:python-wsproto:python-wsproto:1.2.0:*:*:*:*:*:*:*","purl":"pkg:pypi/wsproto@1.2.0","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-wsproto:python_wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_wsproto:python-wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_wsproto:python_wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-wsproto:wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_wsproto:wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:wsproto:python-wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:wsproto:python_wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:wsproto:wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:wsproto:1.2.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/wsproto@1.3.2?package-id=46dc768d8411fb90","type":"library","author":"Benno Rice ","name":"wsproto","version":"1.3.2","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:benno_rice_\\","name":"xxhash","version":"3.6.0","licenses":[{"license":{"name":"BSD"}}],"cpe":"cpe:2.3:a:ifduyue_project:python-xxhash:3.6.0:*:*:*:*:*:*:*","purl":"pkg:pypi/xxhash@3.6.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:ifduyue_project:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ifduyueproject:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ifduyueproject:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_du_project:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_du_project:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-xxhash:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-xxhash:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_xxhash:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_xxhash:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_duproject:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_duproject:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ifduyue_project:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ifduyue:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ifduyue:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ifduyueproject:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_du_project:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-xxhash:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_xxhash:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xxhash:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xxhash:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_du:python-xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_du:python_xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_duproject:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:ifduyue:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:xxhash:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yue_du:xxhash:3.6.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/xxhash-3.6.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/xxhash-3.6.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/xxhash-3.6.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/yarl@1.20.1?package-id=138b2778aafcabfe","type":"library","name":"yarl","version":"1.20.1","cpe":"cpe:2.3:a:python-yarl:python-yarl:1.20.1:*:*:*:*:*:*:*","purl":"pkg:pypi/yarl@1.20.1","properties":[{"name":"syft:package:foundBy","value":"python-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-uv-lock-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-yarl:python_yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_yarl:python-yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_yarl:python_yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-yarl:yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_yarl:yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yarl:python-yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yarl:python_yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yarl:yarl:1.20.1:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/uv.lock"}]},{"bom-ref":"pkg:pypi/yarl@1.22.0?package-id=478907da5a99a633","type":"library","author":"Andrew Svetlov ","name":"yarl","version":"1.22.0","licenses":[{"license":{"id":"Apache-2.0"}}],"cpe":"cpe:2.3:a:andrew_svetlov_project:python-yarl:1.22.0:*:*:*:*:*:*:*","purl":"pkg:pypi/yarl@1.22.0","properties":[{"name":"syft:package:foundBy","value":"python-installed-package-cataloger"},{"name":"syft:package:language","value":"python"},{"name":"syft:package:type","value":"python"},{"name":"syft:package:metadataType","value":"python-package"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:python_yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:python-yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:python_yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov_project:yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:python-yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:python_yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:python-yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:python_yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlovproject:yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-yarl:python-yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-yarl:python_yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_yarl:python-yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_yarl:python_yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew-svetlov:yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:andrew_svetlov:yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python-yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:python_yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python-yarl:yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python_yarl:yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yarl:python-yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yarl:python_yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:python:yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:yarl:yarl:1.22.0:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/.venv/lib/python3.10/site-packages/yarl-1.22.0.dist-info/METADATA"},{"name":"syft:location:1:path","value":"/.venv/lib/python3.10/site-packages/yarl-1.22.0.dist-info/RECORD"},{"name":"syft:location:2:path","value":"/.venv/lib/python3.10/site-packages/yarl-1.22.0.dist-info/top_level.txt"}]},{"bom-ref":"pkg:pypi/zipp@3.23.0?package-id=4559e7e2237e9a48","type":"library","author":"\"Jason R. Coombs\" ","name":"zipp","version":"3.23.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:\\\"jason_r__coombs\\\"_\\= 3:
+            print(f"✅ Sufficient notification points (success + failure paths)")
+        else:
+            print(f"⚠️  Expected at least 3 notification calls, found {llm_notification_count}")
+
+        return True
+    except Exception as e:
+        print(f"❌ Failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+def test_job_endpoint_integration():
+    """Test that /llm/job endpoint extracts and passes webhook_config"""
+    print("\n" + "=" * 60)
+    print("TEST 5: /llm/job Endpoint Integration")
+    print("=" * 60)
+
+    try:
+        job_file = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'job.py')
+
+        with open(job_file, 'r') as f:
+            job_content = f.read()
+
+        # Find the llm_job_enqueue function
+        llm_job_start = job_content.find('async def llm_job_enqueue')
+        llm_job_end = job_content.find('\n\n@router', llm_job_start + 1)
+        if llm_job_end == -1:
+            llm_job_end = job_content.find('\n\nasync def', llm_job_start + 1)
+
+        llm_job_func = job_content[llm_job_start:llm_job_end]
+
+        # Check for webhook_config extraction
+        if 'webhook_config = None' in llm_job_func:
+            print("✅ llm_job_enqueue initializes webhook_config variable")
+        else:
+            print("❌ Missing webhook_config initialization")
+            return False
+
+        if 'if payload.webhook_config:' in llm_job_func:
+            print("✅ llm_job_enqueue checks for payload.webhook_config")
+        else:
+            print("❌ Missing webhook_config check")
+            return False
+
+        if 'webhook_config = payload.webhook_config.model_dump(mode=\'json\')' in llm_job_func:
+            print("✅ llm_job_enqueue converts webhook_config to dict")
+        else:
+            print("❌ Missing webhook_config.model_dump conversion")
+            return False
+
+        if 'webhook_config=webhook_config' in llm_job_func:
+            print("✅ llm_job_enqueue passes webhook_config to handle_llm_request")
+        else:
+            print("❌ Missing webhook_config parameter in handle_llm_request call")
+            return False
+
+        return True
+    except Exception as e:
+        print(f"❌ Failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+def test_create_new_task_integration():
+    """Test that create_new_task stores webhook_config in Redis"""
+    print("\n" + "=" * 60)
+    print("TEST 6: create_new_task Webhook Storage")
+    print("=" * 60)
+
+    try:
+        api_file = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'api.py')
+
+        with open(api_file, 'r') as f:
+            api_content = f.read()
+
+        # Find create_new_task function
+        create_task_start = api_content.find('async def create_new_task')
+        create_task_end = api_content.find('\nasync def ', create_task_start + 1)
+        if create_task_end == -1:
+            create_task_end = len(api_content)
+
+        create_task_func = api_content[create_task_start:create_task_end]
+
+        # Check for webhook_config storage
+        if 'if webhook_config:' in create_task_func:
+            print("✅ create_new_task checks for webhook_config")
+        else:
+            print("❌ Missing webhook_config check in create_new_task")
+            return False
+
+        if 'task_data["webhook_config"] = json.dumps(webhook_config)' in create_task_func:
+            print("✅ create_new_task stores webhook_config in Redis task data")
+        else:
+            print("❌ Missing webhook_config storage in task_data")
+            return False
+
+        # Check that webhook_config is passed to process_llm_extraction
+        if 'webhook_config' in create_task_func and 'background_tasks.add_task' in create_task_func:
+            print("✅ create_new_task passes webhook_config to background task")
+        else:
+            print("⚠️  Could not verify webhook_config passed to background task")
+
+        return True
+    except Exception as e:
+        print(f"❌ Failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+def test_pattern_consistency():
+    """Test that /llm/job follows the same pattern as /crawl/job"""
+    print("\n" + "=" * 60)
+    print("TEST 7: Pattern Consistency with /crawl/job")
+    print("=" * 60)
+
+    try:
+        api_file = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'api.py')
+
+        with open(api_file, 'r') as f:
+            api_content = f.read()
+
+        # Find handle_crawl_job to compare pattern
+        crawl_job_start = api_content.find('async def handle_crawl_job')
+        crawl_job_end = api_content.find('\nasync def ', crawl_job_start + 1)
+        if crawl_job_end == -1:
+            crawl_job_end = len(api_content)
+        crawl_job_func = api_content[crawl_job_start:crawl_job_end]
+
+        # Find process_llm_extraction
+        llm_extract_start = api_content.find('async def process_llm_extraction')
+        llm_extract_end = api_content.find('\nasync def ', llm_extract_start + 1)
+        if llm_extract_end == -1:
+            llm_extract_end = len(api_content)
+        llm_extract_func = api_content[llm_extract_start:llm_extract_end]
+
+        print("Checking pattern consistency...")
+
+        # Both should initialize WebhookDeliveryService
+        crawl_has_service = 'webhook_service = WebhookDeliveryService(config)' in crawl_job_func
+        llm_has_service = 'webhook_service = WebhookDeliveryService(config)' in llm_extract_func
+
+        if crawl_has_service and llm_has_service:
+            print("✅ Both initialize WebhookDeliveryService")
+        else:
+            print(f"❌ Service initialization mismatch (crawl: {crawl_has_service}, llm: {llm_has_service})")
+            return False
+
+        # Both should call notify_job_completion on success
+        crawl_notifies_success = 'status="completed"' in crawl_job_func and 'notify_job_completion' in crawl_job_func
+        llm_notifies_success = 'status="completed"' in llm_extract_func and 'notify_job_completion' in llm_extract_func
+
+        if crawl_notifies_success and llm_notifies_success:
+            print("✅ Both notify on success")
+        else:
+            print(f"❌ Success notification mismatch (crawl: {crawl_notifies_success}, llm: {llm_notifies_success})")
+            return False
+
+        # Both should call notify_job_completion on failure
+        crawl_notifies_failure = 'status="failed"' in crawl_job_func and 'error=' in crawl_job_func
+        llm_notifies_failure = 'status="failed"' in llm_extract_func and 'error=' in llm_extract_func
+
+        if crawl_notifies_failure and llm_notifies_failure:
+            print("✅ Both notify on failure")
+        else:
+            print(f"❌ Failure notification mismatch (crawl: {crawl_notifies_failure}, llm: {llm_notifies_failure})")
+            return False
+
+        print("✅ /llm/job follows the same pattern as /crawl/job")
+        return True
+
+    except Exception as e:
+        print(f"❌ Failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+def main():
+    """Run all tests"""
+    print("\n🧪 LLM Job Webhook Feature Validation")
+    print("=" * 60)
+    print("Testing that /llm/job now supports webhooks like /crawl/job")
+    print("=" * 60 + "\n")
+
+    results = []
+
+    # Run all tests
+    results.append(("LlmJobPayload Model", test_llm_job_payload_model()))
+    results.append(("handle_llm_request Signature", test_handle_llm_request_signature()))
+    results.append(("process_llm_extraction Signature", test_process_llm_extraction_signature()))
+    results.append(("Webhook Integration", test_webhook_integration_in_api()))
+    results.append(("/llm/job Endpoint", test_job_endpoint_integration()))
+    results.append(("create_new_task Storage", test_create_new_task_integration()))
+    results.append(("Pattern Consistency", test_pattern_consistency()))
+
+    # Print summary
+    print("\n" + "=" * 60)
+    print("TEST SUMMARY")
+    print("=" * 60)
+
+    passed = sum(1 for _, result in results if result)
+    total = len(results)
+
+    for test_name, result in results:
+        status = "✅ PASS" if result else "❌ FAIL"
+        print(f"{status} - {test_name}")
+
+    print(f"\n{'=' * 60}")
+    print(f"Results: {passed}/{total} tests passed")
+    print(f"{'=' * 60}")
+
+    if passed == total:
+        print("\n🎉 All tests passed! /llm/job webhook feature is correctly implemented.")
+        print("\n📝 Summary of changes:")
+        print("  1. LlmJobPayload model includes webhook_config field")
+        print("  2. /llm/job endpoint extracts and passes webhook_config")
+        print("  3. handle_llm_request accepts webhook_config parameter")
+        print("  4. create_new_task stores webhook_config in Redis")
+        print("  5. process_llm_extraction sends webhook notifications")
+        print("  6. Follows the same pattern as /crawl/job")
+        return 0
+    else:
+        print(f"\n⚠️  {total - passed} test(s) failed. Please review the output above.")
+        return 1
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/test_webhook_implementation.py b/test_webhook_implementation.py
new file mode 100644
index 0000000..072db8b
--- /dev/null
+++ b/test_webhook_implementation.py
@@ -0,0 +1,307 @@
+"""
+Simple test script to validate webhook implementation without running full server.
+
+This script tests:
+1. Webhook module imports and syntax
+2. WebhookDeliveryService initialization
+3. Payload construction logic
+4. Configuration parsing
+"""
+
+import sys
+import os
+import json
+from datetime import datetime, timezone
+
+# Add deploy/docker to path to import modules
+# sys.path.insert(0, '/home/user/crawl4ai/deploy/docker')
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'deploy', 'docker'))
+
+def test_imports():
+    """Test that all webhook-related modules can be imported"""
+    print("=" * 60)
+    print("TEST 1: Module Imports")
+    print("=" * 60)
+
+    try:
+        from webhook import WebhookDeliveryService
+        print("✅ webhook.WebhookDeliveryService imported successfully")
+    except Exception as e:
+        print(f"❌ Failed to import webhook module: {e}")
+        return False
+
+    try:
+        from schemas import WebhookConfig, WebhookPayload
+        print("✅ schemas.WebhookConfig imported successfully")
+        print("✅ schemas.WebhookPayload imported successfully")
+    except Exception as e:
+        print(f"❌ Failed to import schemas: {e}")
+        return False
+
+    return True
+
+def test_webhook_service_init():
+    """Test WebhookDeliveryService initialization"""
+    print("\n" + "=" * 60)
+    print("TEST 2: WebhookDeliveryService Initialization")
+    print("=" * 60)
+
+    try:
+        from webhook import WebhookDeliveryService
+
+        # Test with default config
+        config = {
+            "webhooks": {
+                "enabled": True,
+                "default_url": None,
+                "data_in_payload": False,
+                "retry": {
+                    "max_attempts": 5,
+                    "initial_delay_ms": 1000,
+                    "max_delay_ms": 32000,
+                    "timeout_ms": 30000
+                },
+                "headers": {
+                    "User-Agent": "Crawl4AI-Webhook/1.0"
+                }
+            }
+        }
+
+        service = WebhookDeliveryService(config)
+
+        print(f"✅ Service initialized successfully")
+        print(f"   - Max attempts: {service.max_attempts}")
+        print(f"   - Initial delay: {service.initial_delay}s")
+        print(f"   - Max delay: {service.max_delay}s")
+        print(f"   - Timeout: {service.timeout}s")
+
+        # Verify calculations
+        assert service.max_attempts == 5, "Max attempts should be 5"
+        assert service.initial_delay == 1.0, "Initial delay should be 1.0s"
+        assert service.max_delay == 32.0, "Max delay should be 32.0s"
+        assert service.timeout == 30.0, "Timeout should be 30.0s"
+
+        print("✅ All configuration values correct")
+
+        return True
+    except Exception as e:
+        print(f"❌ Service initialization failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+def test_webhook_config_model():
+    """Test WebhookConfig Pydantic model"""
+    print("\n" + "=" * 60)
+    print("TEST 3: WebhookConfig Model Validation")
+    print("=" * 60)
+
+    try:
+        from schemas import WebhookConfig
+        from pydantic import ValidationError
+
+        # Test valid config
+        valid_config = {
+            "webhook_url": "https://example.com/webhook",
+            "webhook_data_in_payload": True,
+            "webhook_headers": {"X-Secret": "token123"}
+        }
+
+        config = WebhookConfig(**valid_config)
+        print(f"✅ Valid config accepted:")
+        print(f"   - URL: {config.webhook_url}")
+        print(f"   - Data in payload: {config.webhook_data_in_payload}")
+        print(f"   - Headers: {config.webhook_headers}")
+
+        # Test minimal config
+        minimal_config = {
+            "webhook_url": "https://example.com/webhook"
+        }
+
+        config2 = WebhookConfig(**minimal_config)
+        print(f"✅ Minimal config accepted (defaults applied):")
+        print(f"   - URL: {config2.webhook_url}")
+        print(f"   - Data in payload: {config2.webhook_data_in_payload}")
+        print(f"   - Headers: {config2.webhook_headers}")
+
+        # Test invalid URL
+        try:
+            invalid_config = {
+                "webhook_url": "not-a-url"
+            }
+            config3 = WebhookConfig(**invalid_config)
+            print(f"❌ Invalid URL should have been rejected")
+            return False
+        except ValidationError as e:
+            print(f"✅ Invalid URL correctly rejected")
+
+        return True
+    except Exception as e:
+        print(f"❌ Model validation test failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+def test_payload_construction():
+    """Test webhook payload construction logic"""
+    print("\n" + "=" * 60)
+    print("TEST 4: Payload Construction")
+    print("=" * 60)
+
+    try:
+        # Simulate payload construction from notify_job_completion
+        task_id = "crawl_abc123"
+        task_type = "crawl"
+        status = "completed"
+        urls = ["https://example.com"]
+
+        payload = {
+            "task_id": task_id,
+            "task_type": task_type,
+            "status": status,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "urls": urls
+        }
+
+        print(f"✅ Basic payload constructed:")
+        print(json.dumps(payload, indent=2))
+
+        # Test with error
+        error_payload = {
+            "task_id": "crawl_xyz789",
+            "task_type": "crawl",
+            "status": "failed",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "urls": ["https://example.com"],
+            "error": "Connection timeout"
+        }
+
+        print(f"\n✅ Error payload constructed:")
+        print(json.dumps(error_payload, indent=2))
+
+        # Test with data
+        data_payload = {
+            "task_id": "crawl_def456",
+            "task_type": "crawl",
+            "status": "completed",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "urls": ["https://example.com"],
+            "data": {
+                "results": [
+                    {"url": "https://example.com", "markdown": "# Example"}
+                ]
+            }
+        }
+
+        print(f"\n✅ Data payload constructed:")
+        print(json.dumps(data_payload, indent=2))
+
+        return True
+    except Exception as e:
+        print(f"❌ Payload construction failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+def test_exponential_backoff():
+    """Test exponential backoff calculation"""
+    print("\n" + "=" * 60)
+    print("TEST 5: Exponential Backoff Calculation")
+    print("=" * 60)
+
+    try:
+        initial_delay = 1.0  # 1 second
+        max_delay = 32.0     # 32 seconds
+
+        print("Backoff delays for 5 attempts:")
+        for attempt in range(5):
+            delay = min(initial_delay * (2 ** attempt), max_delay)
+            print(f"  Attempt {attempt + 1}: {delay}s")
+
+        # Verify the sequence: 1s, 2s, 4s, 8s, 16s
+        expected = [1.0, 2.0, 4.0, 8.0, 16.0]
+        actual = [min(initial_delay * (2 ** i), max_delay) for i in range(5)]
+
+        assert actual == expected, f"Expected {expected}, got {actual}"
+        print("✅ Exponential backoff sequence correct")
+
+        return True
+    except Exception as e:
+        print(f"❌ Backoff calculation failed: {e}")
+        return False
+
+def test_api_integration():
+    """Test that api.py imports webhook module correctly"""
+    print("\n" + "=" * 60)
+    print("TEST 6: API Integration")
+    print("=" * 60)
+
+    try:
+        # Check if api.py can import webhook module
+        api_path = os.path.join(os.path.dirname(__file__), 'deploy', 'docker', 'api.py')
+        with open(api_path, 'r') as f:
+            api_content = f.read()
+
+        if 'from webhook import WebhookDeliveryService' in api_content:
+            print("✅ api.py imports WebhookDeliveryService")
+        else:
+            print("❌ api.py missing webhook import")
+            return False
+
+        if 'WebhookDeliveryService(config)' in api_content:
+            print("✅ api.py initializes WebhookDeliveryService")
+        else:
+            print("❌ api.py doesn't initialize WebhookDeliveryService")
+            return False
+
+        if 'notify_job_completion' in api_content:
+            print("✅ api.py calls notify_job_completion")
+        else:
+            print("❌ api.py doesn't call notify_job_completion")
+            return False
+
+        return True
+    except Exception as e:
+        print(f"❌ API integration check failed: {e}")
+        return False
+
+def main():
+    """Run all tests"""
+    print("\n🧪 Webhook Implementation Validation Tests")
+    print("=" * 60)
+
+    results = []
+
+    # Run tests
+    results.append(("Module Imports", test_imports()))
+    results.append(("Service Initialization", test_webhook_service_init()))
+    results.append(("Config Model", test_webhook_config_model()))
+    results.append(("Payload Construction", test_payload_construction()))
+    results.append(("Exponential Backoff", test_exponential_backoff()))
+    results.append(("API Integration", test_api_integration()))
+
+    # Print summary
+    print("\n" + "=" * 60)
+    print("TEST SUMMARY")
+    print("=" * 60)
+
+    passed = sum(1 for _, result in results if result)
+    total = len(results)
+
+    for test_name, result in results:
+        status = "✅ PASS" if result else "❌ FAIL"
+        print(f"{status} - {test_name}")
+
+    print(f"\n{'=' * 60}")
+    print(f"Results: {passed}/{total} tests passed")
+    print(f"{'=' * 60}")
+
+    if passed == total:
+        print("\n🎉 All tests passed! Webhook implementation is valid.")
+        return 0
+    else:
+        print(f"\n⚠️  {total - passed} test(s) failed. Please review the output above.")
+        return 1
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/tests/WEBHOOK_TEST_README.md b/tests/WEBHOOK_TEST_README.md
new file mode 100644
index 0000000..4f3c68a
--- /dev/null
+++ b/tests/WEBHOOK_TEST_README.md
@@ -0,0 +1,251 @@
+# Webhook Feature Test Script
+
+This directory contains a comprehensive test script for the webhook feature implementation.
+
+## Overview
+
+The `test_webhook_feature.sh` script automates the entire process of testing the webhook feature:
+
+1. ✅ Fetches and switches to the webhook feature branch
+2. ✅ Activates the virtual environment
+3. ✅ Installs all required dependencies
+4. ✅ Starts Redis server in background
+5. ✅ Starts Crawl4AI server in background
+6. ✅ Runs webhook integration test
+7. ✅ Verifies job completion via webhook
+8. ✅ Cleans up and returns to original branch
+
+## Prerequisites
+
+- Python 3.10+
+- Virtual environment already created (`venv/` in project root)
+- Git repository with the webhook feature branch
+- `redis-server` (script will attempt to install if missing)
+- `curl` and `lsof` commands available
+
+## Usage
+
+### Quick Start
+
+From the project root:
+
+```bash
+./tests/test_webhook_feature.sh
+```
+
+Or from the tests directory:
+
+```bash
+cd tests
+./test_webhook_feature.sh
+```
+
+### What the Script Does
+
+#### Step 1: Branch Management
+- Saves your current branch
+- Fetches the webhook feature branch from remote
+- Switches to the webhook feature branch
+
+#### Step 2: Environment Setup
+- Activates your existing virtual environment
+- Installs dependencies from `deploy/docker/requirements.txt`
+- Installs Flask for the webhook receiver
+
+#### Step 3: Service Startup
+- Starts Redis server on port 6379
+- Starts Crawl4AI server on port 11235
+- Waits for server health check to pass
+
+#### Step 4: Webhook Test
+- Creates a webhook receiver on port 8080
+- Submits a crawl job for `https://example.com` with webhook config
+- Waits for webhook notification (60s timeout)
+- Verifies webhook payload contains expected data
+
+#### Step 5: Cleanup
+- Stops webhook receiver
+- Stops Crawl4AI server
+- Stops Redis server
+- Returns to your original branch
+
+## Expected Output
+
+```
+[INFO] Starting webhook feature test script
+[INFO] Project root: /path/to/crawl4ai
+[INFO] Step 1: Fetching PR branch...
+[INFO] Current branch: develop
+[SUCCESS] Branch fetched
+[INFO] Step 2: Switching to branch: claude/implement-webhook-crawl-feature-011CULZY1Jy8N5MUkZqXkRVp
+[SUCCESS] Switched to webhook feature branch
+[INFO] Step 3: Activating virtual environment...
+[SUCCESS] Virtual environment activated
+[INFO] Step 4: Installing server dependencies...
+[SUCCESS] Dependencies installed
+[INFO] Step 5a: Starting Redis...
+[SUCCESS] Redis started (PID: 12345)
+[INFO] Step 5b: Starting server on port 11235...
+[INFO] Server started (PID: 12346)
+[INFO] Waiting for server to be ready...
+[SUCCESS] Server is ready!
+[INFO] Step 6: Creating webhook test script...
+[INFO] Running webhook test...
+
+🚀 Submitting crawl job with webhook...
+✅ Job submitted successfully, task_id: crawl_abc123
+⏳ Waiting for webhook notification...
+
+✅ Webhook received: {
+  "task_id": "crawl_abc123",
+  "task_type": "crawl",
+  "status": "completed",
+  "timestamp": "2025-10-22T00:00:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "data": { ... }
+}
+
+✅ Webhook received!
+   Task ID: crawl_abc123
+   Status: completed
+   URLs: ['https://example.com']
+   ✅ Data included in webhook payload
+   📄 Crawled 1 URL(s)
+      - https://example.com: 1234 chars
+
+🎉 Webhook test PASSED!
+
+[INFO] Step 7: Verifying test results...
+[SUCCESS] ✅ Webhook test PASSED!
+[SUCCESS] All tests completed successfully! 🎉
+[INFO] Cleanup will happen automatically...
+[INFO] Starting cleanup...
+[INFO] Stopping webhook receiver...
+[INFO] Stopping server...
+[INFO] Stopping Redis...
+[INFO] Switching back to branch: develop
+[SUCCESS] Cleanup complete
+```
+
+## Troubleshooting
+
+### Server Failed to Start
+
+If the server fails to start, check the logs:
+
+```bash
+tail -100 /tmp/crawl4ai_server.log
+```
+
+Common issues:
+- Port 11235 already in use: `lsof -ti:11235 | xargs kill -9`
+- Missing dependencies: Check that all packages are installed
+
+### Redis Connection Failed
+
+Check if Redis is running:
+
+```bash
+redis-cli ping
+# Should return: PONG
+```
+
+If not running:
+
+```bash
+redis-server --port 6379 --daemonize yes
+```
+
+### Webhook Not Received
+
+The script has a 60-second timeout for webhook delivery. If the webhook isn't received:
+
+1. Check server logs: `/tmp/crawl4ai_server.log`
+2. Verify webhook receiver is running on port 8080
+3. Check network connectivity between components
+
+### Script Interruption
+
+If the script is interrupted (Ctrl+C), cleanup happens automatically via trap. The script will:
+- Kill all background processes
+- Stop Redis
+- Return to your original branch
+
+To manually cleanup if needed:
+
+```bash
+# Kill processes by port
+lsof -ti:11235 | xargs kill -9  # Server
+lsof -ti:8080 | xargs kill -9   # Webhook receiver
+lsof -ti:6379 | xargs kill -9   # Redis
+
+# Return to your branch
+git checkout develop  # or your branch name
+```
+
+## Testing Different URLs
+
+To test with a different URL, modify the script or create a custom test:
+
+```python
+payload = {
+    "urls": ["https://your-url-here.com"],
+    "browser_config": {"headless": True},
+    "crawler_config": {"cache_mode": "bypass"},
+    "webhook_config": {
+        "webhook_url": "http://localhost:8080/webhook",
+        "webhook_data_in_payload": True
+    }
+}
+```
+
+## Files Generated
+
+The script creates temporary files:
+
+- `/tmp/crawl4ai_server.log` - Server output logs
+- `/tmp/test_webhook.py` - Webhook test Python script
+
+These are not cleaned up automatically so you can review them after the test.
+
+## Exit Codes
+
+- `0` - All tests passed successfully
+- `1` - Test failed (check output for details)
+
+## Safety Features
+
+- ✅ Automatic cleanup on exit, interrupt, or error
+- ✅ Returns to original branch on completion
+- ✅ Kills all background processes
+- ✅ Comprehensive error handling
+- ✅ Colored output for easy reading
+- ✅ Detailed logging at each step
+
+## Notes
+
+- The script uses `set -e` to exit on any command failure
+- All background processes are tracked and cleaned up
+- The virtual environment must exist before running
+- Redis must be available (installed or installable via apt-get/brew)
+
+## Integration with CI/CD
+
+This script can be integrated into CI/CD pipelines:
+
+```yaml
+# Example GitHub Actions
+- name: Test Webhook Feature
+  run: |
+    chmod +x tests/test_webhook_feature.sh
+    ./tests/test_webhook_feature.sh
+```
+
+## Support
+
+If you encounter issues:
+
+1. Check the troubleshooting section above
+2. Review server logs at `/tmp/crawl4ai_server.log`
+3. Ensure all prerequisites are met
+4. Open an issue with the full output of the script
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/adaptive/compare_performance.py b/tests/adaptive/compare_performance.py
new file mode 100644
index 0000000..516e49c
--- /dev/null
+++ b/tests/adaptive/compare_performance.py
@@ -0,0 +1,98 @@
+"""
+Compare performance before and after optimizations
+"""
+
+def read_baseline():
+    """Read baseline performance metrics"""
+    with open('performance_baseline.txt', 'r') as f:
+        content = f.read()
+    
+    # Extract key metrics
+    metrics = {}
+    lines = content.split('\n')
+    for i, line in enumerate(lines):
+        if 'Total Time:' in line:
+            metrics['total_time'] = float(line.split(':')[1].strip().split()[0])
+        elif 'Memory Used:' in line:
+            metrics['memory_mb'] = float(line.split(':')[1].strip().split()[0])
+        elif 'validate_coverage:' in line and i+1 < len(lines) and 'Avg Time:' in lines[i+2]:
+            metrics['validate_coverage_ms'] = float(lines[i+2].split(':')[1].strip().split()[0])
+        elif 'select_links:' in line and i+1 < len(lines) and 'Avg Time:' in lines[i+2]:
+            metrics['select_links_ms'] = float(lines[i+2].split(':')[1].strip().split()[0])
+        elif 'calculate_confidence:' in line and i+1 < len(lines) and 'Avg Time:' in lines[i+2]:
+            metrics['calculate_confidence_ms'] = float(lines[i+2].split(':')[1].strip().split()[0])
+            
+    return metrics
+
+
+def print_comparison(before_metrics, after_metrics):
+    """Print performance comparison"""
+    print("\n" + "="*80)
+    print("PERFORMANCE COMPARISON: BEFORE vs AFTER OPTIMIZATIONS")
+    print("="*80)
+    
+    # Total time
+    time_improvement = (before_metrics['total_time'] - after_metrics['total_time']) / before_metrics['total_time'] * 100
+    print(f"\n📊 Total Time:")
+    print(f"   Before: {before_metrics['total_time']:.2f} seconds")
+    print(f"   After:  {after_metrics['total_time']:.2f} seconds")
+    print(f"   Improvement: {time_improvement:.1f}% faster ✅" if time_improvement > 0 else f"   Slower: {-time_improvement:.1f}% ❌")
+    
+    # Memory
+    mem_improvement = (before_metrics['memory_mb'] - after_metrics['memory_mb']) / before_metrics['memory_mb'] * 100
+    print(f"\n💾 Memory Usage:")
+    print(f"   Before: {before_metrics['memory_mb']:.2f} MB")
+    print(f"   After:  {after_metrics['memory_mb']:.2f} MB")
+    print(f"   Improvement: {mem_improvement:.1f}% less memory ✅" if mem_improvement > 0 else f"   More memory: {-mem_improvement:.1f}% ❌")
+    
+    # Key operations
+    print(f"\n⚡ Key Operations:")
+    
+    # Validate coverage
+    if 'validate_coverage_ms' in before_metrics and 'validate_coverage_ms' in after_metrics:
+        val_improvement = (before_metrics['validate_coverage_ms'] - after_metrics['validate_coverage_ms']) / before_metrics['validate_coverage_ms'] * 100
+        print(f"\n   validate_coverage:")
+        print(f"     Before: {before_metrics['validate_coverage_ms']:.1f} ms")
+        print(f"     After:  {after_metrics['validate_coverage_ms']:.1f} ms")
+        print(f"     Improvement: {val_improvement:.1f}% faster ✅" if val_improvement > 0 else f"     Slower: {-val_improvement:.1f}% ❌")
+    
+    # Select links
+    if 'select_links_ms' in before_metrics and 'select_links_ms' in after_metrics:
+        sel_improvement = (before_metrics['select_links_ms'] - after_metrics['select_links_ms']) / before_metrics['select_links_ms'] * 100
+        print(f"\n   select_links:")
+        print(f"     Before: {before_metrics['select_links_ms']:.1f} ms")
+        print(f"     After:  {after_metrics['select_links_ms']:.1f} ms")
+        print(f"     Improvement: {sel_improvement:.1f}% faster ✅" if sel_improvement > 0 else f"     Slower: {-sel_improvement:.1f}% ❌")
+    
+    # Calculate confidence
+    if 'calculate_confidence_ms' in before_metrics and 'calculate_confidence_ms' in after_metrics:
+        calc_improvement = (before_metrics['calculate_confidence_ms'] - after_metrics['calculate_confidence_ms']) / before_metrics['calculate_confidence_ms'] * 100
+        print(f"\n   calculate_confidence:")
+        print(f"     Before: {before_metrics['calculate_confidence_ms']:.1f} ms")
+        print(f"     After:  {after_metrics['calculate_confidence_ms']:.1f} ms")
+        print(f"     Improvement: {calc_improvement:.1f}% faster ✅" if calc_improvement > 0 else f"     Slower: {-calc_improvement:.1f}% ❌")
+    
+    print("\n" + "="*80)
+    
+    # Overall assessment
+    if time_improvement > 50:
+        print("🎉 EXCELLENT OPTIMIZATION! More than 50% performance improvement!")
+    elif time_improvement > 30:
+        print("✅ GOOD OPTIMIZATION! Significant performance improvement!")
+    elif time_improvement > 10:
+        print("👍 DECENT OPTIMIZATION! Noticeable performance improvement!")
+    else:
+        print("🤔 MINIMAL IMPROVEMENT. Further optimization may be needed.")
+    
+    print("="*80)
+
+
+if __name__ == "__main__":
+    # Example usage - you'll run this after implementing optimizations
+    baseline = read_baseline()
+    print("Baseline metrics loaded:")
+    for k, v in baseline.items():
+        print(f"  {k}: {v}")
+    
+    print("\n⚠️  Run the performance test again after optimizations to compare!")
+    print("Then update this script with the new metrics to see the comparison.")
\ No newline at end of file
diff --git a/tests/adaptive/test_adaptive_crawler.py b/tests/adaptive/test_adaptive_crawler.py
new file mode 100644
index 0000000..066b189
--- /dev/null
+++ b/tests/adaptive/test_adaptive_crawler.py
@@ -0,0 +1,293 @@
+"""
+Test and demo script for Adaptive Crawler
+
+This script demonstrates the progressive crawling functionality
+with various configurations and use cases.
+"""
+
+import asyncio
+import json
+from pathlib import Path
+import time
+from typing import Dict, List
+from rich.console import Console
+from rich.table import Table
+from rich.progress import Progress
+from rich import print as rprint
+
+# Add parent directory to path for imports
+import sys
+sys.path.append(str(Path(__file__).parent.parent))
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    AdaptiveCrawler,
+    AdaptiveConfig,
+    CrawlState
+)
+
+
+console = Console()
+
+
+
+
+def print_relevant_content(crawler: AdaptiveCrawler, top_k: int = 3):
+    """Print most relevant content found"""
+    relevant = crawler.get_relevant_content(top_k=top_k)
+    
+    if not relevant:
+        console.print("[yellow]No relevant content found yet.[/yellow]")
+        return
+    
+    console.print(f"\n[bold cyan]Top {len(relevant)} Most Relevant Pages:[/bold cyan]")
+    for i, doc in enumerate(relevant, 1):
+        console.print(f"\n[green]{i}. {doc['url']}[/green]")
+        console.print(f"   Score: {doc['score']:.2f}")
+        # Show snippet
+        content = doc['content'] or ""
+        snippet = content[:200].replace('\n', ' ') + "..." if len(content) > 200 else content
+        console.print(f"   [dim]{snippet}[/dim]")
+
+
+async def test_basic_progressive_crawl():
+    """Test basic progressive crawling functionality"""
+    console.print("\n[bold yellow]Test 1: Basic Progressive Crawl[/bold yellow]")
+    console.print("Testing on Python documentation with query about async/await")
+    
+    config = AdaptiveConfig(
+        confidence_threshold=0.7,
+        max_pages=10,
+        top_k_links=2,
+        min_gain_threshold=0.1
+    )
+    
+    # Create crawler
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(
+            crawler=crawler,
+            config=config
+        )
+        
+        # Start progressive crawl
+        start_time = time.time()
+        state = await prog_crawler.digest(
+            start_url="https://docs.python.org/3/library/asyncio.html",
+            query="async await context managers"
+        )
+        elapsed = time.time() - start_time
+        
+        # Print results
+        prog_crawler.print_stats(detailed=False)
+        prog_crawler.print_stats(detailed=True)
+        print_relevant_content(prog_crawler)
+        
+        console.print(f"\n[green]Crawl completed in {elapsed:.2f} seconds[/green]")
+        console.print(f"Final confidence: {prog_crawler.confidence:.2%}")
+        console.print(f"URLs crawled: {list(state.crawled_urls)[:5]}...")  # Show first 5
+        
+        # Test export functionality
+        export_path = "knowledge_base_export.jsonl"
+        prog_crawler.export_knowledge_base(export_path)
+        console.print(f"[green]Knowledge base exported to {export_path}[/green]")
+        
+        # Clean up
+        Path(export_path).unlink(missing_ok=True)
+
+
+async def test_with_persistence():
+    """Test state persistence and resumption"""
+    console.print("\n[bold yellow]Test 2: Persistence and Resumption[/bold yellow]")
+    console.print("Testing state save/load functionality")
+    
+    state_path = "test_crawl_state.json"
+    
+    config = AdaptiveConfig(
+        confidence_threshold=0.6,
+        max_pages=5,
+        top_k_links=2,
+        save_state=True,
+        state_path=state_path
+    )
+    
+    # First crawl - partial
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(
+            crawler=crawler,
+            config=config
+        )
+        
+        state1 = await prog_crawler.digest(
+            start_url="https://httpbin.org",
+            query="http headers response"
+        )
+        
+        console.print(f"[cyan]First crawl: {len(state1.crawled_urls)} pages[/cyan]")
+        
+    # Resume crawl
+    config.max_pages = 10  # Increase limit
+    
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(
+            crawler=crawler,
+            config=config
+        )
+        
+        state2 = await prog_crawler.digest(
+            start_url="https://httpbin.org",
+            query="http headers response",
+            resume_from=state_path
+        )
+        
+        console.print(f"[green]Resumed crawl: {len(state2.crawled_urls)} total pages[/green]")
+        
+    # Clean up
+    Path(state_path).unlink(missing_ok=True)
+
+
+async def test_different_domains():
+    """Test on different types of websites"""
+    console.print("\n[bold yellow]Test 3: Different Domain Types[/bold yellow]")
+    
+    test_cases = [
+        {
+            "name": "Documentation Site",
+            "url": "https://docs.python.org/3/",
+            "query": "decorators and context managers"
+        },
+        {
+            "name": "API Documentation",  
+            "url": "https://httpbin.org",
+            "query": "http authentication headers"
+        }
+    ]
+    
+    for test in test_cases:
+        console.print(f"\n[cyan]Testing: {test['name']}[/cyan]")
+        console.print(f"URL: {test['url']}")
+        console.print(f"Query: {test['query']}")
+        
+        config = AdaptiveConfig(
+            confidence_threshold=0.6,
+            max_pages=5,
+            top_k_links=2
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            prog_crawler = AdaptiveCrawler(
+                crawler=crawler,
+                config=config
+            )
+            
+            start_time = time.time()
+            state = await prog_crawler.digest(
+                start_url=test['url'],
+                query=test['query']
+            )
+            elapsed = time.time() - start_time
+            
+            # Summary using print_stats
+            prog_crawler.print_stats(detailed=False)
+
+
+async def test_stopping_criteria():
+    """Test different stopping criteria"""
+    console.print("\n[bold yellow]Test 4: Stopping Criteria[/bold yellow]")
+    
+    # Test 1: High confidence threshold
+    console.print("\n[cyan]4.1 High confidence threshold (0.9)[/cyan]")
+    config = AdaptiveConfig(
+        confidence_threshold=0.9,  # Very high
+        max_pages=20,
+        top_k_links=3
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(crawler=crawler, config=config)
+        state = await prog_crawler.digest(
+            start_url="https://docs.python.org/3/library/",
+            query="python standard library"
+        )
+        
+        console.print(f"Pages needed for 90% confidence: {len(state.crawled_urls)}")
+        prog_crawler.print_stats(detailed=False)
+    
+    # Test 2: Page limit
+    console.print("\n[cyan]4.2 Page limit (3 pages max)[/cyan]")
+    config = AdaptiveConfig(
+        confidence_threshold=0.9,
+        max_pages=3,  # Very low limit
+        top_k_links=2
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(crawler=crawler, config=config)
+        state = await prog_crawler.digest(
+            start_url="https://docs.python.org/3/library/",
+            query="python standard library modules"
+        )
+        
+        console.print(f"Stopped by: {'Page limit' if len(state.crawled_urls) >= 3 else 'Other'}")
+        prog_crawler.print_stats(detailed=False)
+
+
+async def test_crawl_patterns():
+    """Analyze crawl patterns and link selection"""
+    console.print("\n[bold yellow]Test 5: Crawl Pattern Analysis[/bold yellow]")
+    
+    config = AdaptiveConfig(
+        confidence_threshold=0.7,
+        max_pages=8,
+        top_k_links=2,
+        min_gain_threshold=0.05
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(crawler=crawler, config=config)
+        
+        # Track crawl progress
+        console.print("\n[cyan]Crawl Progress:[/cyan]")
+        
+        state = await prog_crawler.digest(
+            start_url="https://httpbin.org",
+            query="http methods post get"
+        )
+        
+        # Show crawl order
+        console.print("\n[green]Crawl Order:[/green]")
+        for i, url in enumerate(state.crawl_order, 1):
+            console.print(f"{i}. {url}")
+        
+        # Show new terms discovered per page
+        console.print("\n[green]New Terms Discovered:[/green]")
+        for i, new_terms in enumerate(state.new_terms_history, 1):
+            console.print(f"Page {i}: {new_terms} new terms")
+        
+        # Final metrics
+        console.print(f"\n[yellow]Saturation reached: {state.metrics.get('saturation', 0):.2%}[/yellow]")
+
+
+async def main():
+    """Run all tests"""
+    console.print("[bold magenta]Adaptive Crawler Test Suite[/bold magenta]")
+    console.print("=" * 50)
+    
+    try:
+        # Run tests
+        await test_basic_progressive_crawl()
+        # await test_with_persistence()
+        # await test_different_domains()
+        # await test_stopping_criteria()
+        # await test_crawl_patterns()
+        
+        console.print("\n[bold green]✅ All tests completed successfully![/bold green]")
+        
+    except Exception as e:
+        console.print(f"\n[bold red]❌ Test failed with error: {e}[/bold red]")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    # Run the test suite
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/adaptive/test_confidence_debug.py b/tests/adaptive/test_confidence_debug.py
new file mode 100644
index 0000000..061df61
--- /dev/null
+++ b/tests/adaptive/test_confidence_debug.py
@@ -0,0 +1,182 @@
+"""
+Test script for debugging confidence calculation in adaptive crawler
+Focus: Testing why confidence decreases when crawling relevant URLs
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+from typing import List, Dict
+import math
+
+# Add parent directory to path for imports
+sys.path.append(str(Path(__file__).parent.parent))
+
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.adaptive_crawler import CrawlState, StatisticalStrategy
+from crawl4ai.models import CrawlResult
+
+
+class ConfidenceTestHarness:
+    """Test harness for analyzing confidence calculation"""
+    
+    def __init__(self):
+        self.strategy = StatisticalStrategy()
+        self.test_urls = [
+            'https://docs.python.org/3/library/asyncio.html',
+            'https://docs.python.org/3/library/asyncio-runner.html', 
+            'https://docs.python.org/3/library/asyncio-api-index.html',
+            'https://docs.python.org/3/library/contextvars.html',
+            'https://docs.python.org/3/library/asyncio-stream.html'
+        ]
+        self.query = "async await context manager"
+        
+    async def test_confidence_progression(self):
+        """Test confidence calculation as we crawl each URL"""
+        print(f"Testing confidence for query: '{self.query}'")
+        print("=" * 80)
+        
+        # Initialize state
+        state = CrawlState(query=self.query)
+        
+        # Create crawler
+        async with AsyncWebCrawler() as crawler:
+            for i, url in enumerate(self.test_urls, 1):
+                print(f"\n{i}. Crawling: {url}")
+                print("-" * 80)
+                
+                # Crawl the URL
+                result = await crawler.arun(url=url)
+                
+                # Extract markdown content
+                if hasattr(result, '_results') and result._results:
+                    result = result._results[0]
+                
+                # Create a mock CrawlResult with markdown
+                mock_result = type('CrawlResult', (), {
+                    'markdown': type('Markdown', (), {
+                        'raw_markdown': result.markdown.raw_markdown if hasattr(result, 'markdown') else ''
+                    })(),
+                    'url': url
+                })()
+                
+                # Update state
+                state.knowledge_base.append(mock_result)
+                await self.strategy.update_state(state, [mock_result])
+                
+                # Calculate metrics
+                confidence = await self.strategy.calculate_confidence(state)
+                
+                # Get individual components
+                coverage = state.metrics.get('coverage', 0)
+                consistency = state.metrics.get('consistency', 0)
+                saturation = state.metrics.get('saturation', 0)
+                
+                # Analyze term frequencies
+                query_terms = self.strategy._tokenize(self.query.lower())
+                term_stats = {}
+                for term in query_terms:
+                    term_stats[term] = {
+                        'tf': state.term_frequencies.get(term, 0),
+                        'df': state.document_frequencies.get(term, 0)
+                    }
+                
+                # Print detailed results
+                print(f"State after crawl {i}:")
+                print(f"  Total documents: {state.total_documents}")
+                print(f"  Unique terms: {len(state.term_frequencies)}")
+                print(f"  New terms added: {state.new_terms_history[-1] if state.new_terms_history else 0}")
+                
+                print(f"\nQuery term statistics:")
+                for term, stats in term_stats.items():
+                    print(f"  '{term}': tf={stats['tf']}, df={stats['df']}")
+                
+                print(f"\nMetrics:")
+                print(f"  Coverage: {coverage:.3f}")
+                print(f"  Consistency: {consistency:.3f}")
+                print(f"  Saturation: {saturation:.3f}")
+                print(f"  → Confidence: {confidence:.3f}")
+                
+                # Show coverage calculation details
+                print(f"\nCoverage calculation details:")
+                self._debug_coverage_calculation(state, query_terms)
+                
+                # Alert if confidence decreased
+                if i > 1 and confidence < state.metrics.get('prev_confidence', 0):
+                    print(f"\n⚠️  WARNING: Confidence decreased from {state.metrics.get('prev_confidence', 0):.3f} to {confidence:.3f}")
+                
+                state.metrics['prev_confidence'] = confidence
+    
+    def _debug_coverage_calculation(self, state: CrawlState, query_terms: List[str]):
+        """Debug coverage calculation step by step"""
+        coverage_score = 0.0
+        max_possible_score = 0.0
+        
+        for term in query_terms:
+            tf = state.term_frequencies.get(term, 0)
+            df = state.document_frequencies.get(term, 0)
+            
+            if df > 0:
+                idf = math.log((state.total_documents - df + 0.5) / (df + 0.5) + 1)
+                doc_coverage = df / state.total_documents
+                tf_boost = min(tf / df, 3.0)
+                term_score = doc_coverage * idf * (1 + 0.1 * math.log1p(tf_boost))
+                
+                print(f"    '{term}': doc_cov={doc_coverage:.2f}, idf={idf:.2f}, boost={1 + 0.1 * math.log1p(tf_boost):.2f} → score={term_score:.3f}")
+                coverage_score += term_score
+            else:
+                print(f"    '{term}': not found → score=0.000")
+            
+            max_possible_score += 1.0 * 1.0 * 1.1
+        
+        print(f"    Total: {coverage_score:.3f} / {max_possible_score:.3f} = {coverage_score/max_possible_score if max_possible_score > 0 else 0:.3f}")
+        
+        # New coverage calculation
+        print(f"\n  NEW Coverage calculation (without IDF):")
+        new_coverage = self._calculate_coverage_new(state, query_terms)
+        print(f"    → New Coverage: {new_coverage:.3f}")
+    
+    def _calculate_coverage_new(self, state: CrawlState, query_terms: List[str]) -> float:
+        """New coverage calculation without IDF"""
+        if not query_terms or state.total_documents == 0:
+            return 0.0
+            
+        term_scores = []
+        max_tf = max(state.term_frequencies.values()) if state.term_frequencies else 1
+        
+        for term in query_terms:
+            tf = state.term_frequencies.get(term, 0)
+            df = state.document_frequencies.get(term, 0)
+            
+            if df > 0:
+                # Document coverage: what fraction of docs contain this term
+                doc_coverage = df / state.total_documents
+                
+                # Frequency signal: normalized log frequency
+                freq_signal = math.log(1 + tf) / math.log(1 + max_tf) if max_tf > 0 else 0
+                
+                # Combined score: document coverage with frequency boost
+                term_score = doc_coverage * (1 + 0.5 * freq_signal)
+                
+                print(f"    '{term}': doc_cov={doc_coverage:.2f}, freq_signal={freq_signal:.2f} → score={term_score:.3f}")
+                term_scores.append(term_score)
+            else:
+                print(f"    '{term}': not found → score=0.000")
+                term_scores.append(0.0)
+        
+        # Average across all query terms
+        coverage = sum(term_scores) / len(term_scores)
+        return coverage
+
+
+async def main():
+    """Run the confidence test"""
+    tester = ConfidenceTestHarness()
+    await tester.test_confidence_progression()
+    
+    print("\n" + "=" * 80)
+    print("Test complete!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/adaptive/test_embedding_performance.py b/tests/adaptive/test_embedding_performance.py
new file mode 100644
index 0000000..b7c86f1
--- /dev/null
+++ b/tests/adaptive/test_embedding_performance.py
@@ -0,0 +1,254 @@
+"""
+Performance test for Embedding Strategy optimizations
+Measures time and memory usage before and after optimizations
+"""
+
+import asyncio
+import time
+import tracemalloc
+import numpy as np
+from pathlib import Path
+import sys
+import os
+
+# Add parent directory to path for imports
+sys.path.append(str(Path(__file__).parent.parent.parent))
+
+from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
+from crawl4ai.adaptive_crawler import EmbeddingStrategy, CrawlState
+from crawl4ai.models import CrawlResult
+
+
+class PerformanceMetrics:
+    def __init__(self):
+        self.start_time = 0
+        self.end_time = 0
+        self.start_memory = 0
+        self.peak_memory = 0
+        self.operation_times = {}
+        
+    def start(self):
+        tracemalloc.start()
+        self.start_time = time.perf_counter()
+        self.start_memory = tracemalloc.get_traced_memory()[0]
+        
+    def end(self):
+        self.end_time = time.perf_counter()
+        current, peak = tracemalloc.get_traced_memory()
+        self.peak_memory = peak
+        tracemalloc.stop()
+        
+    def record_operation(self, name: str, duration: float):
+        if name not in self.operation_times:
+            self.operation_times[name] = []
+        self.operation_times[name].append(duration)
+        
+    @property
+    def total_time(self):
+        return self.end_time - self.start_time
+        
+    @property
+    def memory_used_mb(self):
+        return (self.peak_memory - self.start_memory) / 1024 / 1024
+        
+    def print_summary(self, label: str):
+        print(f"\n{'='*60}")
+        print(f"Performance Summary: {label}")
+        print(f"{'='*60}")
+        print(f"Total Time: {self.total_time:.3f} seconds")
+        print(f"Memory Used: {self.memory_used_mb:.2f} MB")
+        
+        if self.operation_times:
+            print("\nOperation Breakdown:")
+            for op, times in self.operation_times.items():
+                avg_time = sum(times) / len(times)
+                total_time = sum(times)
+                print(f"  {op}:")
+                print(f"    - Calls: {len(times)}")
+                print(f"    - Avg Time: {avg_time*1000:.2f} ms")
+                print(f"    - Total Time: {total_time:.3f} s")
+
+
+async def create_mock_crawl_results(n: int) -> list:
+    """Create mock crawl results for testing"""
+    results = []
+    for i in range(n):
+        class MockMarkdown:
+            def __init__(self, content):
+                self.raw_markdown = content
+                
+        class MockResult:
+            def __init__(self, url, content):
+                self.url = url
+                self.markdown = MockMarkdown(content)
+                self.success = True
+                
+        content = f"This is test content {i} about async await coroutines event loops. " * 50
+        result = MockResult(f"https://example.com/page{i}", content)
+        results.append(result)
+    return results
+
+
+async def test_embedding_performance():
+    """Test the performance of embedding strategy operations"""
+    
+    # Configuration
+    n_kb_docs = 30  # Number of documents in knowledge base
+    n_queries = 10  # Number of query variations
+    n_links = 50   # Number of candidate links
+    n_iterations = 5  # Number of calculation iterations
+    
+    print(f"\nTest Configuration:")
+    print(f"- Knowledge Base Documents: {n_kb_docs}")
+    print(f"- Query Variations: {n_queries}")
+    print(f"- Candidate Links: {n_links}")
+    print(f"- Iterations: {n_iterations}")
+    
+    # Create embedding strategy
+    config = AdaptiveConfig(
+        strategy="embedding",
+        max_pages=50,
+        n_query_variations=n_queries,
+        embedding_model="sentence-transformers/all-MiniLM-L6-v2"  # 384 dimensions
+    )
+    
+    # Set up API key if available
+    if os.getenv('OPENAI_API_KEY'):
+        config.embedding_llm_config = {
+            'provider': 'openai/text-embedding-3-small',
+            'api_token': os.getenv('OPENAI_API_KEY'),
+            'embedding_model': 'text-embedding-3-small'
+        }
+    else:
+        config.embedding_llm_config = {
+            'provider': 'openai/gpt-4o-mini',
+            'api_token': 'dummy-key'
+        }
+    
+    strategy = EmbeddingStrategy(
+        embedding_model=config.embedding_model,
+        llm_config=config.embedding_llm_config
+    )
+    strategy.config = config
+    
+    # Initialize state
+    state = CrawlState()
+    state.query = "async await coroutines event loops tasks"
+    
+    # Start performance monitoring
+    metrics = PerformanceMetrics()
+    metrics.start()
+    
+    # 1. Generate query embeddings
+    print("\n1. Generating query embeddings...")
+    start = time.perf_counter()
+    query_embeddings, expanded_queries = await strategy.map_query_semantic_space(
+        state.query, 
+        config.n_query_variations
+    )
+    state.query_embeddings = query_embeddings
+    state.expanded_queries = expanded_queries
+    metrics.record_operation("query_embedding", time.perf_counter() - start)
+    print(f"   Generated {len(query_embeddings)} query embeddings")
+    
+    # 2. Build knowledge base incrementally
+    print("\n2. Building knowledge base...")
+    mock_results = await create_mock_crawl_results(n_kb_docs)
+    
+    for i in range(0, n_kb_docs, 5):  # Add 5 documents at a time
+        batch = mock_results[i:i+5]
+        start = time.perf_counter()
+        await strategy.update_state(state, batch)
+        metrics.record_operation("update_state", time.perf_counter() - start)
+        state.knowledge_base.extend(batch)
+    
+    print(f"   Knowledge base has {len(state.kb_embeddings)} documents")
+    
+    # 3. Test repeated confidence calculations
+    print(f"\n3. Testing {n_iterations} confidence calculations...")
+    for i in range(n_iterations):
+        start = time.perf_counter()
+        confidence = await strategy.calculate_confidence(state)
+        metrics.record_operation("calculate_confidence", time.perf_counter() - start)
+        print(f"   Iteration {i+1}: {confidence:.3f} ({(time.perf_counter() - start)*1000:.1f} ms)")
+    
+    # 4. Test coverage gap calculations
+    print(f"\n4. Testing coverage gap calculations...")
+    for i in range(n_iterations):
+        start = time.perf_counter()
+        gaps = strategy.find_coverage_gaps(state.kb_embeddings, state.query_embeddings)
+        metrics.record_operation("find_coverage_gaps", time.perf_counter() - start)
+        print(f"   Iteration {i+1}: {len(gaps)} gaps ({(time.perf_counter() - start)*1000:.1f} ms)")
+    
+    # 5. Test validation
+    print(f"\n5. Testing validation coverage...")
+    for i in range(n_iterations):
+        start = time.perf_counter()
+        val_score = await strategy.validate_coverage(state)
+        metrics.record_operation("validate_coverage", time.perf_counter() - start)
+        print(f"   Iteration {i+1}: {val_score:.3f} ({(time.perf_counter() - start)*1000:.1f} ms)")
+    
+    # 6. Create mock links for ranking
+    from crawl4ai.models import Link
+    mock_links = []
+    for i in range(n_links):
+        link = Link(
+            href=f"https://example.com/new{i}",
+            text=f"Link about async programming {i}",
+            title=f"Async Guide {i}"
+        )
+        mock_links.append(link)
+    
+    # 7. Test link selection
+    print(f"\n6. Testing link selection with {n_links} candidates...")
+    start = time.perf_counter()
+    scored_links = await strategy.select_links_for_expansion(
+        mock_links,
+        gaps,
+        state.kb_embeddings
+    )
+    metrics.record_operation("select_links", time.perf_counter() - start)
+    print(f"   Scored {len(scored_links)} links in {(time.perf_counter() - start)*1000:.1f} ms")
+    
+    # End monitoring
+    metrics.end()
+    
+    return metrics
+
+
+async def main():
+    """Run performance tests before and after optimizations"""
+    
+    print("="*80)
+    print("EMBEDDING STRATEGY PERFORMANCE TEST")
+    print("="*80)
+    
+    # Test current implementation
+    print("\n📊 Testing CURRENT Implementation...")
+    metrics_before = await test_embedding_performance()
+    metrics_before.print_summary("BEFORE Optimizations")
+    
+    # Store key metrics for comparison
+    total_time_before = metrics_before.total_time
+    memory_before = metrics_before.memory_used_mb
+    
+    # Calculate specific operation costs
+    calc_conf_avg = sum(metrics_before.operation_times.get("calculate_confidence", [])) / len(metrics_before.operation_times.get("calculate_confidence", [1]))
+    find_gaps_avg = sum(metrics_before.operation_times.get("find_coverage_gaps", [])) / len(metrics_before.operation_times.get("find_coverage_gaps", [1]))
+    validate_avg = sum(metrics_before.operation_times.get("validate_coverage", [])) / len(metrics_before.operation_times.get("validate_coverage", [1]))
+    
+    print(f"\n🔍 Key Bottlenecks Identified:")
+    print(f"   - calculate_confidence: {calc_conf_avg*1000:.1f} ms per call")
+    print(f"   - find_coverage_gaps: {find_gaps_avg*1000:.1f} ms per call")
+    print(f"   - validate_coverage: {validate_avg*1000:.1f} ms per call")
+    
+    print("\n" + "="*80)
+    print("EXPECTED IMPROVEMENTS AFTER OPTIMIZATION:")
+    print("- Distance calculations: 80-90% faster (vectorization)")
+    print("- Memory usage: 20-30% reduction (deduplication)")
+    print("- Overall performance: 60-70% improvement")
+    print("="*80)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/adaptive/test_embedding_strategy.py b/tests/adaptive/test_embedding_strategy.py
new file mode 100644
index 0000000..6e34b85
--- /dev/null
+++ b/tests/adaptive/test_embedding_strategy.py
@@ -0,0 +1,634 @@
+"""
+Test and demo script for Embedding-based Adaptive Crawler
+
+This script demonstrates the embedding-based adaptive crawling
+with semantic space coverage and gap-driven expansion.
+"""
+
+import asyncio
+import os
+from pathlib import Path
+import time
+from rich.console import Console
+from rich import print as rprint
+import sys
+
+# Add parent directory to path for imports
+sys.path.append(str(Path(__file__).parent.parent.parent))
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    AdaptiveCrawler,
+    AdaptiveConfig,
+    CrawlState
+)
+
+console = Console()
+
+
+async def test_basic_embedding_crawl():
+    """Test basic embedding-based adaptive crawling"""
+    console.print("\n[bold yellow]Test 1: Basic Embedding-based Crawl[/bold yellow]")
+    console.print("Testing semantic space coverage with query expansion")
+    
+    # Configure with embedding strategy
+    config = AdaptiveConfig(
+        strategy="embedding",
+        confidence_threshold=0.7,  # Not used for stopping in embedding strategy
+        min_gain_threshold=0.01,
+        max_pages=15,
+        top_k_links=3,
+        n_query_variations=8,
+        embedding_model="sentence-transformers/all-MiniLM-L6-v2"  # Fast, good quality
+    )
+    
+    # For query expansion, we need an LLM config
+    llm_config = {
+        'provider': 'openai/gpt-4o-mini',
+        'api_token': os.getenv('OPENAI_API_KEY')
+    }
+    
+    if not llm_config['api_token']:
+        console.print("[red]Warning: OPENAI_API_KEY not set. Using mock data for demo.[/red]")
+        # Continue with mock for demo purposes
+    
+    config.embedding_llm_config = llm_config
+    
+    # Create crawler
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(
+            crawler=crawler,
+            config=config
+        )
+        
+        # Start adaptive crawl
+        start_time = time.time()
+        console.print("\n[cyan]Starting semantic adaptive crawl...[/cyan]")
+        
+        state = await prog_crawler.digest(
+            start_url="https://docs.python.org/3/library/asyncio.html",
+            query="async await coroutines event loops"
+        )
+        elapsed = time.time() - start_time
+        
+        # Print results
+        console.print(f"\n[green]Crawl completed in {elapsed:.2f} seconds[/green]")
+        prog_crawler.print_stats(detailed=False)
+        
+        # Show semantic coverage details
+        console.print("\n[bold cyan]Semantic Coverage Details:[/bold cyan]")
+        if state.expanded_queries:
+            console.print(f"Query expanded to {len(state.expanded_queries)} variations")
+            console.print("Sample variations:")
+            for i, q in enumerate(state.expanded_queries[:3], 1):
+                console.print(f"  {i}. {q}")
+        
+        if state.semantic_gaps:
+            console.print(f"\nSemantic gaps identified: {len(state.semantic_gaps)}")
+        
+        console.print(f"\nFinal confidence: {prog_crawler.confidence:.2%}")
+        console.print(f"Is Sufficient: {'Yes (Validated)' if prog_crawler.is_sufficient else 'No'}")
+        console.print(f"Pages needed: {len(state.crawled_urls)}")
+
+
+async def test_embedding_vs_statistical(use_openai=False):
+    """Compare embedding strategy with statistical strategy"""
+    console.print("\n[bold yellow]Test 2: Embedding vs Statistical Strategy Comparison[/bold yellow]")
+    
+    test_url = "https://httpbin.org"
+    test_query = "http headers authentication api"
+    
+    # Test 1: Statistical strategy
+    console.print("\n[cyan]1. Statistical Strategy:[/cyan]")
+    config_stat = AdaptiveConfig(
+        strategy="statistical",
+        confidence_threshold=0.7,
+        max_pages=10
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        stat_crawler = AdaptiveCrawler(crawler=crawler, config=config_stat)
+        
+        start_time = time.time()
+        state_stat = await stat_crawler.digest(start_url=test_url, query=test_query)
+        stat_time = time.time() - start_time
+        
+        stat_pages = len(state_stat.crawled_urls)
+        stat_confidence = stat_crawler.confidence
+    
+    # Test 2: Embedding strategy
+    console.print("\n[cyan]2. Embedding Strategy:[/cyan]")
+    config_emb = AdaptiveConfig(
+        strategy="embedding",
+        confidence_threshold=0.7,  # Not used for stopping
+        max_pages=10,
+        n_query_variations=5,
+        min_gain_threshold=0.01
+    )
+    
+    # Use OpenAI if available or requested
+    if use_openai and os.getenv('OPENAI_API_KEY'):
+        config_emb.embedding_llm_config = {
+            'provider': 'openai/text-embedding-3-small',
+            'api_token': os.getenv('OPENAI_API_KEY'),
+            'embedding_model': 'text-embedding-3-small'
+        }
+        console.print("[cyan]Using OpenAI embeddings[/cyan]")
+    else:
+        # Default config will try sentence-transformers
+        config_emb.embedding_llm_config = {
+            'provider': 'openai/gpt-4o-mini',
+            'api_token': os.getenv('OPENAI_API_KEY', 'dummy-key')
+        }
+    
+    async with AsyncWebCrawler() as crawler:
+        emb_crawler = AdaptiveCrawler(crawler=crawler, config=config_emb)
+        
+        start_time = time.time()
+        state_emb = await emb_crawler.digest(start_url=test_url, query=test_query)
+        emb_time = time.time() - start_time
+        
+        emb_pages = len(state_emb.crawled_urls)
+        emb_confidence = emb_crawler.confidence
+    
+    # Compare results
+    console.print("\n[bold green]Comparison Results:[/bold green]")
+    console.print(f"Statistical: {stat_pages} pages in {stat_time:.2f}s, confidence: {stat_confidence:.2%}, sufficient: {stat_crawler.is_sufficient}")
+    console.print(f"Embedding:   {emb_pages} pages in {emb_time:.2f}s, confidence: {emb_confidence:.2%}, sufficient: {emb_crawler.is_sufficient}")
+    
+    if emb_pages < stat_pages:
+        efficiency = ((stat_pages - emb_pages) / stat_pages) * 100
+        console.print(f"\n[green]Embedding strategy used {efficiency:.0f}% fewer pages![/green]")
+    
+    # Show validation info for embedding
+    if hasattr(state_emb, 'metrics') and 'validation_confidence' in state_emb.metrics:
+        console.print(f"Embedding validation score: {state_emb.metrics['validation_confidence']:.2%}")
+
+
+async def test_custom_embedding_provider():
+    """Test with different embedding providers"""
+    console.print("\n[bold yellow]Test 3: Custom Embedding Provider[/bold yellow]")
+    
+    # Example with OpenAI embeddings
+    config = AdaptiveConfig(
+        strategy="embedding",
+        confidence_threshold=0.8,  # Not used for stopping
+        max_pages=10,
+        min_gain_threshold=0.01,
+        n_query_variations=5
+    )
+    
+    # Configure to use OpenAI embeddings instead of sentence-transformers
+    config.embedding_llm_config = {
+        'provider': 'openai/text-embedding-3-small',
+        'api_token': os.getenv('OPENAI_API_KEY'),
+        'embedding_model': 'text-embedding-3-small'
+    }
+    
+    if not config.embedding_llm_config['api_token']:
+        console.print("[yellow]Skipping OpenAI embedding test - no API key[/yellow]")
+        return
+    
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(crawler=crawler, config=config)
+        
+        console.print("Using OpenAI embeddings for semantic analysis...")
+        state = await prog_crawler.digest(
+            start_url="https://httpbin.org",
+            query="api endpoints json response"
+        )
+        
+        prog_crawler.print_stats(detailed=False)
+
+
+async def test_knowledge_export_import():
+    """Test exporting and importing semantic knowledge bases"""
+    console.print("\n[bold yellow]Test 4: Semantic Knowledge Base Export/Import[/bold yellow]")
+    
+    config = AdaptiveConfig(
+        strategy="embedding",
+        confidence_threshold=0.7,  # Not used for stopping
+        max_pages=5,
+        min_gain_threshold=0.01,
+        n_query_variations=4
+    )
+    
+    # First crawl
+    async with AsyncWebCrawler() as crawler:
+        crawler1 = AdaptiveCrawler(crawler=crawler, config=config)
+        
+        console.print("\n[cyan]Building initial knowledge base...[/cyan]")
+        state1 = await crawler1.digest(
+            start_url="https://httpbin.org",
+            query="http methods headers"
+        )
+        
+        # Export
+        export_path = "semantic_kb.jsonl"
+        crawler1.export_knowledge_base(export_path)
+        console.print(f"[green]Exported {len(state1.knowledge_base)} documents with embeddings[/green]")
+    
+    # Import and continue
+    async with AsyncWebCrawler() as crawler:
+        crawler2 = AdaptiveCrawler(crawler=crawler, config=config)
+        
+        console.print("\n[cyan]Importing knowledge base...[/cyan]")
+        await crawler2.import_knowledge_base(export_path)
+        
+        # Continue with new query - should be faster
+        console.print("\n[cyan]Extending with new query...[/cyan]")
+        state2 = await crawler2.digest(
+            start_url="https://httpbin.org",
+            query="authentication oauth tokens"
+        )
+        
+        console.print(f"[green]Total knowledge base: {len(state2.knowledge_base)} documents[/green]")
+    
+    # Cleanup
+    Path(export_path).unlink(missing_ok=True)
+
+
+async def test_gap_visualization():
+    """Visualize semantic gaps and coverage"""
+    console.print("\n[bold yellow]Test 5: Semantic Gap Analysis[/bold yellow]")
+    
+    config = AdaptiveConfig(
+        strategy="embedding",
+        confidence_threshold=0.9,  # Not used for stopping
+        max_pages=8,
+        n_query_variations=6,
+        min_gain_threshold=0.01
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(crawler=crawler, config=config)
+        
+        # Initial crawl
+        state = await prog_crawler.digest(
+            start_url="https://docs.python.org/3/library/",
+            query="concurrency threading multiprocessing"
+        )
+        
+        # Analyze gaps
+        console.print("\n[bold cyan]Semantic Gap Analysis:[/bold cyan]")
+        console.print(f"Query variations: {len(state.expanded_queries)}")
+        console.print(f"Knowledge documents: {len(state.knowledge_base)}")
+        console.print(f"Identified gaps: {len(state.semantic_gaps)}")
+        
+        if state.semantic_gaps:
+            console.print("\n[yellow]Gap sizes (distance from coverage):[/yellow]")
+            for i, (_, distance) in enumerate(state.semantic_gaps[:5], 1):
+                console.print(f"  Gap {i}: {distance:.3f}")
+        
+        # Show crawl progression
+        console.print("\n[cyan]Crawl Order (gap-driven selection):[/cyan]")
+        for i, url in enumerate(state.crawl_order[:5], 1):
+            console.print(f"  {i}. {url}")
+
+
+async def test_fast_convergence_with_relevant_query():
+    """Test that both strategies reach high confidence quickly with relevant queries"""
+    console.print("\n[bold yellow]Test 7: Fast Convergence with Relevant Query[/bold yellow]")
+    console.print("Testing that strategies reach 80%+ confidence within 2-3 batches")
+    
+    # Test scenarios
+    test_cases = [
+        {
+            "name": "Python Async Documentation",
+            "url": "https://docs.python.org/3/library/asyncio.html",
+            "query": "async await coroutines event loops tasks"
+        }
+    ]
+    
+    for test_case in test_cases:
+        console.print(f"\n[bold cyan]Testing: {test_case['name']}[/bold cyan]")
+        console.print(f"URL: {test_case['url']}")
+        console.print(f"Query: {test_case['query']}")
+        
+        # Test Embedding Strategy
+        console.print("\n[yellow]Embedding Strategy:[/yellow]")
+        config_emb = AdaptiveConfig(
+            strategy="embedding",
+            confidence_threshold=0.8,
+            max_pages=9,
+            top_k_links=3,
+            min_gain_threshold=0.01,
+            n_query_variations=5
+        )
+        
+        # Configure embeddings
+        config_emb.embedding_llm_config = {
+            'provider': 'openai/gpt-4o-mini',
+            'api_token': os.getenv('OPENAI_API_KEY'),
+        }
+        
+        async with AsyncWebCrawler() as crawler:
+            emb_crawler = AdaptiveCrawler(crawler=crawler, config=config_emb)
+            
+            start_time = time.time()
+            state = await emb_crawler.digest(
+                start_url=test_case['url'],
+                query=test_case['query']
+            )
+            
+            # Get batch breakdown
+            total_pages = len(state.crawled_urls)
+            for i in range(0, total_pages, 3):
+                batch_num = (i // 3) + 1
+                batch_pages = min(3, total_pages - i)
+                pages_so_far = i + batch_pages
+                estimated_confidence = state.metrics.get('confidence', 0) * (pages_so_far / total_pages)
+                
+                console.print(f"Batch {batch_num}: {batch_pages} pages → Confidence: {estimated_confidence:.1%} {'✅' if estimated_confidence >= 0.8 else '❌'}")
+            
+            final_confidence = emb_crawler.confidence
+            console.print(f"[green]Final: {total_pages} pages → Confidence: {final_confidence:.1%} {'✅ (Sufficient!)' if emb_crawler.is_sufficient else '❌'}[/green]")
+            
+            # Show learning metrics for embedding
+            if 'avg_min_distance' in state.metrics:
+                console.print(f"[dim]Avg gap distance: {state.metrics['avg_min_distance']:.3f}[/dim]")
+            if 'validation_confidence' in state.metrics:
+                console.print(f"[dim]Validation score: {state.metrics['validation_confidence']:.1%}[/dim]")
+        
+        # Test Statistical Strategy
+        console.print("\n[yellow]Statistical Strategy:[/yellow]")
+        config_stat = AdaptiveConfig(
+            strategy="statistical",
+            confidence_threshold=0.8,
+            max_pages=9,
+            top_k_links=3,
+            min_gain_threshold=0.01
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            stat_crawler = AdaptiveCrawler(crawler=crawler, config=config_stat)
+            
+            # Track batch progress
+            batch_results = []
+            current_pages = 0
+            
+            # Custom batch tracking
+            start_time = time.time()
+            state = await stat_crawler.digest(
+                start_url=test_case['url'],
+                query=test_case['query']
+            )
+            
+            # Get batch breakdown (every 3 pages)
+            total_pages = len(state.crawled_urls)
+            for i in range(0, total_pages, 3):
+                batch_num = (i // 3) + 1
+                batch_pages = min(3, total_pages - i)
+                # Estimate confidence at this point (simplified)
+                pages_so_far = i + batch_pages
+                estimated_confidence = state.metrics.get('confidence', 0) * (pages_so_far / total_pages)
+                
+                console.print(f"Batch {batch_num}: {batch_pages} pages → Confidence: {estimated_confidence:.1%} {'✅' if estimated_confidence >= 0.8 else '❌'}")
+            
+            final_confidence = stat_crawler.confidence
+            console.print(f"[green]Final: {total_pages} pages → Confidence: {final_confidence:.1%} {'✅ (Sufficient!)' if stat_crawler.is_sufficient else '❌'}[/green]")
+        
+        
+
+
+async def test_irrelevant_query_behavior():
+    """Test how embedding strategy handles completely irrelevant queries"""
+    console.print("\n[bold yellow]Test 8: Irrelevant Query Behavior[/bold yellow]")
+    console.print("Testing embedding strategy with a query that has no semantic relevance to the content")
+    
+    # Test with irrelevant query on Python async documentation
+    test_case = {
+        "name": "Irrelevant Query on Python Docs",
+        "url": "https://docs.python.org/3/library/asyncio.html",
+        "query": "how to cook fried rice with vegetables"
+    }
+    
+    console.print(f"\n[bold cyan]Testing: {test_case['name']}[/bold cyan]")
+    console.print(f"URL: {test_case['url']} (Python async documentation)")
+    console.print(f"Query: '{test_case['query']}' (completely irrelevant)")
+    console.print("\n[dim]Expected behavior: Low confidence, high distances, no convergence[/dim]")
+    
+    # Configure embedding strategy
+    config_emb = AdaptiveConfig(
+        strategy="embedding",
+        confidence_threshold=0.8,
+        max_pages=9,
+        top_k_links=3,
+        min_gain_threshold=0.01,
+        n_query_variations=5,
+        embedding_min_relative_improvement=0.05,  # Lower threshold to see more iterations
+        embedding_min_confidence_threshold=0.1  # Will stop if confidence < 10%
+    )
+    
+    # Configure embeddings using the correct format
+    config_emb.embedding_llm_config = {
+        'provider': 'openai/gpt-4o-mini',
+        'api_token': os.getenv('OPENAI_API_KEY'),
+    }
+    
+    async with AsyncWebCrawler() as crawler:
+        emb_crawler = AdaptiveCrawler(crawler=crawler, config=config_emb)
+        
+        start_time = time.time()
+        state = await emb_crawler.digest(
+            start_url=test_case['url'],
+            query=test_case['query']
+        )
+        elapsed = time.time() - start_time
+        
+        # Analyze results
+        console.print(f"\n[bold]Results after {elapsed:.1f} seconds:[/bold]")
+        
+        # Basic metrics
+        total_pages = len(state.crawled_urls)
+        final_confidence = emb_crawler.confidence
+        
+        console.print(f"\nPages crawled: {total_pages}")
+        console.print(f"Final confidence: {final_confidence:.1%} {'✅' if emb_crawler.is_sufficient else '❌'}")
+        
+        # Distance metrics
+        if 'avg_min_distance' in state.metrics:
+            console.print(f"\n[yellow]Distance Metrics:[/yellow]")
+            console.print(f"  Average minimum distance: {state.metrics['avg_min_distance']:.3f}")
+            console.print(f"  Close neighbors (<0.3): {state.metrics.get('avg_close_neighbors', 0):.1f}")
+            console.print(f"  Very close neighbors (<0.2): {state.metrics.get('avg_very_close_neighbors', 0):.1f}")
+            
+            # Interpret distances
+            avg_dist = state.metrics['avg_min_distance']
+            if avg_dist > 0.8:
+                console.print(f"  [red]→ Very poor match (distance > 0.8)[/red]")
+            elif avg_dist > 0.6:
+                console.print(f"  [yellow]→ Poor match (distance > 0.6)[/yellow]")
+            elif avg_dist > 0.4:
+                console.print(f"  [blue]→ Moderate match (distance > 0.4)[/blue]")
+            else:
+                console.print(f"  [green]→ Good match (distance < 0.4)[/green]")
+        
+        # Show sample expanded queries
+        if state.expanded_queries:
+            console.print(f"\n[yellow]Sample Query Variations Generated:[/yellow]")
+            for i, q in enumerate(state.expanded_queries[:3], 1):
+                console.print(f"  {i}. {q}")
+        
+        # Show crawl progression
+        console.print(f"\n[yellow]Crawl Progression:[/yellow]")
+        for i, url in enumerate(state.crawl_order[:5], 1):
+            console.print(f"  {i}. {url}")
+        if len(state.crawl_order) > 5:
+            console.print(f"  ... and {len(state.crawl_order) - 5} more")
+        
+        # Validation score
+        if 'validation_confidence' in state.metrics:
+            console.print(f"\n[yellow]Validation:[/yellow]")
+            console.print(f"  Validation score: {state.metrics['validation_confidence']:.1%}")
+        
+        # Why it stopped
+        if 'stopped_reason' in state.metrics:
+            console.print(f"\n[yellow]Stopping Reason:[/yellow] {state.metrics['stopped_reason']}")
+            if state.metrics.get('is_irrelevant', False):
+                console.print("[red]→ Query and content are completely unrelated![/red]")
+        elif total_pages >= config_emb.max_pages:
+            console.print(f"\n[yellow]Stopping Reason:[/yellow] Reached max pages limit ({config_emb.max_pages})")
+        
+        # Summary
+        console.print(f"\n[bold]Summary:[/bold]")
+        if final_confidence < 0.2:
+            console.print("[red]✗ As expected: Query is completely irrelevant to content[/red]")
+            console.print("[green]✓ The embedding strategy correctly identified no semantic match[/green]")
+        else:
+            console.print(f"[yellow]⚠ Unexpected: Got {final_confidence:.1%} confidence for irrelevant query[/yellow]")
+            console.print("[yellow]  This may indicate the query variations are too broad[/yellow]")
+
+
+async def test_high_dimensional_handling():
+    """Test handling of high-dimensional embedding spaces"""
+    console.print("\n[bold yellow]Test 6: High-Dimensional Embedding Space Handling[/bold yellow]")
+    console.print("Testing how the system handles 384+ dimensional embeddings")
+    
+    config = AdaptiveConfig(
+        strategy="embedding",
+        confidence_threshold=0.8,  # Not used for stopping
+        max_pages=5,
+        n_query_variations=8,  # Will create 9 points total
+        min_gain_threshold=0.01,
+        embedding_model="sentence-transformers/all-MiniLM-L6-v2"  # 384 dimensions
+    )
+    
+    # Use OpenAI if available, otherwise mock
+    if os.getenv('OPENAI_API_KEY'):
+        config.embedding_llm_config = {
+            'provider': 'openai/text-embedding-3-small',
+            'api_token': os.getenv('OPENAI_API_KEY'),
+            'embedding_model': 'text-embedding-3-small'
+        }
+    else:
+        config.embedding_llm_config = {
+            'provider': 'openai/gpt-4o-mini',
+            'api_token': 'mock-key'
+        }
+    
+    async with AsyncWebCrawler() as crawler:
+        prog_crawler = AdaptiveCrawler(crawler=crawler, config=config)
+        
+        console.print("\n[cyan]Testing with high-dimensional embeddings (384D)...[/cyan]")
+        
+        try:
+            state = await prog_crawler.digest(
+                start_url="https://httpbin.org",
+                query="api endpoints json"
+            )
+            
+            console.print(f"[green]✓ Successfully handled {len(state.expanded_queries)} queries in 384D space[/green]")
+            console.print(f"Coverage shape type: {type(state.coverage_shape)}")
+            
+            if isinstance(state.coverage_shape, dict):
+                console.print(f"Coverage model: centroid + radius")
+                console.print(f"  - Center shape: {state.coverage_shape['center'].shape if 'center' in state.coverage_shape else 'N/A'}")
+                console.print(f"  - Radius: {state.coverage_shape.get('radius', 'N/A'):.3f}")
+            
+        except Exception as e:
+            console.print(f"[red]Error: {e}[/red]")
+            console.print("[yellow]This demonstrates why alpha shapes don't work in high dimensions[/yellow]")
+
+
+async def main():
+    """Run all embedding strategy tests"""
+    console.print("[bold magenta]Embedding-based Adaptive Crawler Test Suite[/bold magenta]")
+    console.print("=" * 60)
+    
+    try:
+        # Check if we have required dependencies
+        has_sentence_transformers = True
+        has_numpy = True
+        
+        try:
+            import numpy
+            console.print("[green]✓ NumPy installed[/green]")
+        except ImportError:
+            has_numpy = False
+            console.print("[red]Missing numpy[/red]")
+            
+        # Try to import sentence_transformers but catch numpy compatibility errors
+        try:
+            import sentence_transformers
+            console.print("[green]✓ Sentence-transformers installed[/green]")
+        except (ImportError, RuntimeError, ValueError) as e:
+            has_sentence_transformers = False
+            console.print(f"[yellow]Warning: sentence-transformers not available[/yellow]")
+            console.print("[yellow]Tests will use OpenAI embeddings if available or mock data[/yellow]")
+        
+        # Run tests based on available dependencies
+        if has_numpy:
+            # Check if we should use OpenAI for embeddings
+            use_openai = not has_sentence_transformers and os.getenv('OPENAI_API_KEY')
+            
+            if not has_sentence_transformers and not os.getenv('OPENAI_API_KEY'):
+                console.print("\n[red]Neither sentence-transformers nor OpenAI API key available[/red]")
+                console.print("[yellow]Please set OPENAI_API_KEY or fix sentence-transformers installation[/yellow]")
+                return
+            
+            # Run all tests
+            # await test_basic_embedding_crawl()
+            # await test_embedding_vs_statistical(use_openai=use_openai)
+            
+            # Run the fast convergence test - this is the most important one
+            # await test_fast_convergence_with_relevant_query()
+            
+            # Test with irrelevant query
+            await test_irrelevant_query_behavior()
+            
+            # Only run OpenAI-specific test if we have API key
+            # if os.getenv('OPENAI_API_KEY'):
+            #     await test_custom_embedding_provider()
+            
+            # # Skip tests that require sentence-transformers when it's not available
+            # if has_sentence_transformers:
+            #     await test_knowledge_export_import()
+            #     await test_gap_visualization()
+            # else:
+            #     console.print("\n[yellow]Skipping tests that require sentence-transformers due to numpy compatibility issues[/yellow]")
+            
+            # This test should work with mock data
+            # await test_high_dimensional_handling()
+        else:
+            console.print("\n[red]Cannot run tests without NumPy[/red]")
+            return
+        
+        console.print("\n[bold green]✅ All tests completed![/bold green]")
+        
+    except Exception as e:
+        console.print(f"\n[bold red]❌ Test failed: {e}[/bold red]")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+    
+    
+    
+    
+    
+    
+    
+    
diff --git a/tests/adaptive/test_llm_embedding.py b/tests/adaptive/test_llm_embedding.py
new file mode 100644
index 0000000..5279474
--- /dev/null
+++ b/tests/adaptive/test_llm_embedding.py
@@ -0,0 +1,154 @@
+import asyncio
+import os
+from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig, LLMConfig
+
+
+async def test_configuration(name: str, config: AdaptiveConfig, url: str, query: str):
+    """Test a specific configuration"""
+    print(f"\n{'='*60}")
+    print(f"Configuration: {name}")
+    print(f"{'='*60}")
+    
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        adaptive = AdaptiveCrawler(crawler, config)
+        result = await adaptive.digest(start_url=url, query=query)
+        
+        print("\n" + "="*50)
+        print("CRAWL STATISTICS")
+        print("="*50)
+        adaptive.print_stats(detailed=False)
+        
+        # Get the most relevant content found
+        print("\n" + "="*50)
+        print("MOST RELEVANT PAGES")
+        print("="*50)
+        
+        relevant_pages = adaptive.get_relevant_content(top_k=5)
+        for i, page in enumerate(relevant_pages, 1):
+            print(f"\n{i}. {page['url']}")
+            print(f"   Relevance Score: {page['score']:.2%}")
+            
+            # Show a snippet of the content
+            content = page['content'] or ""
+            if content:
+                snippet = content[:200].replace('\n', ' ')
+                if len(content) > 200:
+                    snippet += "..."
+                print(f"   Preview: {snippet}")
+        
+        print(f"\n{'='*50}")
+        print(f"Pages crawled: {len(result.crawled_urls)}")
+        print(f"Final confidence: {adaptive.confidence:.1%}")
+        print(f"Stopped reason: {result.metrics.get('stopped_reason', 'max_pages')}")
+        
+        if result.metrics.get('is_irrelevant', False):
+            print("⚠️  Query detected as irrelevant!")
+        
+        return result
+
+
+async def llm_embedding():
+    """Demonstrate various embedding configurations"""
+    
+    print("EMBEDDING STRATEGY CONFIGURATION EXAMPLES")
+    print("=" * 60)
+    
+    # Base URL and query for testing
+    test_url = "https://docs.python.org/3/library/asyncio.html"
+    
+    openai_llm_config = LLMConfig(
+        provider='openai/text-embedding-3-small',
+        api_token=os.getenv('OPENAI_API_KEY'),
+        temperature=0.7,
+        max_tokens=2000
+    )
+    config_openai = AdaptiveConfig(
+        strategy="embedding",
+        max_pages=10,
+        
+        # Use OpenAI embeddings
+        embedding_llm_config=openai_llm_config,
+        # embedding_llm_config={
+        #     'provider': 'openai/text-embedding-3-small',
+        #     'api_token': os.getenv('OPENAI_API_KEY')
+        # },
+        
+        # OpenAI embeddings are high quality, can be stricter
+        embedding_k_exp=4.0,
+        n_query_variations=12
+    )
+    
+    await test_configuration(
+        "OpenAI Embeddings",
+        config_openai,
+        test_url,
+        # "event-driven architecture patterns"
+        "async await context managers coroutines"
+    )
+    return
+    
+    
+
+async def basic_adaptive_crawling():
+    """Basic adaptive crawling example"""
+    
+    # Initialize the crawler
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        # Create an adaptive crawler with default settings (statistical strategy)
+        adaptive = AdaptiveCrawler(crawler)
+        
+        # Note: You can also use embedding strategy for semantic understanding:
+        # from crawl4ai import AdaptiveConfig
+        # config = AdaptiveConfig(strategy="embedding")
+        # adaptive = AdaptiveCrawler(crawler, config)
+        
+        # Start adaptive crawling
+        print("Starting adaptive crawl for Python async programming information...")
+        result = await adaptive.digest(
+            start_url="https://docs.python.org/3/library/asyncio.html",
+            query="async await context managers coroutines"
+        )
+        
+        # Display crawl statistics
+        print("\n" + "="*50)
+        print("CRAWL STATISTICS")
+        print("="*50)
+        adaptive.print_stats(detailed=False)
+        
+        # Get the most relevant content found
+        print("\n" + "="*50)
+        print("MOST RELEVANT PAGES")
+        print("="*50)
+        
+        relevant_pages = adaptive.get_relevant_content(top_k=5)
+        for i, page in enumerate(relevant_pages, 1):
+            print(f"\n{i}. {page['url']}")
+            print(f"   Relevance Score: {page['score']:.2%}")
+            
+            # Show a snippet of the content
+            content = page['content'] or ""
+            if content:
+                snippet = content[:200].replace('\n', ' ')
+                if len(content) > 200:
+                    snippet += "..."
+                print(f"   Preview: {snippet}")
+        
+        # Show final confidence
+        print(f"\n{'='*50}")
+        print(f"Final Confidence: {adaptive.confidence:.2%}")
+        print(f"Total Pages Crawled: {len(result.crawled_urls)}")
+        print(f"Knowledge Base Size: {len(adaptive.state.knowledge_base)} documents")
+        
+        
+        if adaptive.confidence >= 0.8:
+            print("✓ High confidence - can answer detailed questions about async Python")
+        elif adaptive.confidence >= 0.6:
+            print("~ Moderate confidence - can answer basic questions") 
+        else:
+            print("✗ Low confidence - need more information")
+
+
+
+if __name__ == "__main__":
+    asyncio.run(llm_embedding())
+    # asyncio.run(basic_adaptive_crawling())
\ No newline at end of file
diff --git a/tests/adaptive/test_query_llm_config.py b/tests/adaptive/test_query_llm_config.py
new file mode 100644
index 0000000..f20c100
--- /dev/null
+++ b/tests/adaptive/test_query_llm_config.py
@@ -0,0 +1,284 @@
+"""
+E2E tests for separate embedding and query LLM configs (issue #1682).
+
+Tests that AdaptiveConfig.query_llm_config flows correctly through
+AdaptiveCrawler → EmbeddingStrategy → map_query_semantic_space,
+and that the right config is used for embeddings vs query expansion.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+from unittest.mock import patch, MagicMock, AsyncMock
+import numpy as np
+
+sys.path.append(str(Path(__file__).parent.parent.parent))
+
+from crawl4ai import AdaptiveConfig, LLMConfig
+from crawl4ai.adaptive_crawler import EmbeddingStrategy, AdaptiveCrawler
+
+
+# ---------------------------------------------------------------------------
+# Test 1: Config plumbing — AdaptiveConfig → AdaptiveCrawler → EmbeddingStrategy
+# ---------------------------------------------------------------------------
+
+def test_config_plumbing():
+    """query_llm_config flows from AdaptiveConfig through _create_strategy."""
+    config = AdaptiveConfig(
+        strategy="embedding",
+        embedding_llm_config=LLMConfig(provider="openai/text-embedding-3-small", api_token="emb-key"),
+        query_llm_config=LLMConfig(provider="openai/gpt-4o-mini", api_token="query-key"),
+    )
+
+    # Simulate what AdaptiveCrawler.__init__ does
+    with patch("crawl4ai.adaptive_crawler.AsyncWebCrawler"):
+        crawler_mock = MagicMock()
+        adaptive = AdaptiveCrawler(crawler=crawler_mock, config=config)
+
+    strategy = adaptive.strategy
+    assert isinstance(strategy, EmbeddingStrategy)
+
+    # Strategy should have both configs
+    assert strategy.query_llm_config is not None
+    query_dict = strategy._get_query_llm_config_dict()
+    assert query_dict["provider"] == "openai/gpt-4o-mini"
+    assert query_dict["api_token"] == "query-key"
+
+    emb_dict = strategy._get_embedding_llm_config_dict()
+    assert emb_dict["provider"] == "openai/text-embedding-3-small"
+    assert emb_dict["api_token"] == "emb-key"
+
+    print("PASS: test_config_plumbing")
+
+
+# ---------------------------------------------------------------------------
+# Test 2: Backward compat — no query_llm_config falls back to llm_config
+# ---------------------------------------------------------------------------
+
+def test_backward_compat_fallback():
+    """When query_llm_config is not set, falls back to llm_config (legacy)."""
+    strategy = EmbeddingStrategy(
+        embedding_model="sentence-transformers/all-MiniLM-L6-v2",
+        llm_config={"provider": "openai/gpt-4o-mini", "api_token": "shared-key"},
+        query_llm_config=None,
+    )
+    # No AdaptiveConfig attached → should fall back to llm_config
+    result = strategy._get_query_llm_config_dict()
+    assert result["provider"] == "openai/gpt-4o-mini"
+    assert result["api_token"] == "shared-key"
+    print("PASS: test_backward_compat_fallback")
+
+
+def test_backward_compat_no_config():
+    """When nothing is set, returns None (caller uses hardcoded defaults)."""
+    strategy = EmbeddingStrategy()
+    result = strategy._get_query_llm_config_dict()
+    assert result is None
+    print("PASS: test_backward_compat_no_config")
+
+
+# ---------------------------------------------------------------------------
+# Test 3: Fallback priority chain
+# ---------------------------------------------------------------------------
+
+def test_fallback_priority():
+    """Explicit query_llm_config beats AdaptiveConfig beats llm_config."""
+    config = AdaptiveConfig(
+        strategy="embedding",
+        query_llm_config={"provider": "config-level", "api_token": "cfg"},
+    )
+    strategy = EmbeddingStrategy(
+        llm_config={"provider": "legacy-level", "api_token": "leg"},
+        query_llm_config={"provider": "strategy-level", "api_token": "strat"},
+    )
+    strategy.config = config
+
+    # Strategy-level should win
+    result = strategy._get_query_llm_config_dict()
+    assert result["provider"] == "strategy-level"
+
+    # Remove strategy-level → config-level should win
+    strategy.query_llm_config = None
+    result = strategy._get_query_llm_config_dict()
+    assert result["provider"] == "config-level"
+
+    # Remove config-level → legacy llm_config should win
+    config.query_llm_config = None
+    result = strategy._get_query_llm_config_dict()
+    assert result["provider"] == "legacy-level"
+
+    # Remove everything → None
+    strategy.llm_config = None
+    result = strategy._get_query_llm_config_dict()
+    assert result is None
+
+    print("PASS: test_fallback_priority")
+
+
+# ---------------------------------------------------------------------------
+# Test 4: E2E — map_query_semantic_space uses query config, not embedding config
+# ---------------------------------------------------------------------------
+
+async def test_map_query_uses_query_config():
+    """map_query_semantic_space should call perform_completion_with_backoff
+    with the query LLM config (chat model), NOT the embedding config."""
+
+    config = AdaptiveConfig(
+        strategy="embedding",
+        embedding_llm_config=LLMConfig(
+            provider="openai/text-embedding-3-small",
+            api_token="emb-key",
+            base_url="https://emb.example.com",
+        ),
+        query_llm_config=LLMConfig(
+            provider="openai/gpt-4o-mini",
+            api_token="query-key",
+            base_url="https://query.example.com",
+        ),
+    )
+
+    strategy = EmbeddingStrategy(
+        embedding_model="sentence-transformers/all-MiniLM-L6-v2",
+        llm_config=config.embedding_llm_config,
+        query_llm_config=config.query_llm_config,
+    )
+    strategy.config = config
+
+    # Mock perform_completion_with_backoff to capture its arguments
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock()]
+    mock_response.choices[0].message.content = json.dumps({
+        "queries": [f"variation {i}" for i in range(13)]
+    })
+
+    captured_kwargs = {}
+
+    def mock_completion(**kwargs):
+        # Also accept positional-style
+        captured_kwargs.update(kwargs)
+        return mock_response
+
+    # Also mock _get_embeddings to avoid real embedding calls
+    fake_embeddings = np.random.rand(11, 384).astype(np.float32)
+
+    with patch("crawl4ai.utils.perform_completion_with_backoff", side_effect=mock_completion):
+        with patch.object(strategy, "_get_embeddings", new_callable=AsyncMock, return_value=fake_embeddings):
+            await strategy.map_query_semantic_space("test query", n_synthetic=10)
+
+    # Verify the query config was used, NOT the embedding config
+    assert captured_kwargs["provider"] == "openai/gpt-4o-mini", \
+        f"Expected query model, got {captured_kwargs['provider']}"
+    assert captured_kwargs["api_token"] == "query-key", \
+        f"Expected query-key, got {captured_kwargs['api_token']}"
+    assert captured_kwargs["base_url"] == "https://query.example.com", \
+        f"Expected query base_url, got {captured_kwargs['base_url']}"
+
+    # Verify backoff params are passed (bug fix)
+    assert "base_delay" in captured_kwargs
+    assert "max_attempts" in captured_kwargs
+    assert "exponential_factor" in captured_kwargs
+
+    print("PASS: test_map_query_uses_query_config")
+
+
+# ---------------------------------------------------------------------------
+# Test 5: E2E — legacy single-config still works for query expansion
+# ---------------------------------------------------------------------------
+
+async def test_legacy_single_config_for_query():
+    """When only embedding_llm_config is set (old usage), query expansion
+    falls back to it via llm_config → still works."""
+
+    single_config = LLMConfig(
+        provider="openai/gpt-4o-mini",
+        api_token="single-key",
+    )
+
+    config = AdaptiveConfig(
+        strategy="embedding",
+        embedding_llm_config=single_config,
+        # No query_llm_config — legacy usage
+    )
+
+    strategy = EmbeddingStrategy(
+        embedding_model="sentence-transformers/all-MiniLM-L6-v2",
+        llm_config=config.embedding_llm_config,  # This is how _create_strategy passes it
+        # No query_llm_config
+    )
+    strategy.config = config
+
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock()]
+    mock_response.choices[0].message.content = json.dumps({
+        "queries": [f"variation {i}" for i in range(13)]
+    })
+
+    captured_kwargs = {}
+
+    def mock_completion(**kwargs):
+        captured_kwargs.update(kwargs)
+        return mock_response
+
+    fake_embeddings = np.random.rand(11, 384).astype(np.float32)
+
+    with patch("crawl4ai.utils.perform_completion_with_backoff", side_effect=mock_completion):
+        with patch.object(strategy, "_get_embeddings", new_callable=AsyncMock, return_value=fake_embeddings):
+            await strategy.map_query_semantic_space("test query", n_synthetic=10)
+
+    # Should fall back to llm_config (the single shared config)
+    assert captured_kwargs["provider"] == "openai/gpt-4o-mini"
+    assert captured_kwargs["api_token"] == "single-key"
+
+    print("PASS: test_legacy_single_config_for_query")
+
+
+# ---------------------------------------------------------------------------
+# Test 6: LLMConfig.to_dict() includes backoff params (bug fix verification)
+# ---------------------------------------------------------------------------
+
+def test_to_dict_includes_backoff():
+    """_embedding_llm_config_dict now uses to_dict() which includes backoff params."""
+    config = AdaptiveConfig(
+        embedding_llm_config=LLMConfig(
+            provider="openai/text-embedding-3-small",
+            api_token="test",
+            backoff_base_delay=5,
+            backoff_max_attempts=10,
+            backoff_exponential_factor=3,
+        ),
+    )
+    d = config._embedding_llm_config_dict
+    assert d["backoff_base_delay"] == 5
+    assert d["backoff_max_attempts"] == 10
+    assert d["backoff_exponential_factor"] == 3
+    print("PASS: test_to_dict_includes_backoff")
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+async def main():
+    print("=" * 60)
+    print("E2E Tests: Separate Embedding & Query LLM Configs (#1682)")
+    print("=" * 60)
+
+    # Sync tests
+    test_config_plumbing()
+    test_backward_compat_fallback()
+    test_backward_compat_no_config()
+    test_fallback_priority()
+    test_to_dict_includes_backoff()
+
+    # Async tests
+    await test_map_query_uses_query_config()
+    await test_legacy_single_config_for_query()
+
+    print("\n" + "=" * 60)
+    print("ALL TESTS PASSED")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/adversarial/test_domain_mapper_adversarial.py b/tests/adversarial/test_domain_mapper_adversarial.py
new file mode 100644
index 0000000..1456358
--- /dev/null
+++ b/tests/adversarial/test_domain_mapper_adversarial.py
@@ -0,0 +1,136 @@
+"""Adversarial tests for DomainMapper — edge cases, failures, tough scenarios."""
+import asyncio
+import pytest
+import pytest_asyncio
+from crawl4ai import DomainMapper, DomainMapperConfig
+
+
+pytestmark = pytest.mark.network
+
+
+@pytest_asyncio.fixture
+async def mapper():
+    async with DomainMapper() as m:
+        yield m
+
+
+class TestDomainMapperAdversarial:
+
+    @pytest.mark.asyncio
+    async def test_nonexistent_domain(self, mapper):
+        """Domain that doesn't exist should return empty, not crash."""
+        config = DomainMapperConfig(
+            source="sitemap+probe",
+            extract_head=False,
+            verbose=False,
+        )
+        results = await mapper.scan("thiswillneverexist12345678.dev", config)
+        assert results == []
+
+    @pytest.mark.asyncio
+    async def test_invalid_source(self, mapper):
+        """Invalid source should raise ValueError."""
+        config = DomainMapperConfig(source="sitemap+bogus")
+        with pytest.raises(ValueError, match="Invalid source"):
+            await mapper.scan("example.com", config)
+
+    @pytest.mark.asyncio
+    async def test_empty_domain(self, mapper):
+        """Empty domain string should not crash."""
+        config = DomainMapperConfig(
+            source="probe",
+            extract_head=False,
+            verbose=False,
+        )
+        results = await mapper.scan("", config)
+        assert isinstance(results, list)
+
+    @pytest.mark.asyncio
+    async def test_domain_with_scheme(self, mapper):
+        """Domain with https:// prefix should be handled."""
+        config = DomainMapperConfig(
+            source="sitemap",
+            extract_head=False,
+            verbose=False,
+            max_urls=5,
+        )
+        results = await mapper.scan("https://docs.crawl4ai.com", config)
+        assert len(results) >= 1
+
+    @pytest.mark.asyncio
+    async def test_soft_404_filtering_spa(self, mapper):
+        """SPA sites (app.superdesign.dev) should have soft-404s filtered."""
+        config = DomainMapperConfig(
+            source="probe",
+            extract_head=False,
+            soft_404_detection=True,
+            verbose=False,
+        )
+        results = await mapper.scan("app.superdesign.dev", config)
+        # All probed paths on this SPA should be filtered as soft-404s
+        # (the site returns 200 for every path with the same shell)
+        probe_urls = [r for r in results if r["source"] == "probe"]
+        assert len(probe_urls) == 0, \
+            f"Expected 0 valid probe URLs on SPA, got {len(probe_urls)}: {[r['url'] for r in probe_urls]}"
+
+    @pytest.mark.asyncio
+    async def test_rate_limiting(self, mapper):
+        """hits_per_sec=2 should not crash."""
+        config = DomainMapperConfig(
+            source="probe",
+            extract_head=False,
+            hits_per_sec=2,
+            verbose=False,
+        )
+        results = await mapper.scan("docs.crawl4ai.com", config)
+        assert isinstance(results, list)
+
+    @pytest.mark.asyncio
+    async def test_unicode_in_results(self, mapper):
+        """Results should handle unicode URLs gracefully."""
+        config = DomainMapperConfig(
+            source="sitemap",
+            extract_head=False,
+            verbose=False,
+        )
+        results = await mapper.scan("docs.crawl4ai.com", config)
+        for r in results:
+            assert isinstance(r["url"], str)
+
+    @pytest.mark.asyncio
+    async def test_config_clone(self):
+        """DomainMapperConfig.clone() should work."""
+        config = DomainMapperConfig(source="sitemap", max_urls=10)
+        cloned = config.clone(max_urls=20, force=True)
+        assert cloned.max_urls == 20
+        assert cloned.force is True
+        assert cloned.source == "sitemap"  # inherited
+
+    @pytest.mark.asyncio
+    async def test_concurrent_host_scanning(self, mapper):
+        """Multiple hosts scanned in parallel should not race."""
+        config = DomainMapperConfig(
+            source="sitemap+crt+probe",
+            extract_head=False,
+            concurrency=20,
+            verbose=False,
+            force=True,
+        )
+        results = await mapper.scan("superdesign.dev", config)
+        # Verify no duplicate URLs in results
+        urls = [r["url"] for r in results]
+        # Normalized dedup should prevent exact duplicates
+        assert isinstance(results, list)
+        assert len(results) > 0
+
+    @pytest.mark.asyncio
+    async def test_probe_without_crt(self, mapper):
+        """source='probe' alone should still work (just scans base domain)."""
+        config = DomainMapperConfig(
+            source="probe",
+            extract_head=False,
+            verbose=False,
+        )
+        results = await mapper.scan("docs.crawl4ai.com", config)
+        # Should find at least / and /docs
+        assert len(results) >= 1
diff --git a/tests/async/sample_wikipedia.html b/tests/async/sample_wikipedia.html
new file mode 100644
index 0000000..a22b3e3
--- /dev/null
+++ b/tests/async/sample_wikipedia.html
@@ -0,0 +1,2179 @@
+
+
+Apple - Wikipedia
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
    
+	
    
+		
    
+			
+			
+
+
+		
    
+		
    
+			
+
+
+			
+
+		
    
+	
    
+
    
+
    
+	
    
+		
    
+			
    
+		
    
+		
    
+			
    
+		
    
+			
+		
    
+	
    
+	
    
+				
+			
    
+		
    
+		
    
+			
    
+				
    
+					
+					
    Apple
    
+							
+
    
+	
+	
+	
    
+
+		
    
+			
+			
+			
    Edit links
    
+		
    
+
+	
    
+
    
+
    
+				
    
+					
    
+						
    
+							
+						
    
+						
    
+							
+				
+							
+						
    
+					
    
+				
    
+				
    
+					
    
+						
+						
+					
    
+				
    
+				
    
+					
    
+							
    
+		
    This is a good article. Click here for more information.
    
+		
    Page semi-protected
    
+		
    
+
+						
    From Wikipedia, the free encyclopedia
    
+					
    
+					
    
+					
+					
+					
    
+
    This article is about the fruit. For the technology company, see Apple Inc. For other uses, see Apple (disambiguation).
    
+
    "Apple tree" redirects here. For other uses, see Apple tree (disambiguation).
    
+
    
+
+
+
+
    
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    Apple
+
    
+
    'Cripps Pink' apples
+
    
+
    Flowers
+
    Scientific classification Edit this classification
+
    Kingdom:
+Plantae
+
    Clade:
+Tracheophytes
+
    Clade:
+Angiosperms
+
    Clade:
+Eudicots
+
    Clade:
+Rosids
+
    Order:
+Rosales
+
    Family:
+Rosaceae
+
    Genus:
+Malus
+
    Species:
+
    M. domestica
    
+
    Binomial name
+
    Malus domestica
    
+
    Synonyms[1][2]
+
    
+
    • M. communis Desf., 1768
      • 
+
    • M. pumila Mil.
      • 
+
    • M. frutescens Medik.
      • 
+
    • M. paradisiaca (L.) Medikus
      • 
+
    • M. sylvestris Mil.
      • 
+
    • Pyrus malus L.
      • 
+
    • Pyrus malus var. paradisiaca L.
      • 
+
    • Pyrus dioica Moench
    
+
    
+
    An apple is a round, edible fruit produced by an apple tree (Malus spp., among them the domestic or orchard apple; Malus domestica). Apple trees are cultivated worldwide and are the most widely grown species in the genus Malus. The tree originated in Central Asia, where its wild ancestor, Malus sieversii, is still found. Apples have been grown for thousands of years in Eurasia and were introduced to North America by European colonists. Apples have religious and mythological significance in many cultures, including Norse, Greek, and European Christian tradition.
+
    Apples grown from seed tend to be very different from those of their parents, and the resultant fruit frequently lacks desired characteristics. For commercial purposes, including botanical evaluation, apple cultivars are propagated by clonal grafting onto rootstocks. Apple trees grown without rootstocks tend to be larger and much slower to fruit after planting. Rootstocks are used to control the speed of growth and the size of the resulting tree, allowing for easier harvesting.
+
    There are more than 7,500 cultivars of apples. Different cultivars are bred for various tastes and uses, including cooking, eating raw, and cider or apple juice production. Trees and fruit are prone to fungal, bacterial, and pest problems, which can be controlled by a number of organic and non-organic means. In 2010, the fruit's genome was sequenced as part of research on disease control and selective breeding in apple production.
+
    
+
+
    Etymology
    
+
    The word apple, whose Old English ancestor is æppel, is descended from the Proto-Germanic noun *aplaz, descended in turn from Proto-Indo-European *h₂ébōl.[3] As late as the 17th century, the word also functioned as a generic term for all fruit, including nuts. This can be compared to the 14th-century Middle English expression appel of paradis, meaning a banana.[4]
+
    
+
    Description
    
+
    The apple is a deciduous tree, generally standing 2 to 4.5 metres (6 to 15 feet) tall in cultivation and up to 15 m (49 ft) in the wild, though more typically 2 to 10 m (6.5 to 33 ft).[5][1] When cultivated, the size, shape and branch density are determined by rootstock selection and trimming method.[5] Apple trees may naturally have a rounded to erect crown with a dense canopy of leaves.[6] The bark of the trunk is dark gray or gray-brown, but young branches are reddish or dark-brown with a smooth texture.[1][7] When young twigs are covered in very fine downy hairs and become hairless as they become older.[7]
+
    The buds are egg-shaped and dark red or purple in color; they range in size from 3 to 5 millimeters, but are usually less than 4 mm. The bud scales have very hairy edges. When emerging from the buds, the leaves are convolute, meaning that their edges overlap each other.[1] Leaves can be  simple ovals (elliptic), medium or wide in width, somewhat egg-shaped with the wider portion toward their base (ovate), or even with sides that are more parallel to each other instead of curved (oblong) with a narrow pointed end.[7][1] The edges have broadly-angled teeth, but do not have lobes. The top surface of the leaves are glabrescent, almost hairless, while the undersides are densely covered in fine hairs.[1] The leaves are attached alternately by short leaf stems 1-to-3.5 cm (1⁄2-to-1+1⁄2 in) long.[6][1]
+
    Blossoms are produced in spring simultaneously with the budding of the leaves and are produced on spurs and some long shoots.[5] When the flower buds first begin to open the petals are rose-pink and fade to white or light pink when fully open with each flower 3-to-4-centimeter (1-to-1+1⁄2-inch) in diameter.[1] The five-petaled flowers are group in an inflorescence consisting of a cyme with 3–7 flowers.[8] The central flower of the inflorescence is called the "king bloom"; it opens first and can develop a larger fruit.[6] Open apple blossoms are damaged by even brief exposures to temperatures −2 °C (28 °F) or less, although the overwintering wood and buds are hardy down to −40 °C (−40 °F).[8]
+
    
+
      
+		
    • 
+			
      Apple blossoms
      
+			
      Apple blossoms
      
+		
      • 
+		
    • 
+			
      Botanical illustration
      
+			
      Botanical illustration
      
+		
      • 
+
    
+
    Fruit
    
+
    The fruit is a pome that matures in late summer or autumn.[1] The true fruits or carpels are the harder interior chambers inside the apple's core. There are usually five carpels inside an apple, but there may be as few as three. Each of the chambers contains one or two seeds.[9] The edible flesh is formed from the receptacle at the base of the flower.[10]
+
    
+
      
+		
    • 
+			
      How apple fruit derives from flower structures
      
+			
      How apple fruit derives from flower structures
      
+		
      • 
+
    
+
    The seeds are egg- to pear-shaped and may be colored from light brown or tan to a very dark brown, often with red shades or even purplish-black. They may have a blunt or sharp point.[11] The five sepals remain attached and stand out from the surface of the apple.[1]
+
    The size of the fruit varies widely between cultivars, but generally has a diameter between 2.5 and 12 cm (1 and 5 in).[7] The shape is quite variable and may be nearly round, elongated, conical, or short and wide.[12]
+
    The groundcolor of ripe apples is yellow, green, yellow-green or whitish yellow. The overcolor of ripe apples can be orange-red, pink-red, red, purple-red or brown-red.  The overcolor amount can be 0–100%.[13] The skin may be wholly or partly russeted, making it rough and brown. The skin is covered in a protective layer of epicuticular wax.[14] The skin may also be marked with scattered dots.[1] The flesh is generally pale yellowish-white, though it can be pink, yellow or green.[13]
+
    
+
      
+	
    • Apples can have any amount of overcolor, a darker tint over a pale groundcolor.
      • 
+		
    • 
+			
      0% overcolor
      
+			
      0% overcolor
      
+		
      • 
+		
    • 
+			
      100% overcolor
      
+			
      100% overcolor
      
+		
      • 
+
    
+
    Chemistry
    
+
    Important volatile compounds in apples that contribute to their scent and flavour include acetaldehyde, ethyl acetate, 1-butanal, ethanol, 2-methylbutanal, 3-methylbutanal, ethyl propionate, ethyl 2-methylpropionate, ethyl butyrate, ethyl 2-methyl butyrate, hexanal, 1-butanol, 3-methylbutyl acetate, 2-methylbutyl acetate, 1-propyl butyrate, ethyl pentanoate, amyl acetate, 2-methyl-1-butanol, trans-2-hexenal, ethyl hexanoate, hexanol.[15][16]
+
    
+
    Taxonomy
    
+
    The apple as a species has more than 100 alternative scientific names, or synonyms.[17] In modern times, Malus pumila and Malus domestica are the two main names in use. M. pumila is the older name, but M. domestica has become much more commonly used starting in the 21st century, especially in the western world. Two proposals were made to make M. domestica a conserved name: the earlier proposal was voted down by the Committee for Vascular Plants of the IAPT in 2014, but in April 2017 the Committee decided, with a narrow majority, that the newly popular name should be conserved.[18] The General Committee of the IAPT decided in June 2017 to approve this change, officially conserving M. domestica.[19] Nevertheless, some works published after 2017 still use M. pumila as the correct name, under an alternate taxonomy.[2]
+
    When first classified by Linnaeus in 1753, the pears, apples, and quinces were combined into one genus that he named Pyrus and he named the apple as Pyrus malus. This was widely accepted, however the botanist Philip Miller published an alternate classification in The Gardeners Dictionary with the apple species separated from Pyrus in 1754. He did not clearly indicate that by Malus pumila he meant the domesticated apple. Nonetheless, it was used as such by many botanists. When Moritz Balthasar Borkhausen published his scientific description of the apple in 1803 it may have been a new combination of P. malus var. domestica, but this was not directly referenced by Borkhausen.[17] The earliest use of var. domestica for the apple was by Georg Adolf Suckow in 1786.[2]
+
    
+
    Genome
    
+
    Further information: Apple genome
    
+
    Apples are diploid, with two sets of chromosomes per cell (though triploid cultivars, with three sets, are not uncommon), have 17 chromosomes and an estimated genome size of approximately 650 Mb. Several whole genome sequences have been completed and made available. The first one in 2010 was based on the diploid cultivar 'Golden Delicious'.[20] However, this first whole genome sequence contained several errors,[21] in part owing to the high degree of heterozygosity in diploid apples which, in combination with an ancient genome duplication, complicated the assembly. Recently, double- and trihaploid individuals have been sequenced, yielding whole genome sequences of higher quality.[22][23]
+
    The first whole genome assembly was estimated to contain around 57,000 genes,[20] though the more recent genome sequences support estimates between 42,000 and 44,700 protein-coding genes.[22][23] The availability of whole genome sequences has provided evidence that the wild ancestor of the cultivated apple most likely is Malus sieversii. Re-sequencing of multiple accessions has supported this, while also suggesting extensive introgression from Malus sylvestris following domestication.[24]
+
    
+
    Cultivation
    
+
    History
    
+
    Map of the origins of the cultivated apple. The wild origin is in Kazakhstan; hybridisations and repeated domestications followed, modifying many attributes of the fruit.[24]
    
+
    color photograph of a hand holding a red apple
    Wild Malus sieversii apple in Kazakhstan
    
+
    Central Asia is generally considered the center of origin for apples due to the genetic variability in specimens there.[25] The wild ancestor of Malus domestica was Malus sieversii, found growing wild in the mountains of Central Asia in southern Kazakhstan, Kyrgyzstan, Tajikistan, and northwestern China.[5][26] Cultivation of the species, most likely beginning on the forested flanks of the Tian Shan mountains, progressed over a long period of time and permitted secondary introgression of genes from other species into the open-pollinated seeds. Significant exchange with Malus sylvestris, the crabapple, resulted in populations of apples being more related to crabapples than to the more morphologically similar progenitor Malus sieversii. In strains without recent admixture the contribution of the latter predominates.[27][28][29]
+
    The apple is thought to have been domesticated 4,000–10,000 years ago in the Tian Shan mountains, and then to have travelled along the Silk Road to Europe, with hybridization and introgression of wild crabapples from Siberia (M. baccata), the Caucasus (M. orientalis), and Europe (M. sylvestris). Only the M. sieversii trees growing on the western side of the Tian Shan mountains contributed genetically to the domesticated apple, not the isolated population on the eastern side.[24]
+
    Chinese soft apples, such as M. asiatica and M. prunifolia, have been cultivated as dessert apples for more than 2,000 years in China. These are thought to be hybrids between M. baccata and M. sieversii in Kazakhstan.[24]
+
    Among the traits selected for by human growers are size, fruit acidity, color, firmness, and soluble sugar. Unusually for domesticated fruits, the wild M. sieversii origin is only slightly smaller than the modern domesticated apple.[24]
+
    At the Sammardenchia-Cueis site near Udine in Northeastern Italy, seeds from some form of apples have been found in material carbon dated to between 6570 and 5684 BCE.[30] Genetic analysis has not yet been successfully used to determine whether such ancient apples were wild Malus sylvestris or Malus domesticus containing Malus sieversii ancestry. It is hard to distinguish in the archeological record between foraged wild apples and apple plantations.[31]
+
    There is indirect evidence of apple cultivation in the third millennium BCE in the Middle East.[31] There is direct evidence, apple cores, dated to the 10th century BCE from a Judean site between the Sinai and Negev.
+[32] There was substantial apple production in European classical antiquity, and grafting was certainly known then.[31] Grafting is an essential part of modern domesticated apple production, to be able to propagate the best cultivars; it is unclear when apple tree grafting was invented.[31]
+
    
+
    
+
    
+
    
+
    
+
    "Wild Apples"
    by Henry David Thoreau
    Read by Kevin S for LibriVox
    
+
    Duration: 1 hour, 1 minute and 36 seconds.
    
+
    Audio 01:01:35 (full text)
    
+
    Problems playing this file? See media help.
    
+
    
+
    The Roman writer Pliny the Elder describes a method of storage for apples from his time in the 1st century. He says they should be placed in a room with good air circulation from a north facing window on a bed of straw, chaff, or mats with windfalls kept separately.[33] Though methods like this will extend the availabity of reasonably fresh apples, without refrigeration their lifespan is limited. Even sturdy winter apple varieties will only keep well until December in cool climates.[34] For longer storage medieval Europeans strung up cored and peeled apples to dry, either whole or sliced into rings.[35]
+
    Of the many Old World plants that the Spanish introduced to Chiloé Archipelago in the 16th century, apple trees became particularly well adapted.[36] Apples were introduced to North America by colonists in the 17th century,[5] and the first named apple cultivar was introduced in Boston by Reverend William Blaxton in 1640.[37] The only apples native to North America are crab apples.[38]
+
    Apple cultivars brought as seed from Europe were spread along Native American trade routes, as well as being cultivated on colonial farms. An 1845 United States apples nursery catalogue sold 350 of the "best" cultivars, showing the proliferation of new North American cultivars by the early 19th century.[38] In the 20th century, irrigation projects in Eastern Washington began and allowed the development of the multibillion-dollar fruit industry, of which the apple is the leading product.[5]
+
    Until the 20th century, farmers stored apples in frostproof cellars during the winter for their own use or for sale. Improved transportation of fresh apples by train and road replaced the necessity for storage.[39][40] Controlled atmosphere facilities are used to keep apples fresh year-round. Controlled atmosphere facilities use high humidity, low oxygen, and controlled carbon dioxide levels to maintain fruit freshness. They were first researched at Cambridge University in the 1920s and first used in the United States in the 1950s.[41]
+
    
+
    Breeding
    
+
    See also: Fruit tree propagation and Malling series
    
+
    An apple tree in Germany
    
+
    Many apples grow readily from seeds. However, apples must be propagated asexually to obtain cuttings with the characteristics of the parent. This is because seedling apples are "extreme heterozygotes". Rather than resembling their parents, seedlings are all different from each other and from their parents.[42] Triploid cultivars have an additional reproductive barrier in that three sets of chromosomes cannot be divided evenly during meiosis, yielding unequal segregation of the chromosomes (aneuploids). Even in the case when a triploid plant can produce a seed (apples are an example), it occurs infrequently, and seedlings rarely survive.[43]
+
    Because apples are not true breeders when planted as seeds, propagation usually involves grafting of cuttings. The rootstock used for the bottom of the graft can be selected to produce trees of a large variety of sizes, as well as changing the winter hardiness, insect and disease resistance, and soil preference of the resulting tree. Dwarf rootstocks can be used to produce very small trees (less than 3.0 m or 10 ft high at maturity), which bear fruit many years earlier in their life cycle than full size trees, and are easier to harvest.[44]
+
    Dwarf rootstocks for apple trees can be traced as far back as 300 BCE, to the area of Persia and Asia Minor. Alexander the Great sent samples of dwarf apple trees to Aristotle's Lyceum. Dwarf rootstocks became common by the 15th century and later went through several cycles of popularity and decline throughout the world.[45] The majority of the rootstocks used to control size in apples were developed in England in the early 1900s. The East Malling Research Station conducted extensive research into rootstocks, and their rootstocks are given an "M" prefix to designate their origin. Rootstocks marked with an "MM" prefix are Malling-series cultivars later crossed with trees of 'Northern Spy' in Merton, England.[46]
+
    Most new apple cultivars originate as seedlings, which either arise by chance or are bred by deliberately crossing cultivars with promising characteristics.[47] The words "seedling", "pippin", and "kernel" in the name of an apple cultivar suggest that it originated as a seedling. Apples can also form bud sports (mutations on a single branch). Some bud sports turn out to be improved strains of the parent cultivar. Some differ sufficiently from the parent tree to be considered new cultivars.[48]
+
    Apples have been acclimatized in Ecuador at very high altitudes, where they can often, with the needed factors, provide crops twice per year because of constant temperate conditions year-round.[49]
+
    
+
    Pollination
    
+
    See also: Fruit tree pollination
    
+
    Apple blossom from an old Ayrshire cultivar
    
+
    An orchard mason bee on an apple bloom in British Columbia, Canada
    
+
    Apples are self-incompatible; they must cross-pollinate to develop fruit. During the flowering each season, apple growers often utilize pollinators to carry pollen. Honey bees are most commonly used. Orchard mason bees are also used as supplemental pollinators in commercial orchards. Bumblebee queens are sometimes present in orchards, but not usually in sufficient number to be significant pollinators.[48][50]
+
    Cultivars are sometimes classified by the day of peak bloom in the average 30-day blossom period, with pollinizers selected from cultivars within a 6-day overlap period. There are four to seven pollination groups in apples, depending on climate:
+
    
+
    • Group A – Early flowering, 1 to 3 May in England ('Gravenstein', 'Red Astrachan')
      • 
+
    • Group B – 4 to 7 May ('Idared', 'McIntosh')
      • 
+
    • Group C – Mid-season flowering, 8 to 11 May ('Granny Smith', 'Cox's Orange Pippin')
      • 
+
    • Group D – Mid/late season flowering, 12 to 15 May ('Golden Delicious', 'Calville blanc d'hiver')
      • 
+
    • Group E – Late flowering, 16 to 18 May ('Braeburn', 'Reinette d'Orléans')
      • 
+
    • Group F – 19 to 23 May ('Suntan')
      • 
+
    • Group H – 24 to 28 May ('Court-Pendu Gris' – also called Court-Pendu plat)
    
+
    One cultivar can be pollinated by a compatible cultivar from the same group or close (A with A, or A with B, but not A with C or D).[51]
+
    
+
    Maturation and harvest
    
+
    See also: Fruit picking and Fruit tree pruning
    
+
    L. K. Relander, the former President of Finland, with his family picking apples in the 1930s
    
+
    Cultivars vary in their yield and the ultimate size of the tree, even when grown on the same rootstock. Some cultivars, if left unpruned, grow very large—letting them bear more fruit, but making harvesting more difficult. Depending on tree density (number of trees planted per unit surface area), mature trees typically bear 40–200 kg (90–440 lb) of apples each year, though productivity can be close to zero in poor years. Apples are harvested using three-point ladders that are designed to fit amongst the branches. Trees grafted on dwarfing rootstocks bear about 10–80 kg (20–180 lb) of fruit per year.[48]
+
    Some farms with apple orchards open them to the public so consumers can pick their own apples.[52]
+
    Crops ripen at different times of the year according to the cultivar. Cultivar that yield their crop in the summer include 'Sweet Bough' and 'Duchess'; fall producers include 'Blenheim'; winter producers include 'King', 'Swayzie', and 'Tolman Sweet'.[38]
+
    
+
    Storage
    
+
    Different apple cultivars in a wholesale food market
    
+
    Commercially, apples can be stored for months in controlled atmosphere chambers. Apples are commonly stored in chambers with lowered concentrations of oxygen to reduce respiration and slow softening and other changes if the fruit is already fully ripe. The gas ethylene is used by plants as a hormone which promotes ripening, decreasing the time an apple can be stored. For storage longer than about six months the apples are picked earlier, before full ripeness, when ethylene production by the fruit is low. However, in many varieties this increases their sensitivity to carbon dioxide, which also must be controlled.[53]
+
    For home storage, most culitvars of apple can be stored for three weeks in a pantry and four to six weeks from the date of purchase in a refrigerator that maintains 4 to 0 °C (39 to 32 °F).[54][55] Some varieties of apples (e.g. 'Granny Smith' and 'Fuji') have more than three times the storage life of others.[56]
+
    Non-organic apples may be sprayed with a substance 1-methylcyclopropene blocking the apples' ethylene receptors, temporarily preventing them from ripening.[57]
+
    
+
    Pests and diseases
    
+
    Further information: List of apple diseases
    
+
    Codling moth larva tunnelling inside an apple
    
+
    Apple trees are susceptible to fungal and bacterial diseases, and to damage by insect pests. Many commercial orchards pursue a program of chemical sprays to maintain high fruit quality, tree health, and high yields. These prohibit the use of synthetic pesticides, though some older pesticides are allowed. Organic methods include, for instance, introducing its natural predator to reduce the population of a particular pest.
+
    A wide range of pests and diseases can affect the plant. Three of the more common diseases or pests are mildew, aphids, and apple scab.
+
    
+
    • Mildew is characterized by light grey powdery patches appearing on the leaves, shoots and flowers, normally in spring. The flowers turn a creamy yellow color and do not develop correctly. This can be treated similarly to Botrytis—eliminating the conditions that caused the disease and burning the infected plants are among recommended actions.[58]
      • 
+
    • Aphids are small insects with sucking mouthparts. Five species of aphids commonly attack apples: apple grain aphid, rosy apple aphid, apple aphid, spirea aphid, and the woolly apple aphid. The aphid species can be identified by color, time of year, and by differences in the cornicles (small paired projections from their rear).[59] Aphids feed on foliage using needle-like mouth parts to suck out plant juices. When present in high numbers, certain species reduce tree growth and vigor.[60]
      • 
+
    • Apple scab: Apple scab causes leaves to develop olive-brown spots with a velvety texture that later turn brown and become cork-like in texture. The disease also affects the fruit, which also develops similar brown spots with velvety or cork-like textures. Apple scab is spread through fungus growing in old apple leaves on the ground and spreads during warm spring weather to infect the new year's growth.[61]
    
+
    Among the most serious disease problems is a bacterial disease called fireblight, and three fungal diseases: Gymnosporangium rust, black spot,[62] and bitter rot.[63] Codling moths, and the apple maggots of fruit flies, cause serious damage to apple fruits, making them unsaleable. Young apple trees are also prone to mammal pests like mice and deer, which feed on the soft bark of the trees, especially in winter.[61] The larvae of the apple clearwing moth (red-belted clearwing) burrow through the bark and into the phloem of apple trees, potentially causing significant damage.[64]
+
    
+
    Cultivars
    
+
    Main article: List of apple cultivars
    
+
    An assortment of apple cultivars
    
+
    There are more than 7,500 known cultivars (cultivated varieties) of apples.[65] Cultivars vary in their yield and the ultimate size of the tree, even when grown on the same rootstock.[66] Different cultivars are available for temperate and subtropical climates. The UK's National Fruit Collection, which is the responsibility of the Department of Environment, Food, and Rural Affairs, includes a collection of over 2,000 cultivars of apple tree in Kent.[67] The University of Reading, which is responsible for developing the UK national collection database, provides access to search the national collection. The University of Reading's work is part of the European Cooperative Programme for Plant Genetic Resources of which there are 38 countries participating in the Malus/Pyrus work group.[68]
+
    The UK's national fruit collection database contains much information on the characteristics and origin of many apples, including alternative names for what is essentially the same "genetic" apple cultivar. Most of these cultivars are bred for eating fresh (dessert apples), though some are cultivated specifically for cooking (cooking apples) or producing cider. Cider apples are typically too tart and astringent to eat fresh, but they give the beverage a rich flavor that dessert apples cannot.[69]
+
    In the United States there are many apple breeding programs associated with universities. Cornell University has had a program operating since 1880 in Geneva, New York. Among their recent well known apples is the 'SnapDragon' cultivar released in 2013. In the west Washington State University started a program to support their apple industry in 1994 and released the 'Cosmic Crisp' cultivar in 2017. The third most grown apple cultivar in the United States is the 'Honeycrisp', released by the University of Minnesota program in 1991.[70] Unusually for a popular cultivar, the 'Honeycrisp' is not directly related to another popular apple cultivar but instead to two unsuccessful cultivars.[71] In Europe there are also many breeding programs such as the Julius Kühn-Institut, the German federal research center for cultivated plants.[72]
+
    Commercially popular apple cultivars are soft but crisp. Other desirable qualities in modern commercial apple breeding are a colorful skin, absence of russeting, ease of shipping, lengthy storage ability, high yields, disease resistance, common apple shape, and developed flavor.[66] Modern apples are generally sweeter than older cultivars, as popular tastes in apples have varied over time. Most North Americans and Europeans favor sweet, subacid apples, but tart apples have a strong minority following.[73] Extremely sweet apples with barely any acid flavor are popular in Asia,[73] especially the Indian subcontinent.[69]
+
    
+
    Less common apple cultivars from an orchard in Italy
    
+
    Old cultivars are often oddly shaped, russeted, and grow in a variety of textures and colors. Some find them to have better flavor than modern cultivars, but they may have other problems that make them commercially unviable—low yield, disease susceptibility, poor tolerance for storage or transport, or just being the "wrong" size.[74] A few old cultivars are still produced on a large scale, but many have been preserved by home gardeners and farmers that sell directly to local markets. Many unusual and locally important cultivars with their own unique taste and appearance exist; apple conservation campaigns have sprung up around the world to preserve such local cultivars from extinction. In the United Kingdom, old cultivars such as 'Cox's Orange Pippin' and 'Egremont Russet' are still commercially important even though by modern standards they are low yielding and susceptible to disease.[5]
+
    
+
    Production
    
+
    Main article: List of countries by apple production
    
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    Apple production
    
+
    2022, millions of tonnes
    
+
    
+
     China47.6
+
     United States4.8
+
     Turkey4.4
+
     Poland4.3
+
     India2.6
+
    World95.8
+
    Source: FAOSTAT of the United Nations[75]
+
    
+
    World production of apples in 2022 was 96 million tonnes, with China producing 50% of the total (table).[75] Secondary producers were the United States, Turkey, and Poland.[75]
+
    
+
    Toxicity
    
+
    Amygdalin
    
+
    Apple seeds contain small amounts of amygdalin, a sugar and cyanide compound known as a cyanogenic glycoside. Ingesting small amounts of apple seeds causes no ill effects, but consumption of extremely large doses can cause adverse reactions. It may take several hours before the poison takes effect, as cyanogenic glycosides must be hydrolyzed before the cyanide ion is released.[76] The U.S. National Library of Medicine's Hazardous Substances Data Bank records no cases of amygdalin poisoning from consuming apple seeds.[77]
+
    
+
    Allergy
    
+
    One form of apple allergy, often found in northern Europe, is called birch-apple syndrome and is found in people who are also allergic to birch pollen.[78] Allergic reactions are triggered by a protein in apples that is similar to birch pollen, and people affected by this protein can also develop allergies to other fruits, nuts, and vegetables. Reactions, which entail oral allergy syndrome (OAS), generally involve itching and inflammation of the mouth and throat,[78] but in rare cases can also include life-threatening anaphylaxis.[79] This reaction only occurs when raw fruit is consumed—the allergen is neutralized in the cooking process. The variety of apple, maturity and storage conditions can change the amount of allergen present in individual fruits. Long storage times can increase the amount of proteins that cause birch-apple syndrome.[78]
+
    In other areas, such as the Mediterranean, some individuals have adverse reactions to apples because of their similarity to peaches.[78] This form of apple allergy also includes OAS, but often has more severe symptoms, such as vomiting, abdominal pain and urticaria, and can be life-threatening. Individuals with this form of allergy can also develop reactions to other fruits and nuts. Cooking does not break down the protein causing this particular reaction, so affected individuals cannot eat raw or cooked apples. Freshly harvested, over-ripe fruits tend to have the highest levels of the protein that causes this reaction.[78]
+
    Breeding efforts have yet to produce a hypoallergenic fruit suitable for either of the two forms of apple allergy.[78]
+
    
+
    Uses
    
+
    See also: Cooking apple and Cider apple
    
+
    Nutrition
    
+
    
+
+
    Apples, with skin (edible parts)
    Nutritional value per 100 g (3.5 oz)
    Energy218 kJ (52 kcal)
    13.81 g
    Sugars10.39
    Dietary fiber2.4 g
    
+
    0.17 g
    
+
    0.26 g
    
+
+
+
+
+
    Vitamins and minerals
    
+
    VitaminsQuantity 
    %DV†
    Vitamin A equiv.
    0%
     3 μg
    0%
    27 μg
    29 μg
    Thiamine (B1)
    1%
     0.017 mg
    Riboflavin (B2)
    2%
     0.026 mg
    Niacin (B3)
    1%
     0.091 mg
    Pantothenic acid (B5)
    1%
     0.061 mg
    Vitamin B6
    2%
     0.041 mg
    Folate (B9)
    1%
     3 μg
    Vitamin C
    5%
     4.6 mg
    Vitamin E
    1%
     0.18 mg
    Vitamin K
    2%
     2.2 μg
    
+
    MineralsQuantity 
    %DV†
    Calcium
    0%
     6 mg
    Iron
    1%
     0.12 mg
    Magnesium
    1%
     5 mg
    Manganese
    2%
     0.035 mg
    Phosphorus
    1%
     11 mg
    Potassium
    4%
     107 mg
    Sodium
    0%
     1 mg
    Zinc
    0%
     0.04 mg
    
+
    Other constituentsQuantity
    Water85.56 g
    
+
    Link to Full Nutrient Report of USDA Database entry
    †Percentages estimated using US recommendations for adults,[80] except for potassium, which is estimated based on expert recommendation from the National Academies.[81]
    
+
    
+
    A raw apple is 86% water and 14% carbohydrates, with negligible content of fat and protein (table). A reference serving of a raw apple with skin weighing 100 g (3.5 oz) provides 52 calories and a moderate content of dietary fiber (table). Otherwise, there is low content of micronutrients, with the Daily Values of all falling below 10% (table).
+
    
+
    Culinary
    
+
    Further information: List of apple dishes
    
+
    Machine for paring, coring, and slicing apples, from Henry B. Scammell's 1897 handbook Cyclopedia of Valuable Receipts
    
+
    Apples varieties can be grouped as cooking apples, eating apples, and cider apples, the last so astringent as to be "almost inedible".[82] Apples are consumed as juice, raw in salads, baked in pies, cooked into sauces and apple butter, or baked.[83] They are sometimes used as an ingredient in savory foods, such as sausage and stuffing.[84]
+
    Several techniques are used to preserve apples and apple products. Traditional methods include drying and making apple butter.[82] Juice and cider are produced commercially; cider is a significant industry in regions such as the West of England and Normandy.[82]
+
    A toffee apple (UK) or caramel apple (US) is a confection made by coating an apple in hot toffee or caramel candy respectively and allowing it to cool.[85][8] Apples and honey are a ritual food pairing eaten during the Jewish New Year of Rosh Hashanah.[86]
+
    Apples are an important ingredient in many desserts, such as pies, crumbles, and cakes. When cooked, some apple cultivars easily form a puree known as apple sauce, which can be cooked down to form a preserve, apple butter. They are often baked or stewed, and are cooked in some meat dishes.[82]
+
    
+
    
+
    
+
    
+
    Wikibooks Cookbook has a recipe/module on
+
    
+
+
    
+
    
+
    Apples are milled or pressed to produce apple juice, which may be drunk unfiltered (called apple cider in North America), or filtered. Filtered juice is often concentrated and frozen, then reconstituted later and consumed. Apple juice can be fermented to make cider (called hard cider in North America), ciderkin, and vinegar.[8] Through distillation, various alcoholic beverages can be produced, such as applejack, Calvados, and apple brandy.[8][87]
+
    
+
    Organic production
    
+
    Organic apples are commonly produced in the United States.[88] Due to infestations by key insects and diseases, organic production is difficult in Europe.[89] The use of pesticides containing chemicals, such as sulfur, copper, microorganisms, viruses, clay powders, or plant extracts (pyrethrum, neem) has been approved by the EU Organic Standing Committee to improve organic yield and quality.[89] A light coating of kaolin, which forms a physical barrier to some pests, also may help prevent apple sun scalding.[48]
+
    
+
    Non-browning apples
    
+
    Apple skins and seeds contain polyphenols.[90] These are oxidised by the enzyme polyphenol oxidase, which causes browning in sliced or bruised apples, by catalyzing the oxidation of phenolic compounds to o-quinones, a browning factor.[91] Browning reduces apple taste, color, and food value. Arctic apples, a non-browning group of apples introduced to the United States market in 2019, have been genetically modified to silence the expression of polyphenol oxidase, thereby delaying a browning effect and improving apple eating quality.[92][93] The US Food and Drug Administration in 2015, and Canadian Food Inspection Agency in 2017, determined that Arctic apples are as safe and nutritious as conventional apples.[94][95]
+
    
+
    Other products
    
+
    Apple seed oil is obtained by pressing apple seeds for manufacturing cosmetics.[96]
+
    
+
    In culture
    
+
    Main article: Apple (symbolism)
    
+
    Germanic paganism
    
+
    Illustration of girl in a red dress, holding 3 candles in one hand and a basket of apples in the other
    "Brita as Iduna" (1901) by Carl Larsson
    
+
    In Norse mythology, the goddess Iðunn is portrayed in the Prose Edda (written in the 13th century by Snorri Sturluson) as providing apples to the gods that give them eternal youthfulness. The English scholar H. R. Ellis Davidson links apples to religious practices in Germanic paganism, from which Norse paganism developed. She points out that buckets of apples were found in the Oseberg ship burial site in Norway, that fruit and nuts (Iðunn having been described as being transformed into a nut in Skáldskaparmál) have been found in the early graves of the Germanic peoples in England and elsewhere on the continent of Europe, which may have had a symbolic meaning, and that nuts are still a recognized symbol of fertility in southwest England.[97]
+
    Davidson notes a connection between apples and the Vanir, a tribe of gods associated with fertility in Norse mythology, citing an instance of eleven "golden apples" being given to woo the beautiful Gerðr by Skírnir, who was acting as messenger for the major Vanir god Freyr in stanzas 19 and 20 of Skírnismál. Davidson also notes a further connection between fertility and apples in Norse mythology in chapter 2 of the Völsunga saga: when the major goddess Frigg sends King Rerir an apple after he prays to Odin for a child, Frigg's messenger (in the guise of a crow) drops the apple in his lap as he sits atop a mound.[97] Rerir's wife's consumption of the apple results in a six-year pregnancy and the birth (by Caesarean section) of their son—the hero Völsung.[98]
+
    Further, Davidson points out the "strange" phrase "Apples of Hel" used in an 11th-century poem by the skald Thorbiorn Brúnarson. She states this may imply that the apple was thought of by Brúnarson as the food of the dead. Further, Davidson notes that the potentially Germanic goddess Nehalennia is sometimes depicted with apples and that parallels exist in early Irish stories. Davidson asserts that while cultivation of the apple in Northern Europe extends back to at least the time of the Roman Empire and came to Europe from the Near East, the native varieties of apple trees growing in Northern Europe are small and bitter. Davidson concludes that in the figure of Iðunn "we must have a dim reflection of an old symbol: that of the guardian goddess of the life-giving fruit of the other world."[97]
+
    
+
    Greek mythology
    
+
    Heracles with the apple of Hesperides 
    
+
    Apples appear in many religious traditions, including Greek and Roman mythology where it has an ambiguous symbolism of discord, fertility, or courtship.[99] In Greek mythology, the Greek hero Heracles, as a part of his Twelve Labours, was required to travel to the Garden of the Hesperides and pick the golden apples off the Tree of Life growing at its center.[100]
+
    The Greek goddess of discord, Eris, became disgruntled after she was excluded from the wedding of Peleus and Thetis.[101] In retaliation, she tossed a golden apple inscribed Καλλίστη (Kallistē, "For the most beautiful one"), into the wedding party. Three goddesses claimed the apple: Hera, Athena, and Aphrodite. Paris of Troy was appointed to select the recipient. After being bribed by both Hera and Athena, Aphrodite tempted him with the most beautiful woman in the world, Helen of Sparta. He awarded the apple to Aphrodite, thus indirectly causing the Trojan War.[102][103]
+
    The apple was thus considered, in ancient Greece, sacred to Aphrodite. To throw an apple at someone was to symbolically declare one's love; and similarly, to catch it was to symbolically show one's acceptance of that love. An epigram claiming authorship by Plato states:[104]
+
    
+
    I throw the apple at you, and if you are willing to love me, take it and share your girlhood with me; but if your thoughts are what I pray they are not, even then take it, and consider how short-lived is beauty.
    — Plato, Epigram VII
    
+
    Atalanta, also of Greek mythology, raced all her suitors in an attempt to avoid marriage. She outran all but Hippomenes (also known as Melanion, a name possibly derived from melon, the Greek word for both "apple" and fruit in general),[100] who defeated her by cunning, not speed. Hippomenes knew that he could not win in a fair race, so he used three golden apples (gifts of Aphrodite, the goddess of love) to distract Atalanta. It took all three apples and all of his speed, but Hippomenes was finally successful, winning the race and Atalanta's hand.[105][106]
+
    
+
    Celtic mythology
    
+
    In Celtic mythology, the otherworld has many names, including Emain Ablach, "Emain of the Apple-trees". A version of this is Avalon in Arthurian legend, or in Welsh Ynys Afallon, "Island of Apples".[107]
+
    
+
    China
    
+
    Píngānguǒ ("Peace apples") on sale in Beijing for Christmas Eve (2017)
    
+
    In China, apples symbolise peace, since the sounds of the first element ("píng") in the words "apple" (苹果, Píngguǒ) and "peace" (平安, Píng'ān) are homophonous in Mandarin and Cantonese.[3][108] When these two words are combined, the word Píngānguǒ (平安果, "Peace apples") is formed. This association developed further as the name for Christmas Eve in Mandarin is Píngānyè (平安夜, "Peaceful/Quiet Evening"), which made the gifting of apples at this season to friends and associates popular, as a way to wish them peace and safety.[108]
+
    
+
    Christian art
    
+
    Adam and Eve by Albrecht Dürer (1507), showcasing the apple as a symbol of sin
    
+
    Though the forbidden fruit of Eden in the Book of Genesis is not identified, popular Christian tradition has held that it was an apple that Eve coaxed Adam to share with her.[109] The origin of the popular identification with a fruit unknown in the Middle East in biblical times is found in wordplay with the Latin words mālum (an apple) and mălum (an evil), each of which is normally written malum.[110] The tree of the forbidden fruit is called "the tree of the knowledge of good and evil" in Genesis 2:17,[111] and the Latin for "good and evil" is bonum et malum.[112]
+
    Renaissance painters may also have been influenced by the story of the golden apples in the Garden of Hesperides. As a result, in the story of Adam and Eve, the apple became a symbol for knowledge, immortality, temptation, the fall of man into sin, and sin itself. The larynx in the human throat has been called the "Adam's apple" because of a notion that it was caused by the forbidden fruit remaining in the throat of Adam. The apple as symbol of sexual seduction has been used to imply human sexuality, possibly in an ironic vein.[109]
+
    
+
    Proverb
    
+
    The proverb, "An apple a day keeps the doctor away", addressing the supposed health benefits of the fruit, has been traced to 19th-century Wales, where the original phrase was "Eat an apple on going to bed, and you'll keep the doctor from earning his bread".[113] In the 19th century and early 20th, the phrase evolved to "an apple a day, no doctor to pay" and "an apple a day sends the doctor away"; the phrasing now commonly used was first recorded in 1922.[114]
+
    
+
    See also
    
+
+
    References
    
+
    
+
      
+
    1. ^ Jump up to: a b c d e f g h i j k Dickson, Elizabeth E. (28 May 2021). "Malus domestica". Flora of North America. Archived from the original on 28 July 2024. Retrieved 27 July 2024.
+
      2. 
+
    2. ^ Jump up to: a b c "Malus domestica (Suckow) Borkh". Plants of the World Online. Royal Botanic Gardens, Kew. Retrieved 31 July 2024.
+
      3. 
+
    3. ^ Jump up to: a b Lim, Lisa (6 July 2021). "Where the word 'apple' came from and why the forbidden fruit was unlucky to be linked with the fall of man". Language Matters. South China Morning Post. Hong Kong, China: Alibaba Group. Archived from the original on 28 June 2023. Retrieved 28 June 2023.
+
      4. 
+
    4. ^ "Origin and meaning of "apple" by Online Etymology Dictionary". Online Etymology Dictionary. Archived from the original on 21 December 2019. Retrieved 22 November 2019.
+
      5. 
+
    5. ^ Jump up to: a b c d e f g Rieger, Mark. "Apple - Malus domestica". HORT 3020: Intro Fruit Crops. University of Georgia. Archived from the original on 21 January 2008. Retrieved 22 January 2008.
+
      6. 
+
    6. ^ Jump up to: a b c "Apples - Malus domestica". North Carolina Extension Gardener Plant Toolbox. North Carolina State University. Archived from the original on 31 May 2024. Retrieved 31 July 2024.
+
      7. 
+
    7. ^ Jump up to: a b c d Heil, Kenneth D.; O'Kane, Jr., Steve L.; Reeves, Linda Mary; Clifford, Arnold (2013). Flora of the Four Corners Region: Vascular Plants of the San Juan River Drainage, Arizona, Colorado, New Mexico, and Utah (First ed.). St. Louis, Missouri: Missouri Botanical Garden. p. 909. ISBN 978-1-930723-84-9. ISSN 0161-1542. LCCN 2012949654. OCLC 859541992. Retrieved 27 July 2024.
+
      8. 
+
    8. ^ Jump up to: a b c d e Lim, Tong Kwee (2012). "Malus x domestica". Edible Medicinal and Non-Medicinal Plants. Vol. 4, Fruit (First ed.). Dordrecht, the Netherlands: Springer. pp. 414–415. doi:10.1007/978-94-007-4053-2_49. ISBN 978-94-007-4053-2. OCLC 795503871.
+
      9. 
+
    9. ^ Juniper, Barrie E.; Mabberley, David J. (2006). The Story of the Apple (First ed.). Portland, Oregon: Timber Press. p. 27. ISBN 978-0-88192-784-9. LCCN 2006011869. OCLC 67383484. Retrieved 1 August 2024.
+
      10. 
+
    10. ^ "Fruit glossary". Royal Horticultural Society. Archived from the original on 7 August 2024. Retrieved 7 August 2024.
+
      11. 
+
    11. ^ Burford, Tom (2013). Apples of North America : 192 Exceptional Varieties for Gardeners, Growers and Cooks (First ed.). Portland, Oregon: Timber Press. pp. 22, 50, 55, 122, 123, 137, 141, 147, 159, 245, 246. ISBN 978-1-60469-249-5. LCCN 2012045130. OCLC 819860825.
+
      12. 
+
    12. ^ "Shape". Western Agricultural Research Center. Montana State University. Archived from the original on 23 April 2024. Retrieved 30 July 2024.
+
      13. 
+
    13. ^ Jump up to: a b Janick, Jules; Cummins, James N.; Brown, Susan K.; Hemmat, Minou (1996). "Chapter 1: Apples" (PDF). Fruit Breeding. Vol. I: Tree and Tropical Fruits. New York: John Wiley & Sons. pp. 9, 48. ISBN 978-0-471-31014-3. LCCN 95016407. OCLC 1302621533. Archived (PDF) from the original on 19 July 2013. Retrieved 30 August 2024.
+
      14. 
+
    14. ^ "Natural Waxes on Fruits". Postharvest.tfrec.wsu.edu. 29 October 2010. Archived from the original on 24 May 2013. Retrieved 14 June 2013.
+
      15. 
+
    15. ^ Flath, R. A.; Black, D. R.; Forrey, R. R.; McDonald, G. M.; Mon, T. R.; Teranishi, R. (1 August 1969). "Volatiles in Gravenstein Apple Essence Identified by GC-Mass Spectrometry". Journal of Chromatographic Science. 7 (8): 508. doi:10.1093/CHROMSCI/7.8.508.
+
      16. 
+
    16. ^ Flath, Robert A.; Black, Dale Robert.; Guadagni, Dante G.; McFadden, William H.; Schultz, Thomas H. (January 1967). "Identification and organoleptic evaluation of compounds in Delicious apple essence". Journal of Agricultural and Food Chemistry. 15 (1): 29. doi:10.1021/jf60149a032.
+
      17. 
+
    17. ^ Jump up to: a b Qian, Guan-Ze; Liu, Lian-Fen; Tang, Geng-Guo (April 2010). "(1933) Proposal to conserve the name Malus domestica against M. pumila, M. communis, M. frutescens, and Pyrus dioica ( Rosaceae )". Taxon. 59 (2): 650–652. doi:10.1002/tax.592038.
+
      18. 
+
    18. ^ Applequist, Wendy L. (2017). "Report of the Nomenclature Committee for Vascular Plants: 69" (PDF). Taxon. 66 (2): 500–513. doi:10.12705/662.17. Archived (PDF) from the original on 7 May 2024.
+
      19. 
+
    19. ^ Wilson, Karen L. (June 2017). "Report of the General Committee: 18". Taxon. 66 (3): 742. doi:10.12705/663.15.
+
      20. 
+
    20. ^ Jump up to: a b Velasco, Riccardo; Zharkikh, Andrey; Affourtit, Jason; Dhingra, Amit; Cestaro, Alessandro; et al. (2010). "The genome of the domesticated apple (Malus × domestica Borkh.)". Nature Genetics. 42 (10): 833–839. doi:10.1038/ng.654. PMID 20802477. S2CID 14854514.
+
      21. 
+
    21. ^ Di Pierro, Erica A.; Gianfranceschi, Luca; Di Guardo, Mario; Koehorst-Van Putten, Herma J.J.; Kruisselbrink, Johannes W.; et al. (2016). "A high-density, multi-parental SNP genetic map on apple validates a new mapping approach for outcrossing species". Horticulture Research. 3 (1): 16057. Bibcode:2016HorR....316057D. doi:10.1038/hortres.2016.57. PMC 5120355. PMID 27917289.
+
      22. 
+
    22. ^ Jump up to: a b Daccord, Nicolas; Celton, Jean-Marc; Linsmith, Gareth; et al. (2017). "High-quality de novo assembly of the apple genome and methylome dynamics of early fruit development". Nature Genetics. 49 (7). Nature Communications: 1099–1106. doi:10.1038/ng.3886. hdl:10449/42064. PMID 28581499. S2CID 24690391.
+
      23. 
+
    23. ^ Jump up to: a b Zhang, Liyi; Hu, Jiang; Han, Xiaolei; Li, Jingjing; Gao, Yuan; et al. (2019). "A high-quality apple genome assembly reveals the association of a retrotransposon and red fruit colour". Nature Communications. 10 (1). Nature Genetics: 1494. Bibcode:2019NatCo..10.1494Z. doi:10.1038/s41467-019-09518-x. PMC 6445120. PMID 30940818.
+
      24. 
+
    24. ^ Jump up to: a b c d e Duan, Naibin; Bai, Yang; Sun, Honghe; Wang, Nan; Ma, Yumin; et al. (2017). "Genome re-sequencing reveals the history of apple and supports a two-stage model for fruit enlargement". Nature Communications. 8 (1): 249. Bibcode:2017NatCo...8..249D. doi:10.1038/s41467-017-00336-7. PMC 5557836. PMID 28811498.
+
      25. 
+
    25. ^ Richards, Christopher M.; Volk, Gayle M.; Reilley, Ann A.; Henk, Adam D.; Lockwood, Dale R.; et al. (2009). "Genetic diversity and population structure in Malus sieversii, a wild progenitor species of domesticated apple". Tree Genetics & Genomes. 5 (2): 339–347. doi:10.1007/s11295-008-0190-9. S2CID 19847067.
+
      26. 
+
    26. ^ Lauri, Pierre-éric; Maguylo, Karen; Trottier, Catherine (March 2006). "Architecture and size relations: an essay on the apple (Malus × domestica, Rosaceae) tree". American Journal of Botany. 93 (3): 357–368. doi:10.3732/ajb.93.3.357. PMID 21646196. Archived from the original on 20 April 2019. Retrieved 27 July 2024.
+
      27. 
+
    27. ^ Cornille, Amandine; Gladieux, Pierre; Smulders, Marinus J. M.; Roldán-Ruiz, Isabel; Laurens, François; et al. (2012). Mauricio, Rodney (ed.). "New Insight into the History of Domesticated Apple: Secondary Contribution of the European Wild Apple to the Genome of Cultivated Varieties". PLOS Genetics. 8 (5): e1002703. doi:10.1371/journal.pgen.1002703. PMC 3349737. PMID 22589740.
+
      28. 
+
    28. ^ Kean, Sam (17 May 2012). "ScienceShot: The Secret History of the Domesticated Apple". Archived from the original on 11 June 2016.
+
      29. 
+
    29. ^ Coart, E.; Van Glabeke, S.; De Loose, M.; Larsen, A.S.; Roldán-Ruiz, I. (2006). "Chloroplast diversity in the genus Malus: new insights into the relationship between the European wild apple (Malus sylvestris (L.) Mill.) and the domesticated apple (Malus domestica Borkh.)". Mol. Ecol. 15 (8): 2171–2182. Bibcode:2006MolEc..15.2171C. doi:10.1111/j.1365-294x.2006.02924.x. PMID 16780433. S2CID 31481730.
+
      30. 
+
    30. ^ Rottoli, Mauro; Pessina, Andrea (2007). "Chapter 9: Neolithic agriculture in Italy: an update of archaeobotanical data with particular emphasis on northern settlements". In Colledge, Sue; Conolly, James (eds.). The Origins and Spread of Domestic Plants in Southwest Asia and Europe (First ed.). Walnut Creek, California: Left Coast Press; University College London Institute of Archaeology Publications. pp. 142–143. ISBN 978-1-59874-988-5. OCLC 84838157.
+
      31. 
+
    31. ^ Jump up to: a b c d Schlumbaum, Angela; van Glabeke, Sabine; Roldan-Ruiz, Isabel (January 2012). "Towards the onset of fruit tree growing north of the Alps: Ancient DNA from waterlogged apple (Malus sp.) seed fragments". Annals of Anatomy - Anatomischer Anzeiger. 194 (1): 157–162. doi:10.1016/j.aanat.2011.03.004. PMID 21501956.
+
      32. 
+
    32. ^ Sauer, Jonathan D. (1993). Historical Geography of Crop Plants: A Select Roster (First ed.). Boca Raton, Florida: CRC Press. pp. 109–113. ISBN 978-0-8493-8901-6. LCCN 92045590. OCLC 27224696.
+
      33. 
+
    33. ^ Plinius, Gaius Secundus (1855). The Natural History of Pliny. Vol. III. Translated by Bostock, John; Riley, Henry T. London: Henry G. Bohn. p. 303. Retrieved 3 August 2024.
+
      34. 
+
    34. ^ Martin, Alice A. (1976). All About Apples (First ed.). Boston, Massachusetts: Houghton Mifflin Company. pp. 64–65. ISBN 978-0-395-20724-6. OCLC 1733691. Retrieved 3 August 2024.
+
      35. 
+
    35. ^ Adamson, Melitta Weiss (2004). Food in Medieval Times (First ed.). Westport, Connecticut: Greenwood Press. pp. 19–20. ISBN 978-0-313-32147-4. LCCN 2004014054. OCLC 55738647.
+
      36. 
+
    36. ^ Torrejón, Fernando; Cisternas, Marco; Araneda, Alberto (2004). "Efectos ambientales de la colonización española desde el río Maullín al archipiélago de Chiloé, sur de Chile" [Environmental effects of the spanish colonization from de Maullín river to the Chiloé archipelago, southern Chile]. Revista Chilena de Historia Natural (in Spanish). 77 (4): 661–677. doi:10.4067/s0716-078x2004000400009.
+
      37. 
+
    37. ^ Smith, Archibald William (1963). A Gardener's Book of Plant Names : A Handbook of the Meaning and Origins of Plant Names (First ed.). New York: Harper & Row. p. 40. LCCN 62009906. OCLC 710612. Retrieved 10 August 2024.
+
      38. 
+
    38. ^ Jump up to: a b c Poole, Mike (1980). "Heirloom Apples". In Lawrence, James (ed.). The Harrowsmith Reader Volume II. Camden East, Ontario: Camden House Publishing. p. 122. ISBN 978-0-920656-11-2. OCLC 1336124440. Retrieved 10 August 2024.
+
      39. 
+
    39. ^ Van Valen, James M. (1900). History of Bergen County, New Jersey. New York: New Jersey Publishing and Engraving Company. pp. 33–34. OCLC 25697876. Retrieved 9 August 2024.
+
      40. 
+
    40. ^ Brox, Jane (1999). Five Thousand Days Like This One (First ed.). Boston, Massachusetts: Beacon Press. pp. 150–151. ISBN 978-0-8070-2106-4. LCCN 98035051. OCLC 39605684. Retrieved 9 August 2024.
+
      41. 
+
    41. ^ Cohen, Rachel D. (26 November 2018). "Thanks To Science, You Can Eat An Apple Every Day". The Salt. NPR. Archived from the original on 18 June 2024. Retrieved 1 August 2024.
+
      42. 
+
    42. ^ "The Heirloom Apple Orchard". The Jentsch Lab. Cornell University. Archived from the original on 30 July 2024. Retrieved 9 August 2024.
+
      43. 
+
    43. ^ Ranney, Thomas G. "Polyploidy: From Evolution to Landscape Plant Improvement". Proceedings of the 11th Metropolitan Tree Improvement Alliance (METRIA) Conference. 11th Metropolitan Tree Improvement Alliance Conference held in Gresham, Oregon, August 23–24, 2000. METRIA (NCSU.edu). METRIA. Archived from the original on 23 July 2010. Retrieved 7 November 2010.
+
      44. 
+
    44. ^ Lord, William G.; Ouellette, Amy (February 2010). "Dwarf Rootstocks for Apple Trees in the Home Garden" (PDF). University of New Hampshire. Archived from the original (PDF) on 30 September 2013. Retrieved 1 September 2013.
+
      45. 
+
    45. ^ Fallahi, Esmaeil; Colt, W. Michael; Fallahi, Bahar; Chun, Ik-Jo (January 2002). "The Importance of Apple Rootstocks on Tree Growth, Yield, Fruit Quality, Leaf Nutrition, and Photosynthesis with an Emphasis on 'Fuji'". HortTechnology. 12 (1): 38–44. doi:10.21273/HORTTECH.12.1.38. Archived (PDF) from the original on 11 February 2014. Retrieved 9 August 2024.
+
      46. 
+
    46. ^ Parker, M.L. (September 1993). "Apple Rootstocks and Tree Spacing". North Carolina Cooperative Extension Service. Archived from the original on 11 September 2013. Retrieved 1 September 2013.
+
      47. 
+
    47. ^ Ferree, David Curtis; Warrington, Ian J. (2003). Apples: Botany, Production, and Uses. New York: Centre for Agriculture and Bioscience International. pp. 33–35. ISBN 978-0851995922. OCLC 133167834.
+
      48. 
+
    48. ^ Jump up to: a b c d Polomski, Bob; Reighard, Greg. "Apple HGIC 1350". Home & Garden Information Center. Clemson University. Archived from the original on 28 February 2008. Retrieved 22 January 2008.
+
      49. 
+
    49. ^ Barahona, M. (1992). "Adaptation of Apple Varieties in Ecuador". Acta Horticulturae (310): 135–142. doi:10.17660/ActaHortic.1992.310.17.
+
      50. 
+
    50. ^ Adamson, Nancy Lee (2011). An Assessment of Non-Apis Bees as Fruit and Vegetable Crop Pollinators in Southwest Virginia (PDF) (Doctor of Philosophy in Entomology thesis). Virginia Polytechnic Institute and State University. Archived (PDF) from the original on 20 November 2015. Retrieved 15 October 2015.
+
      51. 
+
    51. ^ Powell, L.E. (1986). "The Chilling Requirement in Apple and Its Role in Regulating Time of Flowering in Spring in Cold-Winter Climate". Acta Horticulturae (179). Wageningen, Netherlands: International Society for Horticultural Science: 129–140. doi:10.17660/ActaHortic.1986.179.10. ISBN 978-90-6605-182-9.
+
      52. 
+
    52. ^ Romano, Andrea (10 September 2023). "20 Best Places to Go Apple Picking in the United States". Travel + Leisure. Archived from the original on 21 April 2024. Retrieved 2 August 2024.
+
      53. 
+
    53. ^ Graziano, Jack; Farcuh, Macarena (10 September 2021). "Controlled Atmosphere Storage of Apples". University of Maryland Extension. Archived from the original on 24 March 2023. Retrieved 2 August 2024.
+
      54. 
+
    54. ^ "FoodKeeper App". FoodSafety.gov. United States Department of Health and Human Services. 26 April 2019. Retrieved 17 September 2024.
+
      55. 
+
    55. ^ "4 Steps to Food Safety". FoodSafety.gov. United States Department of Health and Human Services. 12 April 2019. Retrieved 17 September 2024.
+
      56. 
+
    56. ^ "Refrigerated storage of perishable foods". CSIRO. 26 February 2015. Archived from the original on 15 March 2015. Retrieved 25 May 2007.
+
      57. 
+
    57. ^ Karp, David (25 October 2006). "Puff the Magic Preservative: Lasting Crunch, but Less Scent". The New York Times. Archived from the original on 3 August 2011. Retrieved 26 July 2017.
+
      58. 
+
    58. ^ Jackson, H.S. (1914). "Powdery Mildew". In Lowther, Granville; Worthington, William (eds.). The Encyclopedia of Practical Horticulture: A Reference System of Commercial Horticulture, Covering the Practical and Scientific Phases of Horticulture, with Special Reference to Fruits and Vegetables. Vol. I. North Yakima, Washington: The Encyclopedia of Horticulture Corporation. pp. 475–476. Retrieved 1 August 2024.
+
      59. 
+
    59. ^ Lowther, Granville; Worthington, William, eds. (1914). The Encyclopedia of Practical Horticulture: A Reference System of Commercial Horticulture, Covering the Practical and Scientific Phases of Horticulture, with Special Reference to Fruits and Vegetables. Vol. I. North Yakima, Washington: The Encyclopedia of Horticulture Corporation. pp. 45–51. Retrieved 1 August 2024.
+
      60. 
+
    60. ^ Coli, William M.; Los, Lorraine M., eds. (2003). "Insect Pests". 2003-2004 New England Apple Pest Management Guide. University of Massachusetts Amherst. pp. 28–29. Archived from the original on 12 February 2008. Retrieved 3 March 2008.{{cite book}}:  CS1 maint: bot: original URL status unknown (link)
+
      61. 
+
    61. ^ Jump up to: a b Atthowe, Helen; Gilkeson, Linda A.; Kite, L. Patricia; Michalak, Patricia S.; Pleasant, Barbara; Reich, Lee; Scheider, Alfred F. (2009). Bradley, Fern Marshall; Ellis, Bardara W.; Martin, Deborah L. (eds.). The Organic Gardener's Handbook of Natural Pest and Disease Control. New York: Rodale, Inc. pp. 32–34. ISBN 978-1-60529-677-7. LCCN 2009039996. OCLC 419860680.
+
      62. 
+
    62. ^ Coli, William M.; Berkett, Lorraine P.; Spitko, Robin, eds. (2003). "Other Apple Diseases". 2003-2004 New England Apple Pest Management Guide. University of Massachusetts Amherst. pp. 19–27. Archived from the original on 12 February 2008. Retrieved 3 March 2008.{{cite book}}:  CS1 maint: bot: original URL status unknown (link)
+
      63. 
+
    63. ^ Martin, Phillip L.; Krawczyk, Teresa; Khodadadi, Fatemeh; Aćimović, Srđan G.; Peter, Kari A. (2021). "Bitter Rot of Apple in the Mid-Atlantic United States: Causal Species and Evaluation of the Impacts of Regional Weather Patterns and Cultivar Susceptibility". Phytopathology. 111 (6): 966–981. doi:10.1094/PHYTO-09-20-0432-R. ISSN 0031-949X. PMID 33487025. S2CID 231701083.
+
      64. 
+
    64. ^ Erler, Fedai (1 January 2010). "Efficacy of tree trunk coating materials in the control of the apple clearwing, Synanthedon myopaeformis". Journal of Insect Science. 10 (1): 63. doi:10.1673/031.010.6301. PMC 3014806. PMID 20672979.
+
      65. 
+
    65. ^ Elzebroek, A. T. G.; Wind, Koop (2008). Guide to Cultivated Plants. Wallingford, United Kingdom: CABI. p. 27. ISBN 978-1-84593-356-2. LCCN 2007028459. OCLC 156975183. Archived from the original on 20 October 2020. Retrieved 6 October 2020.
+
      66. 
+
    66. ^ Jump up to: a b "Apple – Malus domestica". Natural England. Archived from the original on 12 May 2008. Retrieved 22 January 2008.
+
      67. 
+
    67. ^ "Home". National Fruit Collection. Archived from the original on 15 June 2012. Retrieved 2 December 2012.
+
      68. 
+
    68. ^ "ECPGR Malus/Pyrus Working Group Members". Ecpgr.cgiar.org. 22 July 2002. Archived from the original on 26 August 2014. Retrieved 25 August 2014.
+
      69. 
+
    69. ^ Jump up to: a b Tarjan, Sue (Fall 2006). "Autumn Apple Musings" (PDF). News & Notes of the UCSC Farm & Garden, Center for Agroecology & Sustainable Food Systems. pp. 1–2. Archived from the original (PDF) on 11 August 2007. Retrieved 24 January 2008.
+
      70. 
+
    70. ^ Beck, Kellen (17 October 2020). "How breeders bring out the best in new apples". Mashable. Archived from the original on 31 July 2024. Retrieved 31 July 2024.
+
      71. 
+
    71. ^ Migicovsky, Zoë (22 August 2021). "How a few good apples spawned today's top varieties — and why breeders must branch out". The Conversation. Archived from the original on 31 July 2024. Retrieved 31 July 2024.
+
      72. 
+
    72. ^ Peil, A.; Dunemann, F.; Richter, K.; Hoefer, M.; Király, I.; Flachowsky, H.; Hanke, M.-V. (2008). "Resistance Breeding in Apple at Dresden-Pillnitz". Ecofruit - 13th International Conference on Cultivation Technique and Phytopathological Problems in Organic Fruit-Growing: Proceedings to the Conference from 18thFebruary to 20th February 2008 at Weinsberg/Germany (in German): 220–225. Archived from the original on 28 January 2021. Retrieved 31 July 2024.
+
      73. 
+
    73. ^ Jump up to: a b "World apple situation". Archived from the original on 11 February 2008. Retrieved 24 January 2008.
+
      74. 
+
    74. ^ Weaver, Sue (June–July 2003). "Crops & Gardening – Apples of Antiquity". Hobby Farms Magazine. Archived from the original on 19 February 2017.
+
      75. 
+
    75. ^ Jump up to: a b c "Apple production in 2022; from pick lists: Crops/World Regions/Production Quantity". FAOSTAT, UN Food and Agriculture Organization, Statistics Division. 2024. Archived from the original on 12 November 2016. Retrieved 18 June 2024.
+
      76. 
+
    76. ^ Nelson, Lewis S.; Shih, Richard D.; Balick, Michael J. (2007). Handbook of Poisonous and Injurious Plants (Second ed.). New York: New York Botanical Garden : Springer. pp. 27, 211–212. ISBN 978-0387-31268-2. LCCN 2005938815. OCLC 77537459. Retrieved 11 September 2024.
+
      77. 
+
    77. ^ "Amygdalin". Toxnet, US Library of Medicine. Archived from the original on 21 April 2017. Retrieved 20 April 2017.
+
      78. 
+
    78. ^ Jump up to: a b c d e f "General Information – Apple". Informall. Archived from the original on 23 July 2012. Retrieved 17 October 2011.
+
      79. 
+
    79. ^ Landau, Elizabeth, Oral allergy syndrome may explain mysterious reactions, 8 April 2009, CNN Health, accessed 17 October 2011
+
      80. 
+
    80. ^ United States Food and Drug Administration (2024). "Daily Value on the Nutrition and Supplement Facts Labels". FDA. Archived from the original on 27 March 2024. Retrieved 28 March 2024.
+
      81. 
+
    81. ^ National Academies of Sciences, Engineering, and Medicine; Health and Medicine Division; Food and Nutrition Board; Committee to Review the Dietary Reference Intakes for Sodium and Potassium (2019). Oria, Maria; Harrison, Meghan; Stallings, Virginia A. (eds.). Dietary Reference Intakes for Sodium and Potassium. The National Academies Collection: Reports funded by National Institutes of Health. Washington, DC: National Academies Press (US). ISBN 978-0-309-48834-1. PMID 30844154. Archived from the original on 9 May 2024. Retrieved 21 June 2024.
+
      82. 
+
    82. ^ Jump up to: a b c d Davidson, Alan (2014). "Apple". In Jaine, Tom (ed.). The Oxford Companion to Food. Illustrated by Soun Vannithone (Third ed.). Oxford: Oxford University Press. pp. 27–31. ISBN 978-0-19-967733-7. LCCN 2013957569. OCLC 890807357. OL 27172691M. Retrieved 18 September 2024.
+
      83. 
+
    83. ^ Traverso, Amy (2011). The Apple Lover's Cookbook. Photographs by Squire Fox (First ed.). New York: W.W. Norton & Company. pp. 16, 32, 35, 45, 92, 137, 262–263, 275. ISBN 978-0-393-06599-2. LCCN 2011016560. OCLC 711051767. OL 16450839W.
+
      84. 
+
    84. ^ Kellogg, Kristi (15 January 2015). "81 Best Apple Recipes: Dinners, Desserts, Salads, and More". Epicurious. Archived from the original on 18 October 2020. Retrieved 17 October 2020.
+
      85. 
+
    85. ^ Davidson, Alan (2014). "Toffee Apple". In Jaine, Tom (ed.). The Oxford Companion to Food. Illustrated by Soun Vannithone (Third ed.). Oxford: Oxford University Press. p. 824. ISBN 978-0-19-967733-7. LCCN 2013957569. OCLC 890807357. OL 27172691M. Retrieved 18 September 2024.
+
      86. 
+
    86. ^ Shurpin, Yehuda. "Why All the Symbolic Rosh Hashanah Foods? "בולבול"". Chabad.org. Archived from the original on 21 March 2023. Retrieved 21 March 2023.
+
      87. 
+
    87. ^ Yepsen, Roger B. (2017) [1994]. Apples (Revised and Updated ed.). New York: W.W. Norton & Company. p. 52. ISBN 978-1-68268-019-3. LCCN 2017010136. OCLC 973918728.
+
      88. 
+
    88. ^ "Organic apples". USDA Agricultural Marketing Service. February 2016. Archived from the original on 24 February 2017. Retrieved 23 February 2017.
+
      89. 
+
    89. ^ Jump up to: a b "European Organic Apple Production Demonstrates the Value of Pesticides" (PDF). CropLife Foundation, Washington, DC. December 2011. Archived (PDF) from the original on 24 February 2017. Retrieved 23 February 2017.
+
      90. 
+
    90. ^ Ribeiro, Flávia A.P.; Gomes de Moura, Carolina F.; Aguiar, Odair; de Oliveira, Flavia; Spadari, Regina C.; Oliveira, Nara R.C.; Oshima, Celina T.F.; Ribeiro, Daniel A. (September 2014). "The chemopreventive activity of apple against carcinogenesis: antioxidant activity and cell cycle control". European Journal of Cancer Prevention (Review). 23 (5): 477–480. doi:10.1097/CEJ.0000000000000005. PMID 24366437. S2CID 23026644.
+
      91. 
+
    91. ^ Nicolas, J. J.; Richard-Forget, F. C.; Goupy, P. M.; Amiot, M. J.; Aubert, S. Y. (1 January 1994). "Enzymatic browning reactions in apple and apple products". Critical Reviews in Food Science and Nutrition. 34 (2): 109–157. doi:10.1080/10408399409527653. PMID 8011143.
+
      92. 
+
    92. ^ "PPO silencing". Okanagan Specialty Fruits. 2019. Archived from the original on 27 April 2021. Retrieved 14 November 2019.
+
      93. 
+
    93. ^ "United States: GM non-browning Arctic apple expands into foodservice". Fresh Fruit Portal. 13 August 2019. Archived from the original on 27 June 2021. Retrieved 14 November 2019.
+
      94. 
+
    94. ^ "Okanagan Specialty Fruits: Biotechnology Consultation Agency Response Letter BNF 000132". U.S. Food and Drug Administration. 20 March 2015. Archived from the original on 31 October 2017. Retrieved 14 November 2019.
+
      95. 
+
    95. ^ "Questions and answers: Arctic Apple". Canadian Food Inspection Agency, Government of Canada. 8 September 2017. Archived from the original on 19 September 2018. Retrieved 14 November 2019.
+
      96. 
+
    96. ^ Yu, Xiuzhu; Van De Voort, Frederick R.; Li, Zhixi; Yue, Tianli (2007). "Proximate Composition of the Apple Seed and Characterization of Its Oil". International Journal of Food Engineering. 3 (5). doi:10.2202/1556-3758.1283. S2CID 98590230.
+
      97. 
+
    97. ^ Jump up to: a b c Davidson, Hilda Roderick Ellis (1990) [1st pub. 1964]. Gods and Myths of Northern Europe. London: Penguin Books. pp. 165–166. ISBN 0-14-013627-4. OCLC 29336401.
+
      98. 
+
    98. ^ Davidson, Hilda Ellis (1998). Roles of the Northern Goddess. London; New York: Routledge. pp. 146–147. doi:10.4324/9780203025550. ISBN 0-415-13610-5. LCCN 97018309. OCLC 48138055.
+
      99. 
+
    99. ^ Biedermann, Hans (1992). Dictionary of Symbolism. Translated by Hulbert, James. New York: Facts on File. pp. 16–17. ISBN 978-0-8160-2593-0. LCCN 91044933. OCLC 25092926. Retrieved 3 October 2024.
+
      100. 
+
    100. ^ Jump up to: a b Ruck, Carl A. P.; Staples, Blaise D.; Heinrich, Clark (2001). The apples of Apollo : pagan and Christian mysteries of the Eucharist. Durham, North Carolina: Carolina Academic Press. pp. 64–70. ISBN 978-0-89089-924-3. LCCN 00040351. OCLC 46337324.
+
      101. 
+
    101. ^ "Eris - Greek Goddess of Strife & Discord (Roman Discordia)". Theoi Project. Aaron J. Atsma. Archived from the original on 25 September 2024. Retrieved 26 September 2024.
+
      102. 
+
    102. ^ Lucian (1905). The Works of Lucian of Samosata. Vol. I. Translated by Fowler, H.W.; Fowler, F.G. (First ed.). Oxford: Clarendon Press. pp. 78–85. LCCN 06001045. OCLC 506365. Retrieved 26 September 2024.
+
      103. 
+
    103. ^ "Judgement of Paris - Greek Mythology". Theoi Project. Aaron J. Atsma. Archived from the original on 24 August 2024. Retrieved 26 September 2024.
+
      104. 
+
    104. ^ Plato (1997). "Epigrams". In Cooper, John M.; Hutchinson, D.S. (eds.). Complete Works. Translated by Edmonds, J.M.; Cooper, John M. Indianapolis, Indiana: Hackett Publishing. p. 1744. ISBN 0-87220-349-2. LCCN 96053280. OCLC 36178550. Retrieved 27 September 2024.
+
      105. 
+
    105. ^ Pinsent, John (1969). Greek Mythology (First ed.). London: Paul Hamlyn. p. 79. ISBN 978-0-600-02422-4. LCCN 78449216. OCLC 61702. Retrieved 3 October 2024.
+
      106. 
+
    106. ^ "Atalanta (Atalante) - Arcadian Heroine of Greek Mythology". Theoi Project. Aaron J. Atsma. Archived from the original on 27 September 2024. Retrieved 3 October 2024.
+
      107. 
+
    107. ^ Flieger, Verlyn (2005). Interrupted Music : The Making of Tolkien's Mythology. Kent, Ohio: Kent State University Press. pp. 122–123. ISBN 978-0-87338-824-5. LCCN 2004024490. OCLC 56805947.
+
      108. 
+
    108. ^ Jump up to: a b "Why Do the Chinese Give Apples Around Christmas?". Teach English In China. 22 December 2019. Archived from the original on 1 October 2020. Retrieved 3 September 2024.
+
      109. 
+
    109. ^ Jump up to: a b Macrone, Michael (1998). Brush up your Bible!. New York: Gramercy Books. pp. 15–16, 340–341. ISBN 978-0-517-20189-3. OCLC 38270894. Retrieved 31 July 2024.
+
      110. 
+
    110. ^ Kissling, Paul J. (2004). Genesis. Vol. 1. Joplin, Missouri: College Press. p. 193. ISBN 978-0-89900-875-2. LCCN 2004022577. OCLC 56672257. Archived from the original on 26 January 2021. Retrieved 6 October 2020.
+
      111. 
+
    111. ^ Genesis 2:17
+
      112. 
+
    112. ^ Hendel, Ronald S. (2013). The Book of Genesis: A Biography. Princeton, New Jersey: Princeton University Press. p. 114. ISBN 978-0-69114012-4. LCCN 2012015634. OCLC 788265521. Archived from the original on 5 March 2023. Retrieved 4 October 2024. 
+
      113. 
+
    113. ^ Mieder, Wolfgang; Kingsbury, Stewart A.; Harder, Kelsie B., eds. (1996) [1992]. A Dictionary of American Proverbs (Paperback ed.). New York: Oxford University Press. p. 23. ISBN 978-0-19-511133-0. LCCN 91015508. OCLC 23693799. Retrieved 23 August 2024.
+
      114. 
+
    114. ^ Pollan, Michael (2001). The Botany of Desire: A Plant's-Eye View of the World (First ed.). New York: Random House. pp. 9, 22, 50. ISBN 978-0-375-50129-6. LCCN 00066479. OCLC 49803415.
+
      115. 
+
    
+
    Further reading
    
+
+
    
+
    •  Media related to Apples at Wikimedia Commons
    
+
    
+
    
+
    
+
    
+
+
+
+
+
+
+
+
    
+
    Retrieved from "https://en.wikipedia.org/w/index.php?title=Apple&oldid=1256174424"
    
+					
+				
    
+			
    
+			
+		
    
+		
    
+			
+
+
+		
    
+	
     
+
     
+
    
+	
      
+
      
+
+
+
      
\ No newline at end of file
diff --git a/tests/async/test_0.4.2_browser_manager.py b/tests/async/test_0.4.2_browser_manager.py
new file mode 100644
index 0000000..635b731
--- /dev/null
+++ b/tests/async/test_0.4.2_browser_manager.py
@@ -0,0 +1,160 @@
+import os
+import sys
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+
+# Assuming that the changes made allow different configurations
+# for managed browser, persistent context, and so forth.
+
+
+async def test_default_headless():
+    async with AsyncWebCrawler(
+        headless=True,
+        verbose=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "mobile", "os_type": "android"},
+        use_managed_browser=False,
+        use_persistent_context=False,
+        ignore_https_errors=True,
+        # Testing normal ephemeral context
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_default_headless] success:", result.success)
+        print("HTML length:", len(result.html if result.html else ""))
+
+
+async def test_managed_browser_persistent():
+    # Treating use_persistent_context=True as managed_browser scenario.
+    async with AsyncWebCrawler(
+        headless=False,
+        verbose=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "desktop", "os_type": "mac"},
+        use_managed_browser=True,
+        use_persistent_context=True,  # now should behave same as managed browser
+        user_data_dir="./outpu/test_profile",
+        # This should store and reuse profile data across runs
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.google.com",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_managed_browser_persistent] success:", result.success)
+        print("HTML length:", len(result.html if result.html else ""))
+
+
+async def test_session_reuse():
+    # Test creating a session, using it for multiple calls
+    session_id = "my_session"
+    async with AsyncWebCrawler(
+        headless=False,
+        verbose=True,
+        user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+        # Fixed user-agent for consistency
+        use_managed_browser=False,
+        use_persistent_context=False,
+    ) as crawler:
+        # First call: create session
+        result1 = await crawler.arun(
+            url="https://www.example.com",
+            cache_mode=CacheMode.BYPASS,
+            session_id=session_id,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_session_reuse first call] success:", result1.success)
+
+        # Second call: same session, possibly cookie retained
+        result2 = await crawler.arun(
+            url="https://www.example.com/about",
+            cache_mode=CacheMode.BYPASS,
+            session_id=session_id,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_session_reuse second call] success:", result2.success)
+
+
+async def test_magic_mode():
+    # Test magic mode with override_navigator and simulate_user
+    async with AsyncWebCrawler(
+        headless=False,
+        verbose=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "desktop", "os_type": "windows"},
+        use_managed_browser=False,
+        use_persistent_context=False,
+        magic=True,
+        override_navigator=True,
+        simulate_user=True,
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/business",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_magic_mode] success:", result.success)
+        print("HTML length:", len(result.html if result.html else ""))
+
+
+async def test_proxy_settings():
+    # Test with a proxy (if available) to ensure code runs with proxy
+    async with AsyncWebCrawler(
+        headless=True,
+        verbose=False,
+        user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+        proxy_config={"server": "http://127.0.0.1:8080"},  # Assuming local proxy server for test
+        use_managed_browser=False,
+        use_persistent_context=False,
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://httpbin.org/ip",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_proxy_settings] success:", result.success)
+        if result.success:
+            print("HTML preview:", result.html[:200] if result.html else "")
+
+
+async def test_ignore_https_errors():
+    # Test ignore HTTPS errors with a self-signed or invalid cert domain
+    # This is just conceptual, the domain should be one that triggers SSL error.
+    # Using a hypothetical URL that fails SSL:
+    async with AsyncWebCrawler(
+        headless=True,
+        verbose=True,
+        user_agent="Mozilla/5.0",
+        ignore_https_errors=True,
+        use_managed_browser=False,
+        use_persistent_context=False,
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://self-signed.badssl.com/",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_ignore_https_errors] success:", result.success)
+
+
+async def main():
+    print("Running tests...")
+    # await test_default_headless()
+    # await test_managed_browser_persistent()
+    # await test_session_reuse()
+    # await test_magic_mode()
+    # await test_proxy_settings()
+    await test_ignore_https_errors()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/async/test_0.4.2_config_params.py b/tests/async/test_0.4.2_config_params.py
new file mode 100644
index 0000000..bb2113d
--- /dev/null
+++ b/tests/async/test_0.4.2_config_params.py
@@ -0,0 +1,211 @@
+import os, sys
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai import JsonCssExtractionStrategy
+from crawl4ai.chunking_strategy import RegexChunking
+
+
+# Category 1: Browser Configuration Tests
+async def test_browser_config_object():
+    """Test the new BrowserConfig object with various browser settings"""
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=False,
+        viewport_width=1920,
+        viewport_height=1080,
+        use_managed_browser=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "desktop", "os_type": "windows"},
+    )
+
+    async with AsyncWebCrawler(config=browser_config, verbose=True) as crawler:
+        result = await crawler.arun("https://example.com", cache_mode=CacheMode.BYPASS)
+        assert result.success, "Browser config crawl failed"
+        assert len(result.html) > 0, "No HTML content retrieved"
+
+
+async def test_browser_performance_config():
+    """Test browser configurations focused on performance"""
+    browser_config = BrowserConfig(
+        text_mode=True,
+        light_mode=True,
+        extra_args=["--disable-gpu", "--disable-software-rasterizer"],
+        ignore_https_errors=True,
+        java_script_enabled=False,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun("https://example.com")
+        assert result.success, "Performance optimized crawl failed"
+        assert result.status_code == 200, "Unexpected status code"
+
+
+# Category 2: Content Processing Tests
+async def test_content_extraction_config():
+    """Test content extraction with various strategies"""
+    crawler_config = CrawlerRunConfig(
+        word_count_threshold=300,
+        extraction_strategy=JsonCssExtractionStrategy(
+            schema={
+                "name": "article",
+                "baseSelector": "div",
+                "fields": [{"name": "title", "selector": "h1", "type": "text"}],
+            }
+        ),
+        chunking_strategy=RegexChunking(),
+        content_filter=PruningContentFilter(),
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            "https://example.com/article", config=crawler_config
+        )
+        assert result.extracted_content is not None, "Content extraction failed"
+        assert "title" in result.extracted_content, "Missing expected content field"
+
+
+# Category 3: Cache and Session Management Tests
+async def test_cache_and_session_management():
+    """Test different cache modes and session handling"""
+    browser_config = BrowserConfig(use_persistent_context=True)
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.WRITE_ONLY,
+        process_iframes=True,
+        remove_overlay_elements=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # First request - should write to cache
+        result1 = await crawler.arun("https://example.com", config=crawler_config)
+
+        # Second request - should use fresh fetch due to WRITE_ONLY mode
+        result2 = await crawler.arun("https://example.com", config=crawler_config)
+
+        assert result1.success and result2.success, "Cache mode crawl failed"
+        assert result1.html == result2.html, "Inconsistent results between requests"
+
+
+# Category 4: Media Handling Tests
+async def test_media_handling_config():
+    """Test configurations related to media handling"""
+    # Get the base path for home directroy ~/.crawl4ai/downloads, make sure it exists
+    os.makedirs(os.path.expanduser("~/.crawl4ai/downloads"), exist_ok=True)
+    browser_config = BrowserConfig(
+        viewport_width=1920,
+        viewport_height=1080,
+        accept_downloads=True,
+        downloads_path=os.path.expanduser("~/.crawl4ai/downloads"),
+    )
+    crawler_config = CrawlerRunConfig(
+        screenshot=True,
+        pdf=True,
+        adjust_viewport_to_content=True,
+        wait_for_images=True,
+        screenshot_height_threshold=20000,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun("https://example.com", config=crawler_config)
+        assert result.screenshot is not None, "Screenshot capture failed"
+        assert result.pdf is not None, "PDF generation failed"
+
+
+# Category 5: Anti-Bot and Site Interaction Tests
+async def test_antibot_config():
+    """Test configurations for handling anti-bot measures"""
+    crawler_config = CrawlerRunConfig(
+        simulate_user=True,
+        override_navigator=True,
+        magic=True,
+        wait_for="js:()=>document.querySelector('body')",
+        delay_before_return_html=1.0,
+        log_console=True,
+        cache_mode=CacheMode.BYPASS,
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com", config=crawler_config)
+        assert result.success, "Anti-bot measure handling failed"
+
+
+# Category 6: Parallel Processing Tests
+async def test_parallel_processing():
+    """Test parallel processing capabilities"""
+    crawler_config = CrawlerRunConfig(mean_delay=0.5, max_range=1.0, semaphore_count=5)
+
+    urls = ["https://example.com/1", "https://example.com/2", "https://example.com/3"]
+
+    async with AsyncWebCrawler() as crawler:
+        results = await crawler.arun_many(urls, config=crawler_config)
+        assert len(results) == len(urls), "Not all URLs were processed"
+        assert all(r.success for r in results), "Some parallel requests failed"
+
+
+# Category 7: Backwards Compatibility Tests
+async def test_legacy_parameter_support():
+    """Test that legacy parameters still work"""
+    async with AsyncWebCrawler(
+        headless=True, browser_type="chromium", viewport_width=1024, viewport_height=768
+    ) as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            screenshot=True,
+            word_count_threshold=200,
+            bypass_cache=True,
+            css_selector=".main-content",
+        )
+        assert result.success, "Legacy parameter support failed"
+
+
+# Category 8: Mixed Configuration Tests
+async def test_mixed_config_usage():
+    """Test mixing new config objects with legacy parameters"""
+    browser_config = BrowserConfig(headless=True)
+    crawler_config = CrawlerRunConfig(screenshot=True)
+
+    async with AsyncWebCrawler(
+        config=browser_config,
+        verbose=True,  # legacy parameter
+    ) as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            config=crawler_config,
+            cache_mode=CacheMode.BYPASS,  # legacy parameter
+            css_selector="body",  # legacy parameter
+        )
+        assert result.success, "Mixed configuration usage failed"
+
+
+if __name__ == "__main__":
+
+    async def run_tests():
+        test_functions = [
+            test_browser_config_object,
+            # test_browser_performance_config,
+            # test_content_extraction_config,
+            # test_cache_and_session_management,
+            # test_media_handling_config,
+            # test_antibot_config,
+            # test_parallel_processing,
+            # test_legacy_parameter_support,
+            # test_mixed_config_usage
+        ]
+
+        for test in test_functions:
+            print(f"\nRunning {test.__name__}...")
+            try:
+                await test()
+                print(f"✓ {test.__name__} passed")
+            except AssertionError as e:
+                print(f"✗ {test.__name__} failed: {str(e)}")
+            except Exception as e:
+                print(f"✗ {test.__name__} error: {str(e)}")
+
+    asyncio.run(run_tests())
diff --git a/tests/async/test_async_doanloader.py b/tests/async/test_async_doanloader.py
new file mode 100644
index 0000000..055886c
--- /dev/null
+++ b/tests/async/test_async_doanloader.py
@@ -0,0 +1,247 @@
+import os
+import sys
+import asyncio
+import shutil
+from typing import List
+import tempfile
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+class TestDownloads:
+    def __init__(self):
+        self.temp_dir = tempfile.mkdtemp(prefix="crawl4ai_test_")
+        self.download_dir = os.path.join(self.temp_dir, "downloads")
+        os.makedirs(self.download_dir, exist_ok=True)
+        self.results: List[str] = []
+
+    def cleanup(self):
+        shutil.rmtree(self.temp_dir)
+
+    def log_result(self, test_name: str, success: bool, message: str = ""):
+        result = f"{'✅' if success else '❌'} {test_name}: {message}"
+        self.results.append(result)
+        print(result)
+
+    async def test_basic_download(self):
+        """Test basic file download functionality"""
+        try:
+            async with AsyncWebCrawler(
+                accept_downloads=True, downloads_path=self.download_dir, verbose=True
+            ) as crawler:
+                # Python.org downloads page typically has stable download links
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="""
+                    // Click first download link
+                    const downloadLink = document.querySelector('a[href$=".exe"]');
+                    if (downloadLink) downloadLink.click();
+                    """,
+                )
+
+                success = (
+                    result.downloaded_files is not None
+                    and len(result.downloaded_files) > 0
+                )
+                self.log_result(
+                    "Basic Download",
+                    success,
+                    f"Downloaded {len(result.downloaded_files or [])} files"
+                    if success
+                    else "No files downloaded",
+                )
+        except Exception as e:
+            self.log_result("Basic Download", False, str(e))
+
+    async def test_persistent_context_download(self):
+        """Test downloads with persistent context"""
+        try:
+            user_data_dir = os.path.join(self.temp_dir, "user_data")
+            os.makedirs(user_data_dir, exist_ok=True)
+
+            async with AsyncWebCrawler(
+                accept_downloads=True,
+                downloads_path=self.download_dir,
+                use_persistent_context=True,
+                user_data_dir=user_data_dir,
+                verbose=True,
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="""
+                    const downloadLink = document.querySelector('a[href$=".exe"]');
+                    if (downloadLink) downloadLink.click();
+                    """,
+                )
+
+                success = (
+                    result.downloaded_files is not None
+                    and len(result.downloaded_files) > 0
+                )
+                self.log_result(
+                    "Persistent Context Download",
+                    success,
+                    f"Downloaded {len(result.downloaded_files or [])} files"
+                    if success
+                    else "No files downloaded",
+                )
+        except Exception as e:
+            self.log_result("Persistent Context Download", False, str(e))
+
+    async def test_multiple_downloads(self):
+        """Test multiple simultaneous downloads"""
+        try:
+            async with AsyncWebCrawler(
+                accept_downloads=True, downloads_path=self.download_dir, verbose=True
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="""
+                    // Click multiple download links
+                    const downloadLinks = document.querySelectorAll('a[href$=".exe"]');
+                    downloadLinks.forEach(link => link.click());
+                    """,
+                )
+
+                success = (
+                    result.downloaded_files is not None
+                    and len(result.downloaded_files) > 1
+                )
+                self.log_result(
+                    "Multiple Downloads",
+                    success,
+                    f"Downloaded {len(result.downloaded_files or [])} files"
+                    if success
+                    else "Not enough files downloaded",
+                )
+        except Exception as e:
+            self.log_result("Multiple Downloads", False, str(e))
+
+    async def test_different_browsers(self):
+        """Test downloads across different browser types"""
+        browsers = ["chromium", "firefox", "webkit"]
+
+        for browser_type in browsers:
+            try:
+                async with AsyncWebCrawler(
+                    accept_downloads=True,
+                    downloads_path=self.download_dir,
+                    browser_type=browser_type,
+                    verbose=True,
+                ) as crawler:
+                    result = await crawler.arun(
+                        url="https://www.python.org/downloads/",
+                        js_code="""
+                        const downloadLink = document.querySelector('a[href$=".exe"]');
+                        if (downloadLink) downloadLink.click();
+                        """,
+                    )
+
+                    success = (
+                        result.downloaded_files is not None
+                        and len(result.downloaded_files) > 0
+                    )
+                    self.log_result(
+                        f"{browser_type.title()} Download",
+                        success,
+                        f"Downloaded {len(result.downloaded_files or [])} files"
+                        if success
+                        else "No files downloaded",
+                    )
+            except Exception as e:
+                self.log_result(f"{browser_type.title()} Download", False, str(e))
+
+    async def test_edge_cases(self):
+        """Test various edge cases"""
+
+        # Test 1: Downloads without specifying download path
+        try:
+            async with AsyncWebCrawler(accept_downloads=True, verbose=True) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="document.querySelector('a[href$=\".exe\"]').click()",
+                )
+                self.log_result(
+                    "Default Download Path",
+                    True,
+                    f"Downloaded to default path: {result.downloaded_files[0] if result.downloaded_files else 'None'}",
+                )
+        except Exception as e:
+            self.log_result("Default Download Path", False, str(e))
+
+        # Test 2: Downloads with invalid path
+        try:
+            async with AsyncWebCrawler(
+                accept_downloads=True,
+                downloads_path="/invalid/path/that/doesnt/exist",
+                verbose=True,
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="document.querySelector('a[href$=\".exe\"]').click()",
+                )
+                self.log_result(
+                    "Invalid Download Path", False, "Should have raised an error"
+                )
+        except Exception:
+            self.log_result(
+                "Invalid Download Path", True, "Correctly handled invalid path"
+            )
+
+        # Test 3: Download with accept_downloads=False
+        try:
+            async with AsyncWebCrawler(accept_downloads=False, verbose=True) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="document.querySelector('a[href$=\".exe\"]').click()",
+                )
+                success = result.downloaded_files is None
+                self.log_result(
+                    "Disabled Downloads",
+                    success,
+                    "Correctly ignored downloads"
+                    if success
+                    else "Unexpectedly downloaded files",
+                )
+        except Exception as e:
+            self.log_result("Disabled Downloads", False, str(e))
+
+    async def run_all_tests(self):
+        """Run all test cases"""
+        print("\n🧪 Running Download Tests...\n")
+
+        test_methods = [
+            self.test_basic_download,
+            self.test_persistent_context_download,
+            self.test_multiple_downloads,
+            self.test_different_browsers,
+            self.test_edge_cases,
+        ]
+
+        for test in test_methods:
+            print(f"\n📝 Running {test.__doc__}...")
+            await test()
+            await asyncio.sleep(2)  # Brief pause between tests
+
+        print("\n📊 Test Results Summary:")
+        for result in self.results:
+            print(result)
+
+        successes = len([r for r in self.results if "✅" in r])
+        total = len(self.results)
+        print(f"\nTotal: {successes}/{total} tests passed")
+
+        self.cleanup()
+
+
+async def main():
+    tester = TestDownloads()
+    await tester.run_all_tests()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/async/test_basic_crawling.py b/tests/async/test_basic_crawling.py
new file mode 100644
index 0000000..ee4bb63
--- /dev/null
+++ b/tests/async/test_basic_crawling.py
@@ -0,0 +1,90 @@
+import os
+import sys
+import pytest
+import time
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_successful_crawl():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.url == url
+        assert result.html
+        assert result.markdown
+        assert result.cleaned_html
+
+
+@pytest.mark.asyncio
+async def test_invalid_url():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.invalidurl12345.com"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert not result.success
+        assert result.error_message
+
+
+@pytest.mark.asyncio
+async def test_multiple_urls():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        urls = [
+            "https://www.nbcnews.com/business",
+            "https://www.example.com",
+            "https://www.python.org",
+        ]
+        results = await crawler.arun_many(urls=urls, bypass_cache=True)
+        assert len(results) == len(urls)
+        assert all(result.success for result in results)
+        assert all(result.html for result in results)
+
+
+@pytest.mark.asyncio
+async def test_javascript_execution():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        js_code = "document.body.innerHTML = '
      Modified by JS
      ';"
+        url = "https://www.example.com"
+        result = await crawler.arun(url=url, bypass_cache=True, js_code=js_code)
+        assert result.success
+        assert "
      Modified by JS
      " in result.html
+
+
+@pytest.mark.asyncio
+async def test_concurrent_crawling_performance():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        urls = [
+            "https://www.nbcnews.com/business",
+            "https://www.example.com",
+            "https://www.python.org",
+            "https://www.github.com",
+            "https://www.stackoverflow.com",
+        ]
+
+        start_time = time.time()
+        results = await crawler.arun_many(urls=urls, bypass_cache=True)
+        end_time = time.time()
+
+        total_time = end_time - start_time
+        print(f"Total time for concurrent crawling: {total_time:.2f} seconds")
+
+        assert all(result.success for result in results)
+        assert len(results) == len(urls)
+
+        # Assert that concurrent crawling is faster than sequential
+        # This multiplier may need adjustment based on the number of URLs and their complexity
+        assert (
+            total_time < len(urls) * 5
+        ), f"Concurrent crawling not significantly faster: {total_time:.2f} seconds"
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_browser_lifecycle.py b/tests/async/test_browser_lifecycle.py
new file mode 100644
index 0000000..b7042cc
--- /dev/null
+++ b/tests/async/test_browser_lifecycle.py
@@ -0,0 +1,972 @@
+"""
+Browser lifecycle & concurrency tests.
+
+Covers all the browser launch paths and lock interactions:
+  - Standalone (playwright.launch)
+  - Managed browser (subprocess + CDP connect)
+  - Managed browser with create_isolated_context
+  - Page reuse on shared default context
+  - Context caching / LRU eviction
+  - Session lifecycle across all modes
+  - Concurrent crawls racing for pages / contexts
+  - Recycle interacting with managed browser
+  - Multiple crawlers sharing a managed browser via CDP
+"""
+
+import asyncio
+import time
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+
+# ---------------------------------------------------------------------------
+# Local test server
+# ---------------------------------------------------------------------------
+
+PAGES = {}
+for i in range(100):
+    PAGES[f"/page{i}"] = (
+        f"Page {i}"
+        f"
      Page {i}
      Content for page {i}.
      "
+        f"next"
+    ).encode()
+
+# Login/dashboard for session tests
+PAGES["/login"] = (
+    b"Login"
+    b"
      Login
      Logged in.
      "
+)
+PAGES["/dashboard"] = (
+    b"Dashboard"
+    b"
      Dashboard
      Dashboard content.
      "
+)
+
+
+class Handler(SimpleHTTPRequestHandler):
+    def log_message(self, *a):
+        pass
+
+    def do_GET(self):
+        body = PAGES.get(self.path, PAGES["/page0"])
+        self.send_response(200)
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(body)
+
+
+class _Server(HTTPServer):
+    allow_reuse_address = True
+
+
+@pytest.fixture(scope="module")
+def srv():
+    s = _Server(("127.0.0.1", 0), Handler)
+    port = s.server_address[1]
+    t = threading.Thread(target=s.serve_forever, daemon=True)
+    t.start()
+    yield f"http://127.0.0.1:{port}"
+    s.shutdown()
+
+
+def _u(base, i):
+    return f"{base}/page{i}"
+
+
+def _bm(c):
+    return c.crawler_strategy.browser_manager
+
+
+# ===================================================================
+# SECTION A — Standalone browser (no CDP, no managed browser)
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_standalone_basic_crawl(srv):
+    """Standalone browser: launch, crawl, close. Baseline correctness."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        r = await c.arun(url=_u(srv, 0), config=run)
+        assert r.success
+        assert "Page 0" in r.html
+
+
+@pytest.mark.asyncio
+async def test_standalone_sequential_crawls(srv):
+    """10 sequential pages — each gets its own page, context reused by config sig."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        for i in range(10):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+            assert f"Page {i}" in r.html
+
+
+@pytest.mark.asyncio
+async def test_standalone_concurrent_crawls(srv):
+    """10 concurrent crawls on standalone browser — no crashes,
+    context lock prevents race conditions."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(10)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        assert all(r.success for r in results if not isinstance(r, Exception))
+
+
+@pytest.mark.asyncio
+async def test_standalone_context_reuse(srv):
+    """Two crawls with identical config should reuse the same context.
+    Two crawls with different configs should create different contexts."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        run_a = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        r1 = await c.arun(url=_u(srv, 0), config=run_a)
+        assert r1.success
+        ctx_count_after_first = len(bm.contexts_by_config)
+
+        # Same config → same context
+        r2 = await c.arun(url=_u(srv, 1), config=run_a)
+        assert r2.success
+        assert len(bm.contexts_by_config) == ctx_count_after_first, (
+            "Same config should reuse context"
+        )
+
+        # Different config → new context
+        run_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, verbose=False,
+            override_navigator=True,
+        )
+        r3 = await c.arun(url=_u(srv, 2), config=run_b)
+        assert r3.success
+        assert len(bm.contexts_by_config) == ctx_count_after_first + 1, (
+            "Different config should create new context"
+        )
+
+
+@pytest.mark.asyncio
+async def test_standalone_session_multistep(srv):
+    """Session across 3 pages on standalone browser."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    sess = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, session_id="standalone_sess", verbose=False,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(3):
+            r = await c.arun(url=_u(srv, i), config=sess)
+            assert r.success
+            assert "standalone_sess" in bm.sessions
+
+        # Refcount should be exactly 1
+        _, page, _ = bm.sessions["standalone_sess"]
+        sig = bm._page_to_sig.get(page)
+        if sig:
+            assert bm._context_refcounts.get(sig, 0) == 1
+
+        # Kill session and verify cleanup
+        await c.crawler_strategy.kill_session("standalone_sess")
+        assert "standalone_sess" not in bm.sessions
+        if sig:
+            assert bm._context_refcounts.get(sig, 0) == 0
+
+
+@pytest.mark.asyncio
+async def test_standalone_recycle(srv):
+    """Recycling on standalone browser — close/start cycle."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False, max_pages_before_recycle=5,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+        for i in range(8):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+        # Recycle happened at page 5, pages 6-8 after → counter = 3
+        assert bm._pages_served == 3
+
+
+@pytest.mark.asyncio
+async def test_standalone_recycle_with_concurrent_crawls(srv):
+    """15 concurrent crawls straddling a recycle boundary on standalone."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False, max_pages_before_recycle=5,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(15)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 15
+
+
+# ===================================================================
+# SECTION B — Managed browser (subprocess + CDP)
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_managed_basic_crawl(srv):
+    """Managed browser: start subprocess, connect via CDP, crawl, close."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        r = await c.arun(url=_u(srv, 0), config=run)
+        assert r.success
+        assert "Page 0" in r.html
+
+
+@pytest.mark.asyncio
+async def test_managed_sequential_crawls(srv):
+    """Sequential crawls on managed browser — pages reused from default context."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        for i in range(8):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+
+@pytest.mark.asyncio
+async def test_managed_concurrent_crawls(srv):
+    """Concurrent crawls on managed browser — _global_pages_lock prevents
+    two tasks from grabbing the same page."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(8)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 8
+
+
+@pytest.mark.asyncio
+async def test_managed_page_reuse(srv):
+    """On managed browser (non-isolated), pages should be reused when
+    released back to the pool."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Crawl 3 pages sequentially — page should be reused each time
+        for i in range(3):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # On managed browser, total pages created should be small
+        # (pages reused, not new ones for each crawl)
+        default_ctx = bm.default_context
+        total_pages = len(default_ctx.pages)
+        assert total_pages <= 3, (
+            f"Expected page reuse, but {total_pages} pages exist"
+        )
+
+
+@pytest.mark.asyncio
+async def test_managed_session_multistep(srv):
+    """Multi-step session on managed browser — session page stays alive."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    sess = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, session_id="managed_sess", verbose=False,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        r = await c.arun(url=f"{srv}/login", config=sess)
+        assert r.success
+
+        r = await c.arun(url=f"{srv}/dashboard", config=sess)
+        assert r.success
+
+        assert "managed_sess" in bm.sessions
+
+
+@pytest.mark.asyncio
+async def test_managed_recycle(srv):
+    """Recycling on managed browser — kills subprocess, restarts, crawls resume."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        max_pages_before_recycle=4,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(7):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed after managed recycle"
+
+        # Recycled at 4 → pages 5,6,7 after → counter = 3
+        assert bm._pages_served == 3
+
+
+# ===================================================================
+# SECTION C — Managed browser with create_isolated_context
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_isolated_context_basic(srv):
+    """Isolated context mode: each config gets its own browser context."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        r = await c.arun(url=_u(srv, 0), config=run)
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_concurrent(srv):
+    """Concurrent crawls with isolated contexts — _contexts_lock prevents
+    race conditions in context creation."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(10)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 10
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_caching(srv):
+    """Same config signature → same context. Different config → different context."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        run_a = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        await c.arun(url=_u(srv, 0), config=run_a)
+        count_after_a = len(bm.contexts_by_config)
+
+        # Same config → reuse
+        await c.arun(url=_u(srv, 1), config=run_a)
+        assert len(bm.contexts_by_config) == count_after_a
+
+        # Different config → new context
+        run_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, verbose=False,
+            override_navigator=True,
+        )
+        await c.arun(url=_u(srv, 2), config=run_b)
+        assert len(bm.contexts_by_config) == count_after_a + 1
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_refcount(srv):
+    """Refcount increases with concurrent crawls and decreases on release."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # After a single sequential crawl (page released), refcount should be 0
+        r = await c.arun(url=_u(srv, 0), config=run)
+        assert r.success
+
+        # All contexts should have refcount 0 (page was released)
+        for sig, rc in bm._context_refcounts.items():
+            assert rc == 0, f"Refcount for {sig[:8]}... should be 0, got {rc}"
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_session_with_interleaved(srv):
+    """Session on isolated context + non-session crawls interleaved."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        sess = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="iso_sess", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess)
+        assert r.success
+
+        # Non-session crawls
+        run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        for i in range(5):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # Session still alive
+        assert "iso_sess" in bm.sessions
+        r = await c.arun(url=f"{srv}/dashboard", config=sess)
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_recycle(srv):
+    """Recycling with isolated contexts — all contexts cleared, new ones
+    created fresh on the new browser."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+        max_pages_before_recycle=4,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(6):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+        # Recycled at 4 → 5,6 after → counter = 2
+        assert bm._pages_served == 2
+        # Contexts dict should only have entries from after recycle
+        assert all(rc == 0 for rc in bm._context_refcounts.values()), (
+            "All refcounts should be 0 after sequential crawls"
+        )
+
+
+# ===================================================================
+# SECTION D — Two crawlers sharing one managed browser via CDP URL
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_two_crawlers_share_managed_browser(srv):
+    """Two AsyncWebCrawler instances connect to the same managed browser
+    via its CDP URL. Both should crawl successfully without interfering."""
+    # First crawler owns the managed browser
+    cfg1 = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+
+    async with AsyncWebCrawler(config=cfg1) as c1:
+        bm1 = _bm(c1)
+        # Grab the CDP URL from the managed browser
+        cdp_url = f"http://{bm1.managed_browser.host}:{bm1.managed_browser.debugging_port}"
+
+        # Second crawler connects to the same browser via CDP
+        cfg2 = BrowserConfig(
+            headless=True, verbose=False,
+            cdp_url=cdp_url,
+            cdp_cleanup_on_close=True,
+        )
+        async with AsyncWebCrawler(config=cfg2) as c2:
+            run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+            # Crawl sequentially to avoid page contention on shared context
+            r1 = await c1.arun(url=_u(srv, 0), config=run)
+            r2 = await c2.arun(url=_u(srv, 1), config=run)
+
+            assert r1.success, f"Crawler 1 failed: {r1.error_message}"
+            assert r2.success, f"Crawler 2 failed: {r2.error_message}"
+            assert "Page 0" in r1.html
+            assert "Page 1" in r2.html
+
+
+@pytest.mark.asyncio
+async def test_two_crawlers_concurrent_heavy(srv):
+    """Two crawlers sharing one managed browser, each doing 5 concurrent crawls."""
+    cfg1 = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+
+    async with AsyncWebCrawler(config=cfg1) as c1:
+        bm1 = _bm(c1)
+        cdp_url = f"http://{bm1.managed_browser.host}:{bm1.managed_browser.debugging_port}"
+
+        cfg2 = BrowserConfig(
+            headless=True, verbose=False,
+            cdp_url=cdp_url,
+            cdp_cleanup_on_close=True,
+        )
+        async with AsyncWebCrawler(config=cfg2) as c2:
+            run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+            # Each crawler does 5 sequential crawls while both are connected
+            for i in range(5):
+                r1 = await c1.arun(url=_u(srv, i), config=run)
+                assert r1.success, f"Crawler 1 page {i} failed: {r1.error_message}"
+                r2 = await c2.arun(url=_u(srv, i + 50), config=run)
+                assert r2.success, f"Crawler 2 page {i} failed: {r2.error_message}"
+
+
+# ===================================================================
+# SECTION E — Session lifecycle edge cases
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_session_then_nonsession_then_session(srv):
+    """session crawl → non-session crawl → session crawl.
+    The session should persist across non-session activity."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    sess = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, session_id="interleave_sess", verbose=False,
+    )
+    no_sess = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        r = await c.arun(url=_u(srv, 0), config=sess)
+        assert r.success
+
+        # Non-session crawls
+        for i in range(3):
+            r = await c.arun(url=_u(srv, 10 + i), config=no_sess)
+            assert r.success
+
+        # Session should still exist and work
+        assert "interleave_sess" in bm.sessions
+        r = await c.arun(url=_u(srv, 99), config=sess)
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_multiple_sessions_simultaneous(srv):
+    """3 independent sessions open at the same time, each navigating
+    different pages. They should not interfere."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        sessions = [
+            CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS, session_id=f"sess_{j}", verbose=False,
+            )
+            for j in range(3)
+        ]
+
+        # Step 1: open all sessions
+        for j, s in enumerate(sessions):
+            r = await c.arun(url=_u(srv, j * 10), config=s)
+            assert r.success, f"Session {j} open failed"
+
+        assert len(bm.sessions) == 3
+
+        # Step 2: navigate each session to a second page
+        for j, s in enumerate(sessions):
+            r = await c.arun(url=_u(srv, j * 10 + 1), config=s)
+            assert r.success, f"Session {j} step 2 failed"
+
+        # Step 3: kill sessions one by one, verify others unaffected
+        await c.crawler_strategy.kill_session("sess_0")
+        assert "sess_0" not in bm.sessions
+        assert "sess_1" in bm.sessions
+        assert "sess_2" in bm.sessions
+
+        # Remaining sessions still work
+        r = await c.arun(url=_u(srv, 99), config=sessions[1])
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_session_kill_then_recreate(srv):
+    """Kill a session, then create a new session with the same ID.
+    The new session should work on a fresh page."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    sess = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, session_id="reuse_id", verbose=False,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        r = await c.arun(url=_u(srv, 0), config=sess)
+        assert r.success
+        _, page_v1, _ = bm.sessions["reuse_id"]
+
+        await c.crawler_strategy.kill_session("reuse_id")
+        assert "reuse_id" not in bm.sessions
+
+        # Re-create with same ID
+        r = await c.arun(url=_u(srv, 50), config=sess)
+        assert r.success
+        assert "reuse_id" in bm.sessions
+        _, page_v2, _ = bm.sessions["reuse_id"]
+
+        # Should be a different page object
+        assert page_v1 is not page_v2, "Re-created session should have a new page"
+
+
+# ===================================================================
+# SECTION F — Concurrent recycle + session stress tests
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_concurrent_sessions_and_nonsessions(srv):
+    """Open 2 sessions + fire 10 non-session crawls concurrently with
+    recycle threshold=5. Sessions should block recycle until they're
+    done or killed. All crawls should succeed."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=5,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Open sessions first
+        sess_a = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="stress_a", verbose=False,
+        )
+        sess_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="stress_b", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess_a)
+        assert r.success
+        r = await c.arun(url=f"{srv}/login", config=sess_b)
+        assert r.success
+
+        # Fire 10 concurrent non-session crawls
+        no_sess = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        tasks = [c.arun(url=_u(srv, i), config=no_sess) for i in range(10)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+
+        # Sessions should still be alive (blocking recycle)
+        assert "stress_a" in bm.sessions
+        assert "stress_b" in bm.sessions
+
+        # Use sessions again — should work
+        r = await c.arun(url=f"{srv}/dashboard", config=sess_a)
+        assert r.success
+        r = await c.arun(url=f"{srv}/dashboard", config=sess_b)
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_arun_many_with_session_open(srv):
+    """Session open while arun_many batch runs with recycle enabled.
+    Session survives the batch."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=5,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        sess = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="batch_guard", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess)
+        assert r.success
+
+        no_sess = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        urls = [_u(srv, i) for i in range(12)]
+        results = await c.arun_many(urls, config=no_sess)
+        assert all(r.success for r in results)
+
+        # Session still alive
+        assert "batch_guard" in bm.sessions
+
+
+@pytest.mark.asyncio
+async def test_rapid_recycle_stress(srv):
+    """Recycle threshold=2 with 20 sequential crawls → 10 recycle cycles.
+    Every crawl must succeed. Proves recycle is stable under rapid cycling."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        for i in range(20):
+            r = await c.arun(url=_u(srv, i % 100), config=run)
+            assert r.success, f"Page {i} failed during rapid recycle"
+
+
+@pytest.mark.asyncio
+async def test_rapid_recycle_concurrent(srv):
+    """Recycle threshold=3 with 12 concurrent crawls. Concurrency +
+    rapid recycling together."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(12)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 12
+
+
+# ===================================================================
+# SECTION G — Lock correctness under contention
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_context_lock_no_duplicate_contexts(srv):
+    """Fire 20 concurrent crawls with the same config on isolated context mode.
+    Despite concurrency, only 1 context should be created (all share the
+    same config signature)."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(20)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+
+        # All had the same config → only 1 context should exist
+        assert len(bm.contexts_by_config) == 1, (
+            f"Expected 1 context, got {len(bm.contexts_by_config)} — "
+            f"lock failed to prevent duplicate creation"
+        )
+
+
+@pytest.mark.asyncio
+async def test_page_lock_no_duplicate_pages_managed(srv):
+    """On managed browser (shared default context), concurrent crawls should
+    never get the same page. After all complete, pages_in_use should be empty."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(8)]
+        await asyncio.gather(*tasks)
+
+        # After all crawls complete, no pages should be marked in use
+        piu = bm._get_pages_in_use()
+        assert len(piu) == 0, (
+            f"After all crawls complete, {len(piu)} pages still marked in use"
+        )
+
+
+@pytest.mark.asyncio
+async def test_refcount_correctness_under_concurrency(srv):
+    """Fire 15 concurrent crawls with isolated context. After all complete,
+    all refcounts should be 0."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(15)]
+        await asyncio.gather(*tasks)
+
+        for sig, rc in bm._context_refcounts.items():
+            assert rc == 0, (
+                f"Refcount for context {sig[:8]}... is {rc}, expected 0 "
+                f"after all crawls complete"
+            )
+
+
+# ===================================================================
+# SECTION H — Close / cleanup correctness
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_close_cleans_up_standalone(srv):
+    """After closing standalone crawler, browser and playwright are None."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    c = AsyncWebCrawler(config=cfg)
+    await c.start()
+    bm = _bm(c)
+
+    r = await c.arun(url=_u(srv, 0), config=run)
+    assert r.success
+
+    await c.close()
+    assert bm.browser is None
+    assert bm.playwright is None
+
+
+@pytest.mark.asyncio
+async def test_close_cleans_up_managed(srv):
+    """After closing managed crawler, managed_browser is cleaned up."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    c = AsyncWebCrawler(config=cfg)
+    await c.start()
+    bm = _bm(c)
+
+    r = await c.arun(url=_u(srv, 0), config=run)
+    assert r.success
+
+    await c.close()
+    assert bm.browser is None
+    assert bm.managed_browser is None
+
+
+@pytest.mark.asyncio
+async def test_double_close_safe(srv):
+    """Calling close() twice should not raise."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+
+    c = AsyncWebCrawler(config=cfg)
+    await c.start()
+    r = await c.arun(url=_u(srv, 0), config=CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, verbose=False,
+    ))
+    assert r.success
+
+    await c.close()
+    # Second close should be safe
+    await c.close()
+
+
+# ===================================================================
+# SECTION I — Mixed modes: session + recycle + managed + concurrent
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_managed_isolated_session_recycle_concurrent(srv):
+    """The ultimate stress test: managed browser + isolated contexts +
+    sessions + recycle + concurrent crawls.
+
+    Flow:
+    1. Open session A
+    2. Fire 8 concurrent non-session crawls (threshold=5, but session blocks)
+    3. Kill session A
+    4. Fire 3 more non-session crawls to trigger recycle
+    5. Open session B on the fresh browser
+    6. Verify session B works
+    """
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+        max_pages_before_recycle=5,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+        no_sess = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+        # Step 1: open session
+        sess_a = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="ultimate_a", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess_a)
+        assert r.success
+
+        # Step 2: concurrent non-session crawls
+        tasks = [c.arun(url=_u(srv, i), config=no_sess) for i in range(8)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions in step 2: {excs[:3]}"
+
+        # Session blocks recycle
+        assert "ultimate_a" in bm.sessions
+
+        # Step 3: kill session
+        await c.crawler_strategy.kill_session("ultimate_a")
+
+        # Step 4: trigger recycle
+        for i in range(3):
+            r = await c.arun(url=_u(srv, 80 + i), config=no_sess)
+            assert r.success
+
+        await asyncio.sleep(0.5)
+
+        # Step 5: new session on fresh browser
+        sess_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="ultimate_b", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess_b)
+        assert r.success
+        assert "ultimate_b" in bm.sessions
+
+        # Step 6: verify it works
+        r = await c.arun(url=f"{srv}/dashboard", config=sess_b)
+        assert r.success
diff --git a/tests/async/test_browser_memory.py b/tests/async/test_browser_memory.py
new file mode 100644
index 0000000..cd1685d
--- /dev/null
+++ b/tests/async/test_browser_memory.py
@@ -0,0 +1,1169 @@
+"""
+Tests for browser memory management: memory_saving_mode, browser recycling,
+and CDP session leak fixes.
+
+These are integration tests that launch real browsers and crawl real pages.
+They verify:
+  1. memory_saving_mode Chrome flags are applied
+  2. Browser recycling fires at the right threshold and doesn't break crawling
+  3. Concurrent crawls survive a recycle boundary without errors
+  4. Recycling resets all internal tracking state cleanly
+  5. Memory doesn't grow unbounded over many pages
+  6. CDP session detach fix doesn't regress viewport adjustment
+"""
+
+import asyncio
+import os
+import time
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+import psutil
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+
+# ---------------------------------------------------------------------------
+# Local test server — avoids network flakiness
+# ---------------------------------------------------------------------------
+
+PAGES_HTML = {}
+for i in range(200):
+    PAGES_HTML[f"/page{i}"] = f"""
+Page {i}
+
+
      Test page {i}
      
+
      Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Paragraph {i} with enough text to exercise the content pipeline.
      
+Next
+"""
+
+
+class MemTestHandler(SimpleHTTPRequestHandler):
+    """Serves lightweight HTML pages for memory tests.
+
+    Also serves /login and /dashboard for multi-step session tests.
+    /login sets a cookie, /dashboard checks the cookie to prove session state.
+    """
+
+    def log_message(self, *args):
+        pass  # silent
+
+    def do_GET(self):
+        if self.path == "/login":
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.send_header("Set-Cookie", "auth_token=valid123; Path=/")
+            self.end_headers()
+            self.wfile.write(b"""
+Login
+
      Login Page
      You are now logged in.
      
+Go to dashboard""")
+            return
+
+        if self.path == "/dashboard":
+            cookie = self.headers.get("Cookie", "")
+            if "auth_token=valid123" in cookie:
+                body = "
      Dashboard
      Welcome, authenticated user!
      "
+            else:
+                body = "
      Dashboard
      NOT AUTHENTICATED
      "
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            self.wfile.write(
+                f"Dashboard"
+                f"{body}".encode()
+            )
+            return
+
+        if self.path == "/step1":
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            self.wfile.write(b"""
+Step 1
+
      Step 1
      First step complete
      """)
+            return
+
+        if self.path == "/step2":
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            self.wfile.write(b"""
+Step 2
+
      Step 2
      Second step complete
      """)
+            return
+
+        if self.path == "/step3":
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            self.wfile.write(b"""
+Step 3
+
      Step 3
      Third step complete
      """)
+            return
+
+        html = PAGES_HTML.get(self.path)
+        if html is None:
+            # Fallback for root and unknown paths
+            html = PAGES_HTML["/page0"]
+        self.send_response(200)
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(html.encode())
+
+
+class ReuseAddrHTTPServer(HTTPServer):
+    allow_reuse_address = True
+
+
+@pytest.fixture(scope="module")
+def test_server():
+    """Start a local HTTP server for the test module."""
+    server = ReuseAddrHTTPServer(("127.0.0.1", 0), MemTestHandler)
+    port = server.server_address[1]
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    yield f"http://127.0.0.1:{port}"
+    server.shutdown()
+
+
+def _url(base, i):
+    return f"{base}/page{i}"
+
+
+def _get_chromium_rss_mb():
+    """Sum RSS of all chromium/chrome child processes in MB."""
+    total = 0
+    for proc in psutil.process_iter(["name", "cmdline"]):
+        try:
+            name = (proc.info["name"] or "").lower()
+            cmdline = " ".join(proc.info["cmdline"] or []).lower()
+            if "chrom" in name or "chrom" in cmdline:
+                total += proc.memory_info().rss
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            pass
+    return total / (1024 * 1024)
+
+
+# ---------------------------------------------------------------------------
+# Helpers to reach into BrowserManager internals
+# ---------------------------------------------------------------------------
+
+def _bm(crawler: AsyncWebCrawler):
+    """Shortcut to get the BrowserManager from a crawler."""
+    return crawler.crawler_strategy.browser_manager
+
+
+# ===========================================================================
+# Test 1: memory_saving_mode flag propagation
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_memory_saving_flags_applied(test_server):
+    """Verify --aggressive-cache-discard and --js-flags are in the launch args
+    when memory_saving_mode=True, and absent when False."""
+    config_on = BrowserConfig(
+        headless=True,
+        verbose=False,
+        memory_saving_mode=True,
+    )
+    config_off = BrowserConfig(
+        headless=True,
+        verbose=False,
+        memory_saving_mode=False,
+    )
+
+    async with AsyncWebCrawler(config=config_on) as crawler:
+        bm = _bm(crawler)
+        browser_args = bm._build_browser_args()
+        # _build_browser_args returns a dict with an "args" key
+        args_list = browser_args.get("args", browser_args) if isinstance(browser_args, dict) else browser_args
+        assert "--aggressive-cache-discard" in args_list, (
+            "memory_saving_mode=True should add --aggressive-cache-discard"
+        )
+        assert any("max-old-space-size" in a for a in args_list), (
+            "memory_saving_mode=True should add V8 heap cap"
+        )
+        # Always-on flags should be present regardless
+        assert any("OptimizationHints" in a for a in args_list)
+
+    async with AsyncWebCrawler(config=config_off) as crawler:
+        bm = _bm(crawler)
+        browser_args = bm._build_browser_args()
+        args_list = browser_args.get("args", browser_args) if isinstance(browser_args, dict) else browser_args
+        assert "--aggressive-cache-discard" not in args_list, (
+            "memory_saving_mode=False should NOT add --aggressive-cache-discard"
+        )
+        assert not any("max-old-space-size" in a for a in args_list), (
+            "memory_saving_mode=False should NOT add V8 heap cap"
+        )
+        # Always-on flags should still be there
+        assert any("OptimizationHints" in a for a in args_list)
+
+
+# ===========================================================================
+# Test 2: Always-on flags present in both code paths
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_always_on_flags_present(test_server):
+    """The 3 always-on memory flags should appear in _build_browser_args
+    even with default BrowserConfig."""
+    config = BrowserConfig(headless=True, verbose=False)
+    async with AsyncWebCrawler(config=config) as crawler:
+        browser_args = _bm(crawler)._build_browser_args()
+        args_list = browser_args.get("args", browser_args) if isinstance(browser_args, dict) else browser_args
+        assert any("disable-component-update" in a for a in args_list)
+        assert any("disable-domain-reliability" in a for a in args_list)
+        assert any("OptimizationHints" in a for a in args_list)
+
+
+# ===========================================================================
+# Test 3: Basic recycling — counter increments, recycle fires, crawls resume
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_fires_at_threshold(test_server):
+    """Set max_pages_before_recycle=5, crawl 8 pages sequentially.
+    Verify the counter resets after recycle and all crawls succeed."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        memory_saving_mode=True,
+        max_pages_before_recycle=5,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+        assert bm._pages_served == 0
+
+        results = []
+        for i in range(8):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            results.append(r)
+
+        # All 8 crawls should succeed — recycle happened transparently
+        assert len(results) == 8
+        assert all(r.success for r in results), (
+            f"Failed crawls: {[i for i, r in enumerate(results) if not r.success]}"
+        )
+
+        # After 8 pages with threshold=5, recycle happened once (at page 5).
+        # Pages 6,7,8 served after recycle → counter should be 3.
+        assert bm._pages_served == 3, (
+            f"Expected 3 pages after recycle, got {bm._pages_served}"
+        )
+
+
+# ===========================================================================
+# Test 4: Recycling resets all tracking state
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_clears_tracking_state(test_server):
+    """After a recycle, internal dicts should be clean."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        # Crawl 3 pages → triggers recycle
+        for i in range(3):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            assert r.success
+
+        # Give recycle a moment to complete (it fires in release_page_with_context)
+        await asyncio.sleep(0.5)
+
+        # Recycle should have reset these
+        assert bm._pages_served == 0, f"Counter not reset: {bm._pages_served}"
+        assert sum(bm._context_refcounts.values()) == 0, (
+            f"Refcounts not zero after recycle: {bm._context_refcounts}"
+        )
+
+        # Crawl one more page to prove browser is alive
+        r = await crawler.arun(url=_url(test_server, 99), config=run_config)
+        assert r.success
+        assert bm._pages_served == 1
+
+
+# ===========================================================================
+# Test 5: Concurrent crawls across a recycle boundary
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_concurrent_crawls_across_recycle(test_server):
+    """Launch concurrent crawls that straddle the recycle threshold.
+    Recycling should wait for in-flight crawls to finish, not crash them."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=5,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        # Fire 10 concurrent crawls with threshold=5
+        urls = [_url(test_server, i) for i in range(10)]
+        tasks = [crawler.arun(url=u, config=run_config) for u in urls]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        exceptions = [r for r in results if isinstance(r, Exception)]
+        assert len(exceptions) == 0, (
+            f"Got {len(exceptions)} exceptions during concurrent recycle: "
+            f"{exceptions[:3]}"
+        )
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 10, (
+            f"Only {len(successes)}/10 crawls succeeded"
+        )
+
+
+# ===========================================================================
+# Test 6: Recycle with sessions — sessions cleared, new session works after
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_blocked_by_active_session(test_server):
+    """An active session holds a context refcount, so the browser should NOT
+    recycle while the session is open — even if pages_served >= threshold.
+    This proves recycling is safe around sessions."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=3,
+    )
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+        run_no_session = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+        # Crawl 2 non-session pages (released immediately)
+        for i in range(2):
+            r = await crawler.arun(url=_url(test_server, i), config=run_no_session)
+            assert r.success
+
+        # Create a named session on page 3 — hits the threshold
+        run_with_session = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="test_session",
+            verbose=False,
+        )
+        r = await crawler.arun(url=_url(test_server, 2), config=run_with_session)
+        assert r.success
+        assert "test_session" in bm.sessions
+
+        # We've hit 3 pages (the threshold), but the session holds a refcount
+        # so recycle must NOT fire
+        assert bm._pages_served == 3
+        assert not bm._recycling, (
+            "Recycle should not fire while a session holds a refcount"
+        )
+
+        # Browser should still be alive — use the session again
+        r = await crawler.arun(url=_url(test_server, 50), config=run_with_session)
+        assert r.success, "Session should still work even past recycle threshold"
+
+        # Session reuses the same page, so counter stays at 3
+        # (only get_page increments it, and session reuse skips get_page)
+        assert bm._pages_served >= 3
+        assert not bm._recycling
+
+
+@pytest.mark.asyncio
+async def test_sessions_cleared_by_recycle(test_server):
+    """After a recycle, the sessions dict is empty and new sessions work."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        # Crawl 3 non-session pages → recycle fires (all refcounts 0)
+        for i in range(3):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            assert r.success
+
+        await asyncio.sleep(0.5)
+
+        # Sessions dict cleared by recycle
+        assert len(bm.sessions) == 0, (
+            f"Sessions should be empty after recycle, got {list(bm.sessions.keys())}"
+        )
+
+        # New session should work on the fresh browser
+        run_with_session = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="post_recycle_session",
+            verbose=False,
+        )
+        r = await crawler.arun(url=_url(test_server, 99), config=run_with_session)
+        assert r.success
+        assert "post_recycle_session" in bm.sessions
+
+
+# ===========================================================================
+# Test 7: Multiple recycle cycles — browser survives repeated recycling
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_multiple_recycle_cycles(test_server):
+    """Recycle the browser 4 times (threshold=5, crawl 22 pages).
+    Every single crawl must succeed."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=5,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+        all_results = []
+
+        for i in range(22):
+            r = await crawler.arun(url=_url(test_server, i % 200), config=run_config)
+            all_results.append(r)
+
+        assert all(r.success for r in all_results), (
+            f"Failed at pages: "
+            f"{[i for i, r in enumerate(all_results) if not r.success]}"
+        )
+        # 22 pages, threshold 5 → recycles at 5, 10, 15, 20 → 4 recycles
+        # After last recycle at page 20, pages 21,22 served → counter = 2
+        assert bm._pages_served == 2
+
+
+# ===========================================================================
+# Test 8: Recycling disabled by default (max_pages_before_recycle=0)
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_disabled_by_default(test_server):
+    """With default config (max_pages_before_recycle=0), no recycling happens
+    no matter how many pages are crawled."""
+    config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        for i in range(10):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            assert r.success
+
+        # Counter increments but never resets
+        assert bm._pages_served == 10
+        assert not bm._recycling
+
+
+# ===========================================================================
+# Test 9: _recycle_done event blocks get_page during recycle
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_event_blocks_new_pages(test_server):
+    """Simulate a recycle by manually clearing the event, then verify that
+    get_page blocks until the event is set."""
+    config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        # Manually block the gate
+        bm._recycle_done.clear()
+
+        got_page = False
+
+        async def try_get_page():
+            nonlocal got_page
+            r = await crawler.arun(url=_url(test_server, 0), config=run_config)
+            got_page = r.success
+
+        task = asyncio.create_task(try_get_page())
+
+        # Wait a bit — the crawl should be blocked
+        await asyncio.sleep(0.5)
+        assert not got_page, "get_page should block while _recycle_done is cleared"
+
+        # Release the gate
+        bm._recycle_done.set()
+        await asyncio.wait_for(task, timeout=15.0)
+        assert got_page, "Crawl should succeed after recycle_done is set"
+
+
+# ===========================================================================
+# Test 10: BrowserConfig serialization round-trip
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_config_serialization_roundtrip():
+    """memory_saving_mode and max_pages_before_recycle survive
+    to_dict → from_kwargs → clone round-trips."""
+    original = BrowserConfig(
+        headless=True,
+        memory_saving_mode=True,
+        max_pages_before_recycle=500,
+    )
+
+    # to_dict → from_kwargs
+    d = original.to_dict()
+    assert d["memory_saving_mode"] is True
+    assert d["max_pages_before_recycle"] == 500
+
+    restored = BrowserConfig.from_kwargs(d)
+    assert restored.memory_saving_mode is True
+    assert restored.max_pages_before_recycle == 500
+
+    # clone with override
+    cloned = original.clone(max_pages_before_recycle=1000)
+    assert cloned.memory_saving_mode is True  # inherited
+    assert cloned.max_pages_before_recycle == 1000  # overridden
+
+    # dump / load
+    dumped = original.dump()
+    loaded = BrowserConfig.load(dumped)
+    assert loaded.memory_saving_mode is True
+    assert loaded.max_pages_before_recycle == 500
+
+
+# ===========================================================================
+# Test 11: Memory stays bounded over many pages with recycling
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_memory_bounded_with_recycling(test_server):
+    """Crawl 40 pages with recycling every 10. Measure RSS at page 10
+    (just after first recycle) and at page 40. Memory should not grow
+    significantly — the recycle should keep it bounded.
+
+    This is the core proof that recycling controls memory growth.
+    Without recycling, Chromium RSS grows ~2-5 MB per page.
+    With recycling, it should stay roughly flat."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        memory_saving_mode=True,
+        max_pages_before_recycle=10,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        rss_samples = []
+
+        for i in range(40):
+            r = await crawler.arun(url=_url(test_server, i % 200), config=run_config)
+            assert r.success, f"Page {i} failed"
+
+            # Sample after each recycle boundary + a few extra
+            if (i + 1) % 10 == 0:
+                await asyncio.sleep(0.3)  # let recycle finish
+                rss_samples.append(_get_chromium_rss_mb())
+
+        # We should have 4 samples (at pages 10, 20, 30, 40)
+        assert len(rss_samples) == 4
+
+        # The key assertion: RSS at page 40 should not be dramatically larger
+        # than at page 10. Allow 50% growth as tolerance for GC timing etc.
+        # Without recycling, we'd expect 60-150 MB growth over 30 extra pages.
+        if rss_samples[0] > 0:  # guard against measurement issues
+            growth_ratio = rss_samples[-1] / rss_samples[0]
+            assert growth_ratio < 2.0, (
+                f"Memory grew {growth_ratio:.1f}x from {rss_samples[0]:.0f}MB "
+                f"to {rss_samples[-1]:.0f}MB over 30 pages with recycling. "
+                f"All samples: {[f'{s:.0f}' for s in rss_samples]} MB"
+            )
+
+
+# ===========================================================================
+# Test 12: Memory grows WITHOUT recycling (control test)
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_memory_grows_without_recycling(test_server):
+    """Control test: crawl 30 pages WITHOUT recycling and observe that
+    chromium RSS is higher at the end than at the start.
+    This proves that recycling is what keeps memory bounded."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        memory_saving_mode=False,
+        max_pages_before_recycle=0,  # disabled
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        # Warm up — let initial browser memory stabilize
+        for i in range(3):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            assert r.success
+        await asyncio.sleep(0.3)
+        rss_start = _get_chromium_rss_mb()
+
+        # Crawl 30 more pages
+        for i in range(3, 33):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            assert r.success
+
+        await asyncio.sleep(0.3)
+        rss_end = _get_chromium_rss_mb()
+
+        # RSS should be at least somewhat higher (chromium leaks)
+        # We just need this to not be 0 — proving our measurement works
+        if rss_start > 0:
+            print(
+                f"\n[CONTROL] RSS without recycling: "
+                f"{rss_start:.0f}MB → {rss_end:.0f}MB "
+                f"(+{rss_end - rss_start:.0f}MB over 30 pages)"
+            )
+
+
+# ===========================================================================
+# Test 13: Viewport adjustment doesn't leak CDP sessions
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_viewport_adjustment_no_cdp_leak(test_server):
+    """Crawl several pages that trigger viewport adjustment (scan_full_page).
+    If CDP sessions leak, Chromium's DevTools session count grows and
+    eventually causes slowdowns. We just verify all crawls succeed and
+    the browser stays healthy."""
+    config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        scan_full_page=True,  # triggers fit_to_viewport_adjustment → CDP session
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        for i in range(15):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            assert r.success, f"Page {i} failed with scan_full_page"
+
+
+# ===========================================================================
+# Test 14: Recycle under concurrent load with arun_many
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_with_arun_many(test_server):
+    """Use arun_many to crawl a batch that exceeds the recycle threshold.
+    This tests the dispatcher + recycling interaction."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=5,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        urls = [_url(test_server, i) for i in range(12)]
+        results = await crawler.arun_many(urls, config=run_config)
+
+        successes = [r for r in results if r.success]
+        assert len(successes) == 12, (
+            f"Only {len(successes)}/12 succeeded with arun_many + recycling"
+        )
+
+
+# ===========================================================================
+# Test 15: _global_pages_in_use cleaned after recycle
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_global_pages_in_use_cleared(test_server):
+    """After a recycle, the _global_pages_in_use set for this browser's
+    endpoint should be empty (old pages are dead)."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        for i in range(3):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            assert r.success
+
+        await asyncio.sleep(0.5)
+
+        # After recycle, pages_in_use for old endpoint should be empty
+        from crawl4ai.browser_manager import BrowserManager
+        if bm._browser_endpoint_key:
+            piu = BrowserManager._global_pages_in_use.get(
+                bm._browser_endpoint_key, set()
+            )
+            assert len(piu) == 0, (
+                f"_global_pages_in_use should be empty after recycle, "
+                f"has {len(piu)} stale entries"
+            )
+
+
+# ===========================================================================
+# Test 16: Content integrity across recycle — page content is correct
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_content_integrity_across_recycle(test_server):
+    """Verify that pages crawled AFTER a recycle return correct content,
+    not stale data from before the recycle."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        # Crawl pages 0,1,2 → triggers recycle
+        for i in range(3):
+            r = await crawler.arun(url=_url(test_server, i), config=run_config)
+            assert r.success
+
+        await asyncio.sleep(0.5)
+
+        # Crawl page 150 after recycle — content should match page 150
+        r = await crawler.arun(url=_url(test_server, 150), config=run_config)
+        assert r.success
+        assert "Test page 150" in r.html, (
+            "Content after recycle should be from the correct page"
+        )
+        assert "Paragraph 150" in r.html
+
+
+# ===========================================================================
+# SESSION + RECYCLE INTERACTION TESTS
+# ===========================================================================
+
+
+# ===========================================================================
+# Test 17: Multi-step session crawl — login → dashboard with cookie
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_multistep_session_login_flow(test_server):
+    """Simulate login → dashboard multi-step crawl using session_id.
+    The session preserves cookies, so dashboard should see authenticated state.
+    No recycling involved — baseline session behavior."""
+    config = BrowserConfig(headless=True, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        session_cfg = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="login_flow",
+            verbose=False,
+        )
+
+        # Step 1: login — sets cookie
+        r = await crawler.arun(url=f"{test_server}/login", config=session_cfg)
+        assert r.success
+        assert "Login Page" in r.html
+
+        # Step 2: dashboard — cookie should carry over via session
+        r = await crawler.arun(url=f"{test_server}/dashboard", config=session_cfg)
+        assert r.success
+        assert "Welcome, authenticated user" in r.html, (
+            "Session should carry cookies from login to dashboard"
+        )
+
+
+# ===========================================================================
+# Test 18: Multi-step session survives non-session crawls past threshold
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_session_survives_threshold_with_interleaved_crawls(test_server):
+    """Open a session, then do many non-session crawls that push
+    pages_served past the recycle threshold. The session should prevent
+    recycle from firing (refcount > 0). Then continue using the session
+    and it should still work."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=5,
+    )
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        # Start a session — step 1
+        session_cfg = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="persistent_session",
+            verbose=False,
+        )
+        r = await crawler.arun(url=f"{test_server}/login", config=session_cfg)
+        assert r.success
+        assert "persistent_session" in bm.sessions
+
+        # Fire 8 non-session crawls — pushes pages_served to 9
+        # (1 from session + 8 = 9, well past threshold of 5)
+        no_session = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        for i in range(8):
+            r = await crawler.arun(url=_url(test_server, i), config=no_session)
+            assert r.success, f"Non-session crawl {i} failed"
+
+        # Recycle should NOT have fired — session holds refcount
+        assert bm._pages_served == 9, (
+            f"Expected 9 pages served, got {bm._pages_served}"
+        )
+        assert not bm._recycling
+        assert "persistent_session" in bm.sessions, (
+            "Session should still exist — recycle blocked by refcount"
+        )
+
+        # Session should still work — navigate to dashboard with cookies
+        r = await crawler.arun(url=f"{test_server}/dashboard", config=session_cfg)
+        assert r.success
+        assert "Welcome, authenticated user" in r.html, (
+            "Session cookies should still work after interleaved non-session crawls"
+        )
+
+
+# ===========================================================================
+# Test 19: 3-step session flow with recycle threshold — recycle blocked
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_three_step_session_blocks_recycle(test_server):
+    """3-step session (step1 → step2 → step3) with low threshold.
+    The session's refcount should block recycle for the entire flow."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=2,  # very low threshold
+    )
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        session_cfg = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="multistep",
+            verbose=False,
+        )
+
+        # Step 1
+        r = await crawler.arun(url=f"{test_server}/step1", config=session_cfg)
+        assert r.success
+        assert "Step 1" in r.html
+
+        # Step 2 — pages_served is still 1 (session reuse doesn't increment)
+        # but even if it did, refcount blocks recycle
+        r = await crawler.arun(url=f"{test_server}/step2", config=session_cfg)
+        assert r.success
+        assert "Step 2" in r.html
+
+        # Step 3
+        r = await crawler.arun(url=f"{test_server}/step3", config=session_cfg)
+        assert r.success
+        assert "Step 3" in r.html
+
+        # Session page reuse doesn't increment counter (only get_page does)
+        # Initial creation = 1 page, subsequent calls reuse it
+        assert bm._pages_served == 1
+        assert not bm._recycling
+        assert "multistep" in bm.sessions
+
+
+# ===========================================================================
+# Test 20: Two concurrent sessions — both survive past threshold
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_two_concurrent_sessions_block_recycle(test_server):
+    """Two sessions open at the same time, with non-session crawls interleaved.
+    Both sessions should prevent recycle and remain functional."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=3,
+    )
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        session_a = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="sess_a", verbose=False,
+        )
+        session_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="sess_b", verbose=False,
+        )
+        no_session = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+        # Open session A
+        r = await crawler.arun(url=f"{test_server}/login", config=session_a)
+        assert r.success
+
+        # Open session B
+        r = await crawler.arun(url=f"{test_server}/step1", config=session_b)
+        assert r.success
+
+        # 5 non-session crawls — pages_served goes to 7 (2 sessions + 5)
+        for i in range(5):
+            r = await crawler.arun(url=_url(test_server, i), config=no_session)
+            assert r.success
+
+        # Both sessions hold refcounts → recycle blocked
+        assert not bm._recycling
+        assert "sess_a" in bm.sessions
+        assert "sess_b" in bm.sessions
+
+        # Both sessions still work
+        r = await crawler.arun(url=f"{test_server}/dashboard", config=session_a)
+        assert r.success
+        assert "Welcome, authenticated user" in r.html
+
+        r = await crawler.arun(url=f"{test_server}/step2", config=session_b)
+        assert r.success
+        assert "Step 2" in r.html
+
+
+# ===========================================================================
+# Test 21: Session killed, then recycle fires on next non-session crawl
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_fires_after_session_killed(test_server):
+    """Session blocks recycle. After session is killed (refcount drops to 0),
+    the next non-session crawl that pushes past threshold triggers recycle."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=3,
+    )
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+        no_session = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+        # Open a session (1 page)
+        session_cfg = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="temp_sess", verbose=False,
+        )
+        r = await crawler.arun(url=f"{test_server}/step1", config=session_cfg)
+        assert r.success
+
+        # 3 non-session crawls (4 pages total, threshold=3, but session blocks)
+        for i in range(3):
+            r = await crawler.arun(url=_url(test_server, i), config=no_session)
+            assert r.success
+
+        pages_before_kill = bm._pages_served
+        assert pages_before_kill == 4
+        assert not bm._recycling
+
+        # Kill the session — refcount drops to 0
+        await crawler.crawler_strategy.kill_session("temp_sess")
+        assert "temp_sess" not in bm.sessions
+
+        # One more crawl — should trigger recycle (pages_served=5 >= 3, refcounts=0)
+        r = await crawler.arun(url=_url(test_server, 99), config=no_session)
+        assert r.success
+
+        await asyncio.sleep(0.5)
+
+        # Recycle should have fired — counter reset
+        assert bm._pages_served < pages_before_kill, (
+            f"Expected counter reset after recycle, got {bm._pages_served}"
+        )
+
+
+# ===========================================================================
+# Test 22: Concurrent session crawls — same session from multiple tasks
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_concurrent_same_session_crawls(test_server):
+    """Multiple asyncio tasks using the same session_id concurrently.
+    The session page should be shared safely between them."""
+    config = BrowserConfig(headless=True, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        session_cfg = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="shared_session",
+            verbose=False,
+        )
+
+        # Login first to set cookie
+        r = await crawler.arun(url=f"{test_server}/login", config=session_cfg)
+        assert r.success
+
+        # Fire 5 concurrent crawls on the same session
+        urls = [f"{test_server}/page{i}" for i in range(5)]
+        tasks = [
+            crawler.arun(url=u, config=session_cfg) for u in urls
+        ]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        exceptions = [r for r in results if isinstance(r, Exception)]
+        # Some may fail due to navigation conflicts (same page, concurrent goto),
+        # but there should be no crashes or browser death
+        assert len(exceptions) == 0, (
+            f"Exceptions in concurrent same-session crawls: {exceptions[:3]}"
+        )
+
+
+# ===========================================================================
+# Test 23: Session + recycling — session killed mid-batch, recycle fires,
+#           new session works after
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_session_lifecycle_across_recycle(test_server):
+    """Full lifecycle: create session → use it → kill it → recycle fires →
+    create new session → use it. End-to-end proof that recycling is safe."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=4,
+    )
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+        no_session = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+        # Phase 1: create and use a session
+        sess_v1 = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="lifecycle_sess", verbose=False,
+        )
+        r = await crawler.arun(url=f"{test_server}/login", config=sess_v1)
+        assert r.success
+
+        r = await crawler.arun(url=f"{test_server}/dashboard", config=sess_v1)
+        assert r.success
+        assert "Welcome, authenticated user" in r.html
+
+        # Phase 2: kill session
+        await crawler.crawler_strategy.kill_session("lifecycle_sess")
+
+        # Phase 3: push past threshold with non-session crawls
+        for i in range(5):
+            r = await crawler.arun(url=_url(test_server, i), config=no_session)
+            assert r.success
+
+        await asyncio.sleep(0.5)
+
+        # Recycle should have happened (session killed, refcount=0)
+        assert bm._pages_served < 6, (
+            f"Expected reset after recycle, got {bm._pages_served}"
+        )
+
+        # Phase 4: new session on the fresh browser
+        sess_v2 = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="lifecycle_sess_v2", verbose=False,
+        )
+        r = await crawler.arun(url=f"{test_server}/login", config=sess_v2)
+        assert r.success
+        assert "lifecycle_sess_v2" in bm.sessions
+
+        r = await crawler.arun(url=f"{test_server}/dashboard", config=sess_v2)
+        assert r.success
+        assert "Welcome, authenticated user" in r.html, (
+            "New session after recycle should work with cookies"
+        )
+
+
+# ===========================================================================
+# Test 24: Parallel sessions + non-session crawls with arun_many
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_session_with_arun_many_interleaved(test_server):
+    """Open a session, then fire arun_many for non-session URLs.
+    The session should survive the batch and remain usable after."""
+    config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        max_pages_before_recycle=10,
+    )
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        # Open session
+        session_cfg = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="batch_sess", verbose=False,
+        )
+        r = await crawler.arun(url=f"{test_server}/login", config=session_cfg)
+        assert r.success
+
+        # Batch of non-session crawls
+        no_session = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        urls = [_url(test_server, i) for i in range(8)]
+        results = await crawler.arun_many(urls, config=no_session)
+        assert all(r.success for r in results), "All batch crawls should succeed"
+
+        # Session still alive
+        assert "batch_sess" in bm.sessions
+        r = await crawler.arun(url=f"{test_server}/dashboard", config=session_cfg)
+        assert r.success
+        assert "Welcome, authenticated user" in r.html
+
+
+# ===========================================================================
+# Test 25: Session refcount tracking correctness
+# ===========================================================================
+
+@pytest.mark.asyncio
+async def test_session_refcount_stays_at_one(test_server):
+    """Verify that a session holds exactly 1 refcount throughout its
+    lifecycle, regardless of how many times it's reused."""
+    config = BrowserConfig(headless=True, verbose=False)
+
+    async with AsyncWebCrawler(config=config) as crawler:
+        bm = _bm(crawler)
+
+        session_cfg = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="refcount_test", verbose=False,
+        )
+
+        # Create session
+        r = await crawler.arun(url=f"{test_server}/step1", config=session_cfg)
+        assert r.success
+
+        # Find the session's context signature
+        _, page, _ = bm.sessions["refcount_test"]
+        sig = bm._page_to_sig.get(page)
+        if sig:
+            refcount = bm._context_refcounts.get(sig, 0)
+            assert refcount == 1, (
+                f"Session should hold exactly 1 refcount, got {refcount}"
+            )
+
+        # Reuse session multiple times — refcount should stay at 1
+        for url in ["/step2", "/step3", "/dashboard"]:
+            r = await crawler.arun(url=f"{test_server}{url}", config=session_cfg)
+            assert r.success
+
+            if sig:
+                refcount = bm._context_refcounts.get(sig, 0)
+                assert refcount == 1, (
+                    f"After reuse, refcount should still be 1, got {refcount}"
+                )
+
+        # Kill session — refcount should drop to 0
+        await crawler.crawler_strategy.kill_session("refcount_test")
+        if sig:
+            refcount = bm._context_refcounts.get(sig, 0)
+            assert refcount == 0, (
+                f"After kill, refcount should be 0, got {refcount}"
+            )
diff --git a/tests/async/test_browser_recycle_v2.py b/tests/async/test_browser_recycle_v2.py
new file mode 100644
index 0000000..eb9bb33
--- /dev/null
+++ b/tests/async/test_browser_recycle_v2.py
@@ -0,0 +1,386 @@
+"""
+Tests for version-based browser recycling.
+
+The new recycle approach:
+1. When pages_served hits threshold, bump _browser_version
+2. Old signatures go to _pending_cleanup
+3. New requests get new contexts (different version = different signature)
+4. When old context's refcount hits 0, it gets cleaned up
+5. No blocking — old and new browsers coexist during transition
+
+These tests use small thresholds (3-5 pages) to verify the mechanics.
+"""
+
+import asyncio
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+
+# ---------------------------------------------------------------------------
+# Local test server
+# ---------------------------------------------------------------------------
+
+PAGES = {}
+for i in range(100):
+    PAGES[f"/page{i}"] = (
+        f"Page {i}"
+        f"
      Page {i}
      Content for page {i}.
      "
+    ).encode()
+
+
+class Handler(SimpleHTTPRequestHandler):
+    def log_message(self, *a):
+        pass
+
+    def do_GET(self):
+        body = PAGES.get(self.path, PAGES["/page0"])
+        self.send_response(200)
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(body)
+
+
+class _Server(HTTPServer):
+    allow_reuse_address = True
+
+
+@pytest.fixture(scope="module")
+def srv():
+    s = _Server(("127.0.0.1", 0), Handler)
+    port = s.server_address[1]
+    t = threading.Thread(target=s.serve_forever, daemon=True)
+    t.start()
+    yield f"http://127.0.0.1:{port}"
+    s.shutdown()
+
+
+def _u(base, i):
+    return f"{base}/page{i}"
+
+
+def _bm(c):
+    return c.crawler_strategy.browser_manager
+
+
+# ===================================================================
+# SECTION A — Version bump mechanics
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_version_bump_on_threshold(srv):
+    """Browser version should bump when threshold is reached."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        assert bm._browser_version == 1
+
+        # Crawl 2 pages — no bump yet
+        for i in range(2):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        assert bm._browser_version == 1, "Version should still be 1 after 2 pages"
+        assert bm._pages_served == 2
+
+        # 3rd page hits threshold (3) and triggers bump AFTER the page is served
+        r = await c.arun(url=_u(srv, 2), config=run)
+        assert r.success
+        assert bm._browser_version == 2, "Version should bump after 3rd page"
+        assert bm._pages_served == 0, "Counter resets after bump"
+
+        # 4th page is first page of version 2
+        r = await c.arun(url=_u(srv, 3), config=run)
+        assert r.success
+        assert bm._pages_served == 1
+
+
+@pytest.mark.asyncio
+async def test_signature_changes_after_version_bump(srv):
+    """Same CrawlerRunConfig should produce different signatures after version bump."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Get signature before bump
+        sig_v1 = bm._make_config_signature(run)
+
+        # Crawl 2 pages
+        for i in range(2):
+            await c.arun(url=_u(srv, i), config=run)
+
+        # 3rd request triggers bump
+        await c.arun(url=_u(srv, 2), config=run)
+
+        # Signature should be different now
+        sig_v2 = bm._make_config_signature(run)
+        assert sig_v1 != sig_v2, "Signature should change after version bump"
+
+
+@pytest.mark.asyncio
+async def test_no_version_bump_when_disabled(srv):
+    """Version should stay at 1 when max_pages_before_recycle=0."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=0,  # Disabled
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(20):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        assert bm._browser_version == 1, "Version should not bump when disabled"
+        assert bm._pages_served == 20
+
+
+# ===================================================================
+# SECTION B — Pending cleanup mechanics
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_old_signature_goes_to_pending_cleanup(srv):
+    """Version bump works and old contexts get cleaned up."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Crawl 2 pages — creates signature for version 1, bumps on 2nd
+        for i in range(2):
+            await c.arun(url=_u(srv, i), config=run)
+
+        # After 2 pages with threshold=2, version should have bumped
+        assert bm._browser_version == 2
+
+        # Since sequential crawls release pages immediately (refcount=0),
+        # old contexts get cleaned up right away. Pending cleanup should be empty.
+        # This is correct behavior — cleanup is eager when possible.
+        assert len(bm._pending_cleanup) == 0
+
+
+@pytest.mark.asyncio
+async def test_cleanup_happens_when_refcount_hits_zero(srv):
+    """Old context should be closed when its refcount drops to 0."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Sequential crawls: each page is released before next request
+        # So refcount is always 0 between requests, and cleanup happens immediately
+        for i in range(10):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # Should have bumped twice (at 3 and 6) with version now at 3
+        # But since refcount=0 immediately, pending_cleanup should be empty
+        assert len(bm._pending_cleanup) == 0, "All old contexts should be cleaned up"
+
+
+# ===================================================================
+# SECTION C — Concurrent crawls with recycling
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_concurrent_crawls_dont_block_on_recycle(srv):
+    """Concurrent crawls should not block — old browser drains while new one serves."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=5,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Launch 20 concurrent crawls
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(20)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # All should succeed — no blocking, no errors
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 20, f"Only {len(successes)} succeeded"
+
+        # Version should have bumped multiple times
+        assert bm._browser_version >= 2, "Should have recycled at least once"
+
+
+@pytest.mark.asyncio
+async def test_high_concurrency_with_small_threshold(srv):
+    """Stress test: 50 concurrent crawls with threshold=3."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # 50 concurrent crawls with threshold of 3 — many version bumps
+        tasks = [c.arun(url=_u(srv, i % 100), config=run) for i in range(50)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 50
+
+
+# ===================================================================
+# SECTION D — Safety cap (max pending browsers)
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_safety_cap_limits_pending_browsers(srv):
+    """Should not exceed _max_pending_browsers old browsers draining."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+        bm._max_pending_browsers = 2  # Lower cap for testing
+
+        # Run enough crawls to potentially exceed the cap
+        for i in range(15):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # Pending cleanup should never have exceeded the cap
+        # (We can't directly test this during execution, but if it works without
+        # deadlock/timeout, the cap logic is functioning)
+        assert len(bm._pending_cleanup) <= bm._max_pending_browsers
+
+
+# ===================================================================
+# SECTION E — Managed browser mode
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_managed_browser_recycle(srv):
+    """Recycling should work with managed browser mode."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(10):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+        # Version should have bumped
+        assert bm._browser_version >= 2
+
+
+@pytest.mark.asyncio
+async def test_managed_browser_isolated_context_recycle(srv):
+    """Recycling with managed browser + isolated contexts."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(10):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+        assert bm._browser_version >= 2
+
+
+# ===================================================================
+# SECTION F — Edge cases
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_threshold_of_one(srv):
+    """Edge case: threshold=1 means version bump after every page."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=1,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(5):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # With threshold=1, each page triggers a bump after being served:
+        # Page 0: served, counter=1 >= 1, bump -> version=2, counter=0
+        # Page 1: served, counter=1 >= 1, bump -> version=3, counter=0
+        # ... etc.
+        # After 5 pages, should have bumped 5 times
+        assert bm._browser_version == 6  # Started at 1, bumped 5 times
+
+
+@pytest.mark.asyncio
+async def test_different_configs_get_separate_cleanup_tracking(srv):
+    """Different CrawlerRunConfigs should track separately in pending cleanup."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        run_a = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        run_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, verbose=False,
+            override_navigator=True,  # Different config
+        )
+
+        # Alternate between configs
+        for i in range(6):
+            cfg_to_use = run_a if i % 2 == 0 else run_b
+            r = await c.arun(url=_u(srv, i), config=cfg_to_use)
+            assert r.success
+
+        # Both configs should work fine
+        assert bm._browser_version >= 2
diff --git a/tests/async/test_caching.py b/tests/async/test_caching.py
new file mode 100644
index 0000000..d7f6efb
--- /dev/null
+++ b/tests/async/test_caching.py
@@ -0,0 +1,87 @@
+import os
+import sys
+import pytest
+import asyncio
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_caching():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # First crawl (should not use cache)
+        start_time = asyncio.get_event_loop().time()
+        result1 = await crawler.arun(url=url, bypass_cache=True)
+        end_time = asyncio.get_event_loop().time()
+        time_taken1 = end_time - start_time
+
+        assert result1.success
+
+        # Second crawl (should use cache)
+        start_time = asyncio.get_event_loop().time()
+        result2 = await crawler.arun(url=url, bypass_cache=False)
+        end_time = asyncio.get_event_loop().time()
+        time_taken2 = end_time - start_time
+
+        assert result2.success
+        assert time_taken2 < time_taken1  # Cached result should be faster
+
+
+@pytest.mark.asyncio
+async def test_bypass_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # First crawl
+        result1 = await crawler.arun(url=url, bypass_cache=False)
+        assert result1.success
+
+        # Second crawl with bypass_cache=True
+        result2 = await crawler.arun(url=url, bypass_cache=True)
+        assert result2.success
+
+        # Content should be different (or at least, not guaranteed to be the same)
+        assert result1.html != result2.html or result1.markdown != result2.markdown
+
+
+@pytest.mark.asyncio
+async def test_clear_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # Crawl and cache
+        await crawler.arun(url=url, bypass_cache=False)
+
+        # Clear cache
+        await crawler.aclear_cache()
+
+        # Check cache size
+        cache_size = await crawler.aget_cache_size()
+        assert cache_size == 0
+
+
+@pytest.mark.asyncio
+async def test_flush_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # Crawl and cache
+        await crawler.arun(url=url, bypass_cache=False)
+
+        # Flush cache
+        await crawler.aflush_cache()
+
+        # Check cache size
+        cache_size = await crawler.aget_cache_size()
+        assert cache_size == 0
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_chunking_and_extraction_strategies.py b/tests/async/test_chunking_and_extraction_strategies.py
new file mode 100644
index 0000000..7c628bc
--- /dev/null
+++ b/tests/async/test_chunking_and_extraction_strategies.py
@@ -0,0 +1,86 @@
+import os
+import sys
+import pytest
+import json
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai import LLMConfig
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+from crawl4ai.chunking_strategy import RegexChunking
+from crawl4ai import LLMExtractionStrategy
+
+
+@pytest.mark.asyncio
+async def test_regex_chunking():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        chunking_strategy = RegexChunking(patterns=["\n\n"])
+        result = await crawler.arun(
+            url=url, chunking_strategy=chunking_strategy, bypass_cache=True
+        )
+        assert result.success
+        assert result.extracted_content
+        chunks = json.loads(result.extracted_content)
+        assert len(chunks) > 1  # Ensure multiple chunks were created
+
+
+# @pytest.mark.asyncio
+# async def test_cosine_strategy():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://www.nbcnews.com/business"
+#         extraction_strategy = CosineStrategy(word_count_threshold=10, max_dist=0.2, linkage_method="ward", top_k=3, sim_threshold=0.3)
+#         result = await crawler.arun(
+#             url=url,
+#             extraction_strategy=extraction_strategy,
+#             bypass_cache=True
+#         )
+#         assert result.success
+#         assert result.extracted_content
+#         extracted_data = json.loads(result.extracted_content)
+#         assert len(extracted_data) > 0
+#         assert all('tags' in item for item in extracted_data)
+
+
+@pytest.mark.asyncio
+async def test_llm_extraction_strategy():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        extraction_strategy = LLMExtractionStrategy(
+            llm_config=LLMConfig(provider="openai/gpt-4o-mini",api_token=os.getenv("OPENAI_API_KEY")),
+            instruction="Extract only content related to technology",
+        )
+        result = await crawler.arun(
+            url=url, extraction_strategy=extraction_strategy, bypass_cache=True
+        )
+        assert result.success
+        assert result.extracted_content
+        extracted_data = json.loads(result.extracted_content)
+        assert len(extracted_data) > 0
+        assert all("content" in item for item in extracted_data)
+
+
+# @pytest.mark.asyncio
+# async def test_combined_chunking_and_extraction():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://www.nbcnews.com/business"
+#         chunking_strategy = RegexChunking(patterns=["\n\n"])
+#         extraction_strategy = CosineStrategy(word_count_threshold=10, max_dist=0.2, linkage_method="ward", top_k=3, sim_threshold=0.3)
+#         result = await crawler.arun(
+#             url=url,
+#             chunking_strategy=chunking_strategy,
+#             extraction_strategy=extraction_strategy,
+#             bypass_cache=True
+#         )
+#         assert result.success
+#         assert result.extracted_content
+#         extracted_data = json.loads(result.extracted_content)
+#         assert len(extracted_data) > 0
+#         assert all('tags' in item for item in extracted_data)
+#         assert all('content' in item for item in extracted_data)
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_content_extraction.py b/tests/async/test_content_extraction.py
new file mode 100644
index 0000000..509a387
--- /dev/null
+++ b/tests/async/test_content_extraction.py
@@ -0,0 +1,108 @@
+import os
+import sys
+import pytest
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_extract_markdown():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.markdown
+        assert isinstance(result.markdown, str)
+        assert len(result.markdown) > 0
+
+
+@pytest.mark.asyncio
+async def test_extract_cleaned_html():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.cleaned_html
+        assert isinstance(result.cleaned_html, str)
+        assert len(result.cleaned_html) > 0
+
+
+@pytest.mark.asyncio
+async def test_extract_media():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.media
+        media = result.media
+        assert isinstance(media, dict)
+        assert "images" in media
+        assert isinstance(media["images"], list)
+        for image in media["images"]:
+            assert "src" in image
+            assert "alt" in image
+            assert "type" in image
+
+
+@pytest.mark.asyncio
+async def test_extract_links():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.links
+        links = result.links
+        assert isinstance(links, dict)
+        assert "internal" in links
+        assert "external" in links
+        assert isinstance(links["internal"], list)
+        assert isinstance(links["external"], list)
+        for link in links["internal"] + links["external"]:
+            assert "href" in link
+            assert "text" in link
+
+
+@pytest.mark.asyncio
+async def test_extract_metadata():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.metadata
+        metadata = result.metadata
+        assert isinstance(metadata, dict)
+        assert "title" in metadata
+        assert isinstance(metadata["title"], str)
+
+
+@pytest.mark.asyncio
+async def test_css_selector_extraction():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        css_selector = "h1, h2, h3"
+        result = await crawler.arun(
+            url=url, bypass_cache=True, css_selector=css_selector
+        )
+        assert result.success
+        assert result.markdown
+        assert all(heading in result.markdown for heading in ["#", "##", "###"])
+
+@pytest.mark.asyncio
+async def test_base_tag_link_extraction():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://sohamkukreti.github.io/portfolio"
+        result = await crawler.arun(url=url)
+        assert result.success
+        assert result.links
+        assert isinstance(result.links, dict)
+        assert "internal" in result.links
+        assert "external" in result.links
+        assert any("github.com" in x["href"] for x in result.links["external"])
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_content_filter_bm25.py b/tests/async/test_content_filter_bm25.py
new file mode 100644
index 0000000..f05a8af
--- /dev/null
+++ b/tests/async/test_content_filter_bm25.py
@@ -0,0 +1,183 @@
+import os, sys
+import pytest
+from bs4 import BeautifulSoup
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+
+@pytest.fixture
+def basic_html():
+    return """
+    
+        
+            Test Article
+            
+            
+        
+        
+            
      Main Heading
      
+            
      
+                
      This is a long paragraph with more than fifty words. It continues with more text to ensure we meet the minimum word count threshold. We need to make sure this paragraph is substantial enough to be considered for extraction according to our filtering rules. This should be enough words now.
      
+                
      Skip this nav content
      
+            
      
+        
+    
+    """
+
+
+@pytest.fixture
+def wiki_html():
+    return """
+    
+        
+            Wikipedia Article
+        
+        
+            
      Article Title
      
+            
      Section 1
      
+            
      Short but important section header description.
      
+            
      
+                
      Long paragraph with sufficient words to meet the minimum threshold. This paragraph continues with more text to ensure we have enough content for proper testing. We need to make sure this has enough words to pass our filters and be considered valid content for extraction purposes.
      
+            
      
+        
+    
+    """
+
+
+@pytest.fixture
+def no_meta_html():
+    return """
+    
+        
+            
      Simple Page
      
+            
      First paragraph that should be used as fallback for query when no meta tags exist. This text needs to be long enough to serve as a meaningful fallback for our content extraction process.
      
+        
+    
+    """
+
+
+class TestBM25ContentFilter:
+    def test_basic_extraction(self, basic_html):
+        """Test basic content extraction functionality"""
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(basic_html)
+
+        assert contents, "Should extract content"
+        assert len(contents) >= 1, "Should extract at least one content block"
+        assert "long paragraph" in " ".join(contents).lower()
+        assert "navigation" not in " ".join(contents).lower()
+
+    def test_user_query_override(self, basic_html):
+        """Test that user query overrides metadata extraction"""
+        user_query = "specific test query"
+        filter = BM25ContentFilter(user_query=user_query)
+
+        # Access internal state to verify query usage
+        soup = BeautifulSoup(basic_html, "lxml")
+        extracted_query = filter.extract_page_query(soup.find("head"))
+
+        assert extracted_query == user_query
+        assert "Test description" not in extracted_query
+
+    def test_header_extraction(self, wiki_html):
+        """Test that headers are properly extracted despite length"""
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(wiki_html)
+
+        combined_content = " ".join(contents).lower()
+        assert "section 1" in combined_content, "Should include section header"
+        assert "article title" in combined_content, "Should include main title"
+
+    def test_no_metadata_fallback(self, no_meta_html):
+        """Test fallback behavior when no metadata is present"""
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(no_meta_html)
+
+        assert contents, "Should extract content even without metadata"
+        assert "First paragraph" in " ".join(
+            contents
+        ), "Should use first paragraph content"
+
+    def test_empty_input(self):
+        """Test handling of empty input"""
+        filter = BM25ContentFilter()
+        assert filter.filter_content("") == []
+        assert filter.filter_content(None) == []
+
+    def test_malformed_html(self):
+        """Test handling of malformed HTML"""
+        malformed_html = "
      Unclosed paragraph
      Nested content
      "
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(malformed_html)
+
+        assert isinstance(contents, list), "Should return list even with malformed HTML"
+
+    def test_threshold_behavior(self, basic_html):
+        """Test different BM25 threshold values"""
+        strict_filter = BM25ContentFilter(bm25_threshold=2.0)
+        lenient_filter = BM25ContentFilter(bm25_threshold=0.5)
+
+        strict_contents = strict_filter.filter_content(basic_html)
+        lenient_contents = lenient_filter.filter_content(basic_html)
+
+        assert len(strict_contents) <= len(
+            lenient_contents
+        ), "Strict threshold should extract fewer elements"
+
+    def test_html_cleaning(self, basic_html):
+        """Test HTML cleaning functionality"""
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(basic_html)
+
+        cleaned_content = " ".join(contents)
+        assert "class=" not in cleaned_content, "Should remove class attributes"
+        assert "style=" not in cleaned_content, "Should remove style attributes"
+        assert "
+            
      {'
      Test content. ' * 1000}
      
+        
+        """
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(large_html)
+        assert contents, "Should handle large content blocks"
+
+    @pytest.mark.parametrize(
+        "unwanted_tag", ["script", "style", "nav", "footer", "header"]
+    )
+    def test_excluded_tags(self, unwanted_tag):
+        """Test that specific tags are properly excluded"""
+        html = f"""
+        
+            <{unwanted_tag}>Should not appear
+            
      Should appear
      
+        
+        """
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(html)
+
+        combined_content = " ".join(contents).lower()
+        assert "should not appear" not in combined_content
+
+    def test_performance(self, basic_html):
+        """Test performance with timer"""
+        filter = BM25ContentFilter()
+
+        import time
+
+        start = time.perf_counter()
+        filter.filter_content(basic_html)
+        duration = time.perf_counter() - start
+
+        assert duration < 1.0, f"Processing took too long: {duration:.2f} seconds"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])
diff --git a/tests/async/test_content_filter_prune.py b/tests/async/test_content_filter_prune.py
new file mode 100644
index 0000000..1f75a9e
--- /dev/null
+++ b/tests/async/test_content_filter_prune.py
@@ -0,0 +1,170 @@
+import os, sys
+import pytest
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+
+@pytest.fixture
+def basic_html():
+    return """
+    
+        
+            
      
+                
      Main Article
      
+                
      This is a high-quality paragraph with substantial text content. It contains enough words to pass the threshold and has good text density without too many links. This kind of content should survive the pruning process.
      
+                
      Low quality sidebar content
      
+                
+            
      
+        
+    
+    """
+
+
+@pytest.fixture
+def link_heavy_html():
+    return """
+    
+        
+            
      
+                
      Good content paragraph that should remain.
      
+                
      
+                    Link 1
+                    Link 2
+                    Link 3
+                    Link 4
+                
      
+            
      
+        
+    
+    """
+
+
+@pytest.fixture
+def mixed_content_html():
+    return """
+    
+        
+            
      
+                
      Article Title
      
+                
      Short summary.
      
+                
      
+                    
      Long high-quality paragraph with substantial content that should definitely survive the pruning process. This content has good text density and proper formatting which makes it valuable for retention.
      
+                
      
+                
      
+                    
      Short comment 1
      
+                    
      Short comment 2
      
+                
      
+            
      
+        
+    
+    """
+
+
+class TestPruningContentFilter:
+    def test_basic_pruning(self, basic_html):
+        """Test basic content pruning functionality"""
+        filter = PruningContentFilter(min_word_threshold=5)
+        contents = filter.filter_content(basic_html)
+
+        combined_content = " ".join(contents).lower()
+        assert "high-quality paragraph" in combined_content
+        assert "sidebar content" not in combined_content
+        assert "share buttons" not in combined_content
+
+    def test_min_word_threshold(self, mixed_content_html):
+        """Test minimum word threshold filtering"""
+        filter = PruningContentFilter(min_word_threshold=10)
+        contents = filter.filter_content(mixed_content_html)
+
+        combined_content = " ".join(contents).lower()
+        assert "short summary" not in combined_content
+        assert "long high-quality paragraph" in combined_content
+        assert "short comment" not in combined_content
+
+    def test_threshold_types(self, basic_html):
+        """Test fixed vs dynamic thresholds"""
+        fixed_filter = PruningContentFilter(threshold_type="fixed", threshold=0.48)
+        dynamic_filter = PruningContentFilter(threshold_type="dynamic", threshold=0.45)
+
+        fixed_contents = fixed_filter.filter_content(basic_html)
+        dynamic_contents = dynamic_filter.filter_content(basic_html)
+
+        assert len(fixed_contents) != len(
+            dynamic_contents
+        ), "Fixed and dynamic thresholds should yield different results"
+
+    def test_link_density_impact(self, link_heavy_html):
+        """Test handling of link-heavy content"""
+        filter = PruningContentFilter(threshold_type="dynamic")
+        contents = filter.filter_content(link_heavy_html)
+
+        combined_content = " ".join(contents).lower()
+        assert "good content paragraph" in combined_content
+        assert (
+            len([c for c in contents if "href" in c]) < 2
+        ), "Should prune link-heavy sections"
+
+    def test_tag_importance(self, mixed_content_html):
+        """Test tag importance in scoring"""
+        filter = PruningContentFilter(threshold_type="dynamic")
+        contents = filter.filter_content(mixed_content_html)
+
+        has_article = any("article" in c.lower() for c in contents)
+        has_h1 = any("h1" in c.lower() for c in contents)
+        assert has_article or has_h1, "Should retain important tags"
+
+    def test_empty_input(self):
+        """Test handling of empty input"""
+        filter = PruningContentFilter()
+        assert filter.filter_content("") == []
+        assert filter.filter_content(None) == []
+
+    def test_malformed_html(self):
+        """Test handling of malformed HTML"""
+        malformed_html = "
      Unclosed div
      Nestedcontent
      "
+        filter = PruningContentFilter()
+        contents = filter.filter_content(malformed_html)
+        assert isinstance(contents, list)
+
+    def test_performance(self, basic_html):
+        """Test performance with timer"""
+        filter = PruningContentFilter()
+
+        import time
+
+        start = time.perf_counter()
+        filter.filter_content(basic_html)
+        duration = time.perf_counter() - start
+
+        # Extra strict on performance since you mentioned milliseconds matter
+        assert duration < 0.1, f"Processing took too long: {duration:.3f} seconds"
+
+    @pytest.mark.parametrize(
+        "threshold,expected_count",
+        [
+            (0.3, 4),  # Very lenient
+            (0.48, 2),  # Default
+            (0.7, 1),  # Very strict
+        ],
+    )
+    def test_threshold_levels(self, mixed_content_html, threshold, expected_count):
+        """Test different threshold levels"""
+        filter = PruningContentFilter(threshold_type="fixed", threshold=threshold)
+        contents = filter.filter_content(mixed_content_html)
+        assert (
+            len(contents) <= expected_count
+        ), f"Expected {expected_count} or fewer elements with threshold {threshold}"
+
+    def test_consistent_output(self, basic_html):
+        """Test output consistency across multiple runs"""
+        filter = PruningContentFilter()
+        first_run = filter.filter_content(basic_html)
+        second_run = filter.filter_content(basic_html)
+        assert first_run == second_run, "Output should be consistent"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])
diff --git a/tests/async/test_content_scraper_strategy.py b/tests/async/test_content_scraper_strategy.py
new file mode 100644
index 0000000..00022cd
--- /dev/null
+++ b/tests/async/test_content_scraper_strategy.py
@@ -0,0 +1,216 @@
+import os
+import sys
+import time
+import csv
+from tabulate import tabulate
+from dataclasses import dataclass
+from typing import List
+
+parent_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+# This test compares the same strategy with itself now since WebScrapingStrategy is deprecated
+
+
+@dataclass
+class TestResult:
+    name: str
+    success: bool
+    images: int
+    internal_links: int
+    external_links: int
+    markdown_length: int
+    execution_time: float
+
+
+class StrategyTester:
+    def __init__(self):
+        self.new_scraper = LXMLWebScrapingStrategy()
+        self.current_scraper = LXMLWebScrapingStrategy()  # Same strategy now
+        with open(__location__ + "/sample_wikipedia.html", "r", encoding="utf-8") as f:
+            self.WIKI_HTML = f.read()
+        self.results = {"new": [], "current": []}
+
+    def run_test(self, name: str, **kwargs) -> tuple[TestResult, TestResult]:
+        results = []
+        for scraper in [self.new_scraper, self.current_scraper]:
+            start_time = time.time()
+            result = scraper._get_content_of_website_optimized(
+                url="https://en.wikipedia.org/wiki/Test", html=self.WIKI_HTML, **kwargs
+            )
+            execution_time = time.time() - start_time
+
+            test_result = TestResult(
+                name=name,
+                success=result["success"],
+                images=len(result["media"]["images"]),
+                internal_links=len(result["links"]["internal"]),
+                external_links=len(result["links"]["external"]),
+                markdown_length=len(result["markdown"]),
+                execution_time=execution_time,
+            )
+            results.append(test_result)
+
+        return results[0], results[1]  # new, current
+
+    def run_all_tests(self):
+        test_cases = [
+            ("Basic Extraction", {}),
+            ("Exclude Tags", {"excluded_tags": ["table", "div.infobox", "div.navbox"]}),
+            ("Word Threshold", {"word_count_threshold": 50}),
+            ("CSS Selector", {"css_selector": "div.mw-parser-output > p"}),
+            (
+                "Link Exclusions",
+                {
+                    "exclude_external_links": True,
+                    "exclude_social_media_links": True,
+                    "exclude_domains": ["facebook.com", "twitter.com"],
+                },
+            ),
+            (
+                "Media Handling",
+                {
+                    "exclude_external_images": True,
+                    "image_description_min_word_threshold": 20,
+                },
+            ),
+            ("Text Only", {"only_text": True, "remove_forms": True}),
+            ("HTML Cleaning", {"clean_html": True, "keep_data_attributes": True}),
+            (
+                "HTML2Text Options",
+                {
+                    "html2text": {
+                        "skip_internal_links": True,
+                        "single_line_break": True,
+                        "mark_code": True,
+                        "preserve_tags": ["pre", "code"],
+                    }
+                },
+            ),
+        ]
+
+        all_results = []
+        for name, kwargs in test_cases:
+            try:
+                new_result, current_result = self.run_test(name, **kwargs)
+                all_results.append((name, new_result, current_result))
+            except Exception as e:
+                print(f"Error in {name}: {str(e)}")
+
+        self.save_results_to_csv(all_results)
+        self.print_comparison_table(all_results)
+
+    def save_results_to_csv(self, all_results: List[tuple]):
+        csv_file = os.path.join(__location__, "strategy_comparison_results.csv")
+        with open(csv_file, "w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow(
+                [
+                    "Test Name",
+                    "Strategy",
+                    "Success",
+                    "Images",
+                    "Internal Links",
+                    "External Links",
+                    "Markdown Length",
+                    "Execution Time",
+                ]
+            )
+
+            for name, new_result, current_result in all_results:
+                writer.writerow(
+                    [
+                        name,
+                        "New",
+                        new_result.success,
+                        new_result.images,
+                        new_result.internal_links,
+                        new_result.external_links,
+                        new_result.markdown_length,
+                        f"{new_result.execution_time:.3f}",
+                    ]
+                )
+                writer.writerow(
+                    [
+                        name,
+                        "Current",
+                        current_result.success,
+                        current_result.images,
+                        current_result.internal_links,
+                        current_result.external_links,
+                        current_result.markdown_length,
+                        f"{current_result.execution_time:.3f}",
+                    ]
+                )
+
+    def print_comparison_table(self, all_results: List[tuple]):
+        table_data = []
+        headers = [
+            "Test Name",
+            "Strategy",
+            "Success",
+            "Images",
+            "Internal Links",
+            "External Links",
+            "Markdown Length",
+            "Time (s)",
+        ]
+
+        for name, new_result, current_result in all_results:
+            # Check for differences
+            differences = []
+            if new_result.images != current_result.images:
+                differences.append("images")
+            if new_result.internal_links != current_result.internal_links:
+                differences.append("internal_links")
+            if new_result.external_links != current_result.external_links:
+                differences.append("external_links")
+            if new_result.markdown_length != current_result.markdown_length:
+                differences.append("markdown")
+
+            # Add row for new strategy
+            new_row = [
+                name,
+                "New",
+                new_result.success,
+                new_result.images,
+                new_result.internal_links,
+                new_result.external_links,
+                new_result.markdown_length,
+                f"{new_result.execution_time:.3f}",
+            ]
+            table_data.append(new_row)
+
+            # Add row for current strategy
+            current_row = [
+                "",
+                "Current",
+                current_result.success,
+                current_result.images,
+                current_result.internal_links,
+                current_result.external_links,
+                current_result.markdown_length,
+                f"{current_result.execution_time:.3f}",
+            ]
+            table_data.append(current_row)
+
+            # Add difference summary if any
+            if differences:
+                table_data.append(
+                    ["", "⚠️ Differences", ", ".join(differences), "", "", "", "", ""]
+                )
+
+            # Add empty row for better readability
+            table_data.append([""] * len(headers))
+
+        print("\nStrategy Comparison Results:")
+        print(tabulate(table_data, headers=headers, tablefmt="grid"))
+
+
+if __name__ == "__main__":
+    tester = StrategyTester()
+    tester.run_all_tests()
diff --git a/tests/async/test_crawler_strategy.py b/tests/async/test_crawler_strategy.py
new file mode 100644
index 0000000..337b5aa
--- /dev/null
+++ b/tests/async/test_crawler_strategy.py
@@ -0,0 +1,73 @@
+import os
+import sys
+import pytest
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_custom_user_agent():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        custom_user_agent = "MyCustomUserAgent/1.0"
+        crawler.crawler_strategy.update_user_agent(custom_user_agent)
+        url = "https://httpbin.org/user-agent"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert custom_user_agent in result.html
+
+
+@pytest.mark.asyncio
+async def test_custom_headers():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        custom_headers = {"X-Test-Header": "TestValue"}
+        crawler.crawler_strategy.set_custom_headers(custom_headers)
+        url = "https://httpbin.org/headers"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert "X-Test-Header" in result.html
+        assert "TestValue" in result.html
+
+
+@pytest.mark.asyncio
+async def test_javascript_execution():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        js_code = "document.body.innerHTML = '
      Modified by JS
      ';"
+        url = "https://www.example.com"
+        result = await crawler.arun(url=url, bypass_cache=True, js_code=js_code)
+        assert result.success
+        assert "
      Modified by JS
      " in result.html
+
+
+@pytest.mark.asyncio
+async def test_hook_execution():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+
+        async def test_hook(page):
+            await page.evaluate("document.body.style.backgroundColor = 'red';")
+            return page
+
+        crawler.crawler_strategy.set_hook("after_goto", test_hook)
+        url = "https://www.example.com"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert "background-color: red" in result.html
+
+
+@pytest.mark.asyncio
+async def test_screenshot():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.example.com"
+        result = await crawler.arun(url=url, bypass_cache=True, screenshot=True)
+        assert result.success
+        assert result.screenshot
+        assert isinstance(result.screenshot, str)
+        assert len(result.screenshot) > 0
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_database_operations.py b/tests/async/test_database_operations.py
new file mode 100644
index 0000000..db0d328
--- /dev/null
+++ b/tests/async/test_database_operations.py
@@ -0,0 +1,90 @@
+import os
+import sys
+import pytest
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_cache_url():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.example.com"
+        # First run to cache the URL
+        result1 = await crawler.arun(url=url, bypass_cache=True)
+        assert result1.success
+
+        # Second run to retrieve from cache
+        result2 = await crawler.arun(url=url, bypass_cache=False)
+        assert result2.success
+        assert result2.html == result1.html
+
+
+@pytest.mark.asyncio
+async def test_bypass_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.python.org"
+        # First run to cache the URL
+        result1 = await crawler.arun(url=url, bypass_cache=True)
+        assert result1.success
+
+        # Second run bypassing cache
+        result2 = await crawler.arun(url=url, bypass_cache=True)
+        assert result2.success
+        assert (
+            result2.html != result1.html
+        )  # Content might be different due to dynamic nature of websites
+
+
+@pytest.mark.asyncio
+async def test_cache_size():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        initial_size = await crawler.aget_cache_size()
+
+        url = "https://www.nbcnews.com/business"
+        await crawler.arun(url=url, bypass_cache=True)
+
+        new_size = await crawler.aget_cache_size()
+        assert new_size == initial_size + 1
+
+
+@pytest.mark.asyncio
+async def test_clear_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.example.org"
+        await crawler.arun(url=url, bypass_cache=True)
+
+        initial_size = await crawler.aget_cache_size()
+        assert initial_size > 0
+
+        await crawler.aclear_cache()
+        new_size = await crawler.aget_cache_size()
+        assert new_size == 0
+
+
+@pytest.mark.asyncio
+async def test_flush_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.example.net"
+        await crawler.arun(url=url, bypass_cache=True)
+
+        initial_size = await crawler.aget_cache_size()
+        assert initial_size > 0
+
+        await crawler.aflush_cache()
+        new_size = await crawler.aget_cache_size()
+        assert new_size == 0
+
+        # Try to retrieve the previously cached URL
+        result = await crawler.arun(url=url, bypass_cache=False)
+        assert (
+            result.success
+        )  # The crawler should still succeed, but it will fetch the content anew
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_dispatchers.py b/tests/async/test_dispatchers.py
new file mode 100644
index 0000000..99cf4a9
--- /dev/null
+++ b/tests/async/test_dispatchers.py
@@ -0,0 +1,170 @@
+import pytest
+import time
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    MemoryAdaptiveDispatcher,
+    SemaphoreDispatcher,
+    RateLimiter,
+    CrawlerMonitor,
+    DisplayMode,
+    CacheMode,
+)
+
+
+@pytest.fixture
+def browser_config():
+    return BrowserConfig(headless=True, verbose=False)
+
+
+@pytest.fixture
+def run_config():
+    return CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+
+@pytest.fixture
+def test_urls():
+    return [
+        "http://example.com",
+        "http://example.com/page1",
+        "http://example.com/page2",
+    ]
+
+
+@pytest.mark.asyncio
+class TestDispatchStrategies:
+    async def test_memory_adaptive_basic(self, browser_config, run_config, test_urls):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(
+                memory_threshold_percent=70.0, max_session_permit=2, check_interval=0.1
+            )
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            assert all(r.success for r in results)
+
+    async def test_memory_adaptive_with_rate_limit(
+        self, browser_config, run_config, test_urls
+    ):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(
+                memory_threshold_percent=70.0,
+                max_session_permit=2,
+                check_interval=0.1,
+                rate_limiter=RateLimiter(
+                    base_delay=(0.1, 0.2), max_delay=1.0, max_retries=2
+                ),
+            )
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            assert all(r.success for r in results)
+
+    async def test_semaphore_basic(self, browser_config, run_config, test_urls):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = SemaphoreDispatcher(semaphore_count=2)
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            assert all(r.success for r in results)
+
+    async def test_semaphore_with_rate_limit(
+        self, browser_config, run_config, test_urls
+    ):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = SemaphoreDispatcher(
+                semaphore_count=2,
+                rate_limiter=RateLimiter(
+                    base_delay=(0.1, 0.2), max_delay=1.0, max_retries=2
+                ),
+            )
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            assert all(r.success for r in results)
+
+    async def test_memory_adaptive_memory_error(
+        self, browser_config, run_config, test_urls
+    ):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(
+                memory_threshold_percent=1.0,  # Set unrealistically low threshold
+                max_session_permit=2,
+                check_interval=0.1,
+                memory_wait_timeout=1.0,  # Short timeout for testing
+            )
+            with pytest.raises(MemoryError):
+                await crawler.arun_many(
+                    test_urls, config=run_config, dispatcher=dispatcher
+                )
+
+    async def test_empty_urls(self, browser_config, run_config):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(max_session_permit=2)
+            results = await crawler.arun_many(
+                [], config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == 0
+
+    async def test_single_url(self, browser_config, run_config):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(max_session_permit=2)
+            results = await crawler.arun_many(
+                ["http://example.com"], config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == 1
+            assert results[0].success
+
+    async def test_invalid_urls(self, browser_config, run_config):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(max_session_permit=2)
+            results = await crawler.arun_many(
+                ["http://invalid.url.that.doesnt.exist"],
+                config=run_config,
+                dispatcher=dispatcher,
+            )
+            assert len(results) == 1
+            assert not results[0].success
+
+    async def test_rate_limit_backoff(self, browser_config, run_config):
+        urls = ["http://example.com"] * 5  # Multiple requests to same domain
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(
+                max_session_permit=2,
+                rate_limiter=RateLimiter(
+                    base_delay=(0.1, 0.2),
+                    max_delay=1.0,
+                    max_retries=2,
+                    rate_limit_codes=[200],  # Force rate limiting for testing
+                ),
+            )
+            start_time = time.time()
+            results = await crawler.arun_many(
+                urls, config=run_config, dispatcher=dispatcher
+            )
+            duration = time.time() - start_time
+            assert len(results) == len(urls)
+            assert duration > 1.0  # Ensure rate limiting caused delays
+
+    async def test_monitor_integration(self, browser_config, run_config, test_urls):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            monitor = CrawlerMonitor(
+                max_visible_rows=5, display_mode=DisplayMode.DETAILED
+            )
+            dispatcher = MemoryAdaptiveDispatcher(max_session_permit=2, monitor=monitor)
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            # Check monitor stats
+            assert len(monitor.stats) == len(test_urls)
+            assert all(stat.end_time is not None for stat in monitor.stats.values())
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v", "--asyncio-mode=auto"])
diff --git a/tests/async/test_edge_cases.py b/tests/async/test_edge_cases.py
new file mode 100644
index 0000000..d3adb53
--- /dev/null
+++ b/tests/async/test_edge_cases.py
@@ -0,0 +1,133 @@
+import os
+import re
+import sys
+import pytest
+from bs4 import BeautifulSoup
+import asyncio
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+# @pytest.mark.asyncio
+# async def test_large_content_page():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://en.wikipedia.org/wiki/List_of_largest_known_stars"  # A page with a large table
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert len(result.html) > 1000000  # Expecting more than 1MB of content
+
+# @pytest.mark.asyncio
+# async def test_minimal_content_page():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://example.com"  # A very simple page
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert len(result.html) < 10000  # Expecting less than 10KB of content
+
+# @pytest.mark.asyncio
+# async def test_single_page_application():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://reactjs.org/"  # React's website is a SPA
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert "react" in result.html.lower()
+
+# @pytest.mark.asyncio
+# async def test_page_with_infinite_scroll():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://news.ycombinator.com/"  # Hacker News has infinite scroll
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert "hacker news" in result.html.lower()
+
+# @pytest.mark.asyncio
+# async def test_page_with_heavy_javascript():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://www.airbnb.com/"  # Airbnb uses a lot of JavaScript
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert "airbnb" in result.html.lower()
+
+# @pytest.mark.asyncio
+# async def test_page_with_mixed_content():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://github.com/"  # GitHub has a mix of static and dynamic content
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert "github" in result.html.lower()
+
+
+# Add this test to your existing test file
+@pytest.mark.asyncio
+async def test_typescript_commits_multi_page():
+    first_commit = ""
+
+    async def on_execution_started(page):
+        nonlocal first_commit
+        try:
+            # Check if the page firct commit h4 text is different from the first commit (use document.querySelector('li.Box-sc-g0xbh4-0 h4'))
+            while True:
+                await page.wait_for_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await page.query_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await commit.evaluate("(element) => element.textContent")
+                commit = re.sub(r"\s+", "", commit)
+                if commit and commit != first_commit:
+                    first_commit = commit
+                    break
+                await asyncio.sleep(0.5)
+        except Exception as e:
+            print(f"Warning: New content didn't appear after JavaScript execution: {e}")
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        js_next_page = """
+        const button = document.querySelector('a[data-testid="pagination-next-button"]');
+        if (button) button.click();
+        """
+
+        for page in range(3):  # Crawl 3 pages
+            result = await crawler.arun(
+                url=url,  # Only use URL for the first page
+                session_id=session_id,
+                css_selector="li.Box-sc-g0xbh4-0",
+                js=js_next_page
+                if page > 0
+                else None,  # Don't click 'next' on the first page
+                bypass_cache=True,
+                js_only=page > 0,  # Use js_only for subsequent pages
+            )
+
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            # Parse the HTML and extract commits
+            soup = BeautifulSoup(result.cleaned_html, "html.parser")
+            commits = soup.select("li")
+            # Take first commit find h4 extract text
+            first_commit = commits[0].find("h4").text
+            first_commit = re.sub(r"\s+", "", first_commit)
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        # Clean up the session
+        await crawler.crawler_strategy.kill_session(session_id)
+
+        # Assertions
+        assert (
+            len(all_commits) >= 90
+        ), f"Expected at least 90 commits, but got {len(all_commits)}"
+
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_error_handling.py b/tests/async/test_error_handling.py
new file mode 100644
index 0000000..ae4af6c
--- /dev/null
+++ b/tests/async/test_error_handling.py
@@ -0,0 +1,78 @@
+# import os
+# import sys
+# import pytest
+# import asyncio
+
+# # Add the parent directory to the Python path
+# parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+# sys.path.append(parent_dir)
+
+# from crawl4ai.async_webcrawler import AsyncWebCrawler
+# from crawl4ai.utils import InvalidCSSSelectorError
+
+# class AsyncCrawlerWrapper:
+#     def __init__(self):
+#         self.crawler = None
+
+#     async def setup(self):
+#         self.crawler = AsyncWebCrawler(verbose=True)
+#         await self.crawler.awarmup()
+
+#     async def cleanup(self):
+#         if self.crawler:
+#             await self.crawler.aclear_cache()
+
+# @pytest.fixture(scope="module")
+# def crawler_wrapper():
+#     wrapper = AsyncCrawlerWrapper()
+#     asyncio.get_event_loop().run_until_complete(wrapper.setup())
+#     yield wrapper
+#     asyncio.get_event_loop().run_until_complete(wrapper.cleanup())
+
+# @pytest.mark.asyncio
+# async def test_network_error(crawler_wrapper):
+#     url = "https://www.nonexistentwebsite123456789.com"
+#     result = await crawler_wrapper.crawler.arun(url=url, bypass_cache=True)
+#     assert not result.success
+#     assert "Failed to crawl" in result.error_message
+
+# # @pytest.mark.asyncio
+# # async def test_timeout_error(crawler_wrapper):
+# #     # Simulating a timeout by using a very short timeout value
+# #     url = "https://www.nbcnews.com/business"
+# #     result = await crawler_wrapper.crawler.arun(url=url, bypass_cache=True, timeout=0.001)
+# #     assert not result.success
+# #     assert "timeout" in result.error_message.lower()
+
+# # @pytest.mark.asyncio
+# # async def test_invalid_css_selector(crawler_wrapper):
+# #     url = "https://www.nbcnews.com/business"
+# #     with pytest.raises(InvalidCSSSelectorError):
+# #         await crawler_wrapper.crawler.arun(url=url, bypass_cache=True, css_selector="invalid>>selector")
+
+# # @pytest.mark.asyncio
+# # async def test_js_execution_error(crawler_wrapper):
+# #     url = "https://www.nbcnews.com/business"
+# #     invalid_js = "This is not valid JavaScript code;"
+# #     result = await crawler_wrapper.crawler.arun(url=url, bypass_cache=True, js=invalid_js)
+# #     assert not result.success
+# #     assert "JavaScript" in result.error_message
+
+# # @pytest.mark.asyncio
+# # async def test_empty_page(crawler_wrapper):
+# #     # Use a URL that typically returns an empty page
+# #     url = "http://example.com/empty"
+# #     result = await crawler_wrapper.crawler.arun(url=url, bypass_cache=True)
+# #     assert result.success  # The crawl itself should succeed
+# #     assert not result.markdown.strip()  # The markdown content should be empty or just whitespace
+
+# # @pytest.mark.asyncio
+# # async def test_rate_limiting(crawler_wrapper):
+# #     # Simulate rate limiting by making multiple rapid requests
+# #     url = "https://www.nbcnews.com/business"
+# #     results = await asyncio.gather(*[crawler_wrapper.crawler.arun(url=url, bypass_cache=True) for _ in range(10)])
+# #     assert any(not result.success and "rate limit" in result.error_message.lower() for result in results)
+
+# # Entry point for debugging
+# if __name__ == "__main__":
+#     pytest.main([__file__, "-v"])
diff --git a/tests/async/test_evaluation_scraping_methods_performance.configs.py b/tests/async/test_evaluation_scraping_methods_performance.configs.py
new file mode 100644
index 0000000..797cf68
--- /dev/null
+++ b/tests/async/test_evaluation_scraping_methods_performance.configs.py
@@ -0,0 +1,705 @@
+import json
+import time
+from bs4 import BeautifulSoup
+from crawl4ai.content_scraping_strategy import (
+    WebScrapingStrategy,
+    LXMLWebScrapingStrategy,
+)
+from typing import Dict, List, Tuple
+import difflib
+from lxml import html as lhtml, etree
+
+
+def normalize_dom(element):
+    """
+    Recursively normalizes an lxml HTML element:
+      - Removes comment nodes
+      - Sorts attributes on each node
+      - Removes  if you want (optional)
+    Returns the same element (mutated).
+    """
+    # Remove comment nodes
+    comments = element.xpath("//comment()")
+    for c in comments:
+        p = c.getparent()
+        if p is not None:
+            p.remove(c)
+
+    # If you'd like to remove , or unify /, you could do so here.
+    # For example, remove  entirely:
+    # heads = element.xpath('//head')
+    # for h in heads:
+    #     parent = h.getparent()
+    #     if parent is not None:
+    #         parent.remove(h)
+
+    # Sort attributes (to avoid false positives due to attr order)
+    for el in element.iter():
+        if el.attrib:
+            # Convert to a sorted list of (k, v), then reassign
+            sorted_attribs = sorted(el.attrib.items())
+            el.attrib.clear()
+            for k, v in sorted_attribs:
+                el.set(k, v)
+
+    return element
+
+
+def strip_html_body(root):
+    """
+    If 'root' is , find its  child and move all of 's children
+    into a new 
      . Return that 
      .
+
+    If 'root' is , similarly move all of its children into a new 
       and return it.
+
+    Otherwise, return 'root' as-is.
+    """
+    tag_name = (root.tag or "").lower()
+
+    # Case 1: The root is 
+    if tag_name == "html":
+        bodies = root.xpath("./body")
+        if bodies:
+            body = bodies[0]
+            new_div = lhtml.Element("div")
+            for child in body:
+                new_div.append(child)
+            return new_div
+        else:
+            # No  found; just return the  root
+            return root
+
+    # Case 2: The root is 
+    elif tag_name == "body":
+        new_div = lhtml.Element("div")
+        for child in root:
+            new_div.append(child)
+        return new_div
+
+    # Case 3: Neither  nor 
+    else:
+        return root
+
+
+def compare_nodes(node1, node2, differences, path="/"):
+    """
+    Recursively compare two lxml nodes, appending textual differences to `differences`.
+    `path` is used to indicate the location in the tree (like an XPath).
+    """
+    # 1) Compare tag names
+    if node1.tag != node2.tag:
+        differences.append(f"Tag mismatch at {path}: '{node1.tag}' vs. '{node2.tag}'")
+        return
+
+    # 2) Compare attributes
+    # By now, they are sorted in normalize_dom()
+    attrs1 = list(node1.attrib.items())
+    attrs2 = list(node2.attrib.items())
+    if attrs1 != attrs2:
+        differences.append(
+            f"Attribute mismatch at {path}/{node1.tag}: {attrs1} vs. {attrs2}"
+        )
+
+    # 3) Compare text (trim or unify whitespace as needed)
+    text1 = (node1.text or "").strip()
+    text2 = (node2.text or "").strip()
+    # Normalize whitespace
+    text1 = " ".join(text1.split())
+    text2 = " ".join(text2.split())
+    if text1 != text2:
+        # If you prefer ignoring newlines or multiple whitespace, do a more robust cleanup
+        differences.append(
+            f"Text mismatch at {path}/{node1.tag}: '{text1}' vs. '{text2}'"
+        )
+
+    # 4) Compare number of children
+    children1 = list(node1)
+    children2 = list(node2)
+    if len(children1) != len(children2):
+        differences.append(
+            f"Child count mismatch at {path}/{node1.tag}: {len(children1)} vs. {len(children2)}"
+        )
+        return  # If counts differ, no point comparing child by child
+
+    # 5) Recursively compare each child
+    for i, (c1, c2) in enumerate(zip(children1, children2)):
+        # Build a path for child
+        child_path = f"{path}/{node1.tag}[{i}]"
+        compare_nodes(c1, c2, differences, child_path)
+
+    # 6) Compare tail text
+    tail1 = (node1.tail or "").strip()
+    tail2 = (node2.tail or "").strip()
+    if tail1 != tail2:
+        differences.append(
+            f"Tail mismatch after {path}/{node1.tag}: '{tail1}' vs. '{tail2}'"
+        )
+
+
+def compare_html_structurally(html1, html2):
+    """
+    Compare two HTML strings using a structural approach with lxml.
+    Returns a list of differences (if any). If empty, they're effectively the same.
+    """
+    # 1) Parse both
+    try:
+        tree1 = lhtml.fromstring(html1)
+    except etree.ParserError:
+        return ["Error parsing HTML1"]
+
+    try:
+        tree2 = lhtml.fromstring(html2)
+    except etree.ParserError:
+        return ["Error parsing HTML2"]
+
+    # 2) Normalize both DOMs (remove comments, sort attributes, etc.)
+    tree1 = normalize_dom(tree1)
+    tree2 = normalize_dom(tree2)
+
+    # 3) Possibly strip / wrappers for better apples-to-apples comparison
+    tree1 = strip_html_body(tree1)
+    tree2 = strip_html_body(tree2)
+
+    # 4) Compare recursively
+    differences = []
+    compare_nodes(tree1, tree2, differences, path="")
+    return differences
+
+
+def generate_large_html(n_elements=1000):
+    html = [""]
+    for i in range(n_elements):
+        html.append(
+            f"""
+            
      
+                
      Heading {i}
      
+                
      This is paragraph {i} with some content and a link
      
+                Image {i}
+                
        
+                    
      • List item {i}.1
        • 
+                    
      • List item {i}.2
        • 
+                
      
+            
      
+        """
+        )
+    html.append("")
+    return "".join(html)
+
+
+def generate_complicated_html():
+    """
+    HTML with multiple domains, forms, data attributes,
+    various images, comments, style, and noscript to test all parameter toggles.
+    """
+    return """
+    
+    
+      
+        Complicated Test Page
+        
+        
+        
+      
+      
+        
+        
+        
      
+          
      Main Title of the Page
      
+          
+        
      
+        
+        
+        
+        
      
+          
+          
+        
      
+        
+        
      
+          
      
+            
      Article Title
      
+            
      
+              This paragraph has a good amount of text to exceed word_count_threshold if it's 
+              set to something small. But it might not exceed a very high threshold.
+            
      
+            
+            Descriptive alt text
+            
+            Icon
+            
+            
      Another short text. Local Link
      
+          
      
+        
      
+        
+        
      
+          
      Promo text Ad Link
      
+        
      
+        
+        
+        
+        
+        
+        
+        
      
+          
      This is hidden
      
+        
      
+        
+        
      
+          Footer Info © 2025
+        
      
+      
+    
+    """
+
+
+def get_test_scenarios():
+    """
+    Returns a dictionary of parameter sets (test scenarios) for the scraper.
+    Each scenario name maps to a dictionary of keyword arguments
+    that will be passed into scrap() for testing various features.
+    """
+    TEST_SCENARIOS = {
+        # "default": {},
+        # "exclude_domains": {
+        #     "exclude_domains": {"images.example.com", "ads.example.com"}
+        # },
+        # "exclude_social_media_links": {
+        #     "exclude_social_media_links": True
+        # },
+        # "high_word_threshold": {
+        #     "word_count_threshold": 100
+        # },
+        # "keep_data_attrs": {
+        #     "keep_data_attributes": True
+        # },
+        # "remove_forms_and_comments": {
+        #     "remove_forms": True,
+        #     "remove_comments": True
+        # },
+        # "exclude_tags_and_selector": {
+        #     "excluded_tags": ["aside", "script"],
+        #     "excluded_selector": ".social-widget"
+        # },
+        # "only_text_mode": {
+        #     "only_text": True
+        # },
+        # "combo_mode": {
+        #     "exclude_domains": {"images.example.com", "ads.example.com"},
+        #     "exclude_social_media_links": True,
+        #     "remove_forms": True,
+        #     "remove_comments": True,
+        #     "excluded_tags": ["aside"],
+        #     "excluded_selector": "#promo-section",
+        #     "only_text": False,
+        #     "keep_data_attributes": True,
+        #     "word_count_threshold": 20
+        # },
+        # "exclude_external_images": {
+        #     "exclude_external_images": True,
+        #     "exclude_social_media_links": True
+        # },
+        # "strict_image_scoring": {
+        #     "image_score_threshold": 3,
+        #     "image_description_min_word_threshold": 10
+        # },
+        # "custom_css_selector": {
+        #     "css_selector": "section#promo-section"
+        # },
+        # "remove_noscript": {
+        #     "excluded_tags": ["noscript"]
+        # },
+        # "exclude_external_links": {
+        #     "exclude_external_links": True
+        # },
+        # "large_word_count": {
+        #     "word_count_threshold": 500
+        # },
+        # "super_strict_images": {
+        #     "image_score_threshold": 5,
+        #     "image_description_min_word_threshold": 15
+        # },
+        # "exclude_style_and_script": {
+        #     "excluded_tags": ["style", "script"]
+        # },
+        # "keep_data_and_remove_forms": {
+        #     "keep_data_attributes": True,
+        #     "remove_forms": True
+        # },
+        # "only_text_high_word_count": {
+        #     "only_text": True,
+        #     "word_count_threshold": 40
+        # },
+        # "reduce_to_selector": {
+        #     "css_selector": "section > article"
+        # },
+        # "exclude_all_links": {
+        #     # Removes all external links and also excludes example.com & social.com
+        #     "exclude_domains": {"example.com", "social.com", "facebook.com"},
+        #     "exclude_external_links": True
+        # },
+        # "comprehensive_removal": {
+        #     # Exclude multiple tags, remove forms & comments,
+        #     # and also remove targeted selectors
+        #     "excluded_tags": ["aside", "noscript", "script"],
+        #     "excluded_selector": "#promo-section, .social-widget",
+        #     "remove_comments": True,
+        #     "remove_forms": True
+        # }
+    }
+    return TEST_SCENARIOS
+
+
+class ScraperEquivalenceTester:
+    def __init__(self):
+        self.test_cases = {
+            "basic": self.generate_basic_html(),
+            "complex": self.generate_complex_html(),
+            "malformed": self.generate_malformed_html(),
+            # 'real_world': self.load_real_samples()
+        }
+
+    def generate_basic_html(self):
+        return generate_large_html(1000)  # Your existing function
+
+    def generate_complex_html(self):
+        return """
+        
+            
      
+                
+                
+            
      
+        
+        """
+
+    def generate_malformed_html(self):
+        return """
+        
      Unclosed div
+        
      Unclosed paragraph
+        Link
+        
+        
+        
+        
+        """
+
+    def load_real_samples(self):
+        # Load some real-world HTML samples you've collected
+        samples = {
+            "article": open("tests/samples/article.html").read(),
+            "product": open("tests/samples/product.html").read(),
+            "blog": open("tests/samples/blog.html").read(),
+        }
+        return samples
+
+    def deep_compare_links(self, old_links: Dict, new_links: Dict) -> List[str]:
+        """Detailed comparison of link structures"""
+        differences = []
+
+        for category in ["internal", "external"]:
+            old_urls = {link["href"] for link in old_links[category]}
+            new_urls = {link["href"] for link in new_links[category]}
+
+            missing = old_urls - new_urls
+            extra = new_urls - old_urls
+
+            if missing:
+                differences.append(f"Missing {category} links: {missing}")
+            if extra:
+                differences.append(f"Extra {category} links: {extra}")
+
+            # Compare link attributes for common URLs
+            common = old_urls & new_urls
+            for url in common:
+                old_link = next(l for l in old_links[category] if l["href"] == url)
+                new_link = next(l for l in new_links[category] if l["href"] == url)
+
+                for attr in ["text", "title"]:
+                    if old_link[attr] != new_link[attr]:
+                        differences.append(
+                            f"Link attribute mismatch for {url} - {attr}:"
+                            f" old='{old_link[attr]}' vs new='{new_link[attr]}'"
+                        )
+
+        return differences
+
+    def deep_compare_media(self, old_media: Dict, new_media: Dict) -> List[str]:
+        """Detailed comparison of media elements"""
+        differences = []
+
+        for media_type in ["images", "videos", "audios"]:
+            old_srcs = {item["src"] for item in old_media[media_type]}
+            new_srcs = {item["src"] for item in new_media[media_type]}
+
+            missing = old_srcs - new_srcs
+            extra = new_srcs - old_srcs
+
+            if missing:
+                differences.append(f"Missing {media_type}: {missing}")
+            if extra:
+                differences.append(f"Extra {media_type}: {extra}")
+
+            # Compare media attributes for common sources
+            common = old_srcs & new_srcs
+            for src in common:
+                old_item = next(m for m in old_media[media_type] if m["src"] == src)
+                new_item = next(m for m in new_media[media_type] if m["src"] == src)
+
+                for attr in ["alt", "description"]:
+                    if old_item.get(attr) != new_item.get(attr):
+                        differences.append(
+                            f"{media_type} attribute mismatch for {src} - {attr}:"
+                            f" old='{old_item.get(attr)}' vs new='{new_item.get(attr)}'"
+                        )
+
+        return differences
+
+    def compare_html_content(self, old_html: str, new_html: str) -> List[str]:
+        """Compare HTML content structure and text"""
+        # return compare_html_structurally(old_html, new_html)
+        differences = []
+
+        def normalize_html(html: str) -> Tuple[str, str]:
+            soup = BeautifulSoup(html, "lxml")
+            # Get both structure and text
+            structure = " ".join(tag.name for tag in soup.find_all())
+            text = " ".join(soup.get_text().split())
+            return structure, text
+
+        old_structure, old_text = normalize_html(old_html)
+        new_structure, new_text = normalize_html(new_html)
+
+        # Compare structure
+        if abs(len(old_structure) - len(new_structure)) > 100:
+            # if old_structure != new_structure:
+            diff = difflib.unified_diff(
+                old_structure.split(), new_structure.split(), lineterm=""
+            )
+            differences.append("HTML structure differences:\n" + "\n".join(diff))
+
+        # Compare text content
+        if abs(len(old_text) - len(new_text)) > 100:
+            # if old_text != new_text:
+            # Show detailed text differences
+            text_diff = difflib.unified_diff(
+                old_text.split(), new_text.split(), lineterm=""
+            )
+            differences.append("Text content differences:\n" + "\n".join(text_diff))
+
+        return differences
+
+    def compare_results(
+        self, old_result: Dict, new_result: Dict
+    ) -> Dict[str, List[str]]:
+        """Comprehensive comparison of scraper outputs"""
+        differences = {}
+
+        # Compare links
+        link_differences = self.deep_compare_links(
+            old_result["links"], new_result["links"]
+        )
+        if link_differences:
+            differences["links"] = link_differences
+
+        # Compare media
+        media_differences = self.deep_compare_media(
+            old_result["media"], new_result["media"]
+        )
+        if media_differences:
+            differences["media"] = media_differences
+
+        # Compare HTML
+        html_differences = self.compare_html_content(
+            old_result["cleaned_html"], new_result["cleaned_html"]
+        )
+        if html_differences:
+            differences["html"] = html_differences
+
+        return differences
+
+    def run_tests(self) -> Dict:
+        """Run comparison tests using the complicated HTML with multiple parameter scenarios."""
+        # We'll still keep some "test_cases" logic from above (basic, complex, malformed).
+        # But we add a new section for the complicated HTML scenarios.
+
+        results = {"tests": [], "summary": {"passed": 0, "failed": 0}}
+
+        # 1) First, run the existing 3 built-in test cases (basic, complex, malformed).
+        # for case_name, html in self.test_cases.items():
+        #     print(f"\nTesting built-in case: {case_name}...")
+
+        #     original = WebScrapingStrategy()
+        #     lxml = LXMLWebScrapingStrategy()
+
+        #     start = time.time()
+        #     orig_result = original.scrap("http://test.com", html)
+        #     orig_time = time.time() - start
+
+        #     print("\nOriginal Mode:")
+        #     print(f"Cleaned HTML size: {len(orig_result['cleaned_html'])/1024:.2f} KB")
+        #     print(f"Images: {len(orig_result['media']['images'])}")
+        #     print(f"External links: {len(orig_result['links']['external'])}")
+        #     print(f"Times - Original: {orig_time:.3f}s")
+
+        #     start = time.time()
+        #     lxml_result = lxml.scrap("http://test.com", html)
+        #     lxml_time = time.time() - start
+
+        #     print("\nLXML Mode:")
+        #     print(f"Cleaned HTML size: {len(lxml_result['cleaned_html'])/1024:.2f} KB")
+        #     print(f"Images: {len(lxml_result['media']['images'])}")
+        #     print(f"External links: {len(lxml_result['links']['external'])}")
+        #     print(f"Times - LXML: {lxml_time:.3f}s")
+
+        #     # Compare
+        #     diffs = {}
+        #     link_diff = self.deep_compare_links(orig_result['links'], lxml_result['links'])
+        #     if link_diff:
+        #         diffs['links'] = link_diff
+
+        #     media_diff = self.deep_compare_media(orig_result['media'], lxml_result['media'])
+        #     if media_diff:
+        #         diffs['media'] = media_diff
+
+        #     html_diff = self.compare_html_content(orig_result['cleaned_html'], lxml_result['cleaned_html'])
+        #     if html_diff:
+        #         diffs['html'] = html_diff
+
+        #     test_result = {
+        #         'case': case_name,
+        #         'lxml_mode': {
+        #             'differences': diffs,
+        #             'execution_time': lxml_time
+        #         },
+        #         'original_time': orig_time
+        #     }
+        #     results['tests'].append(test_result)
+
+        #     if not diffs:
+        #         results['summary']['passed'] += 1
+        #     else:
+        #         results['summary']['failed'] += 1
+
+        # 2) Now, run the complicated HTML with multiple parameter scenarios.
+        complicated_html = generate_complicated_html()
+        print("\n=== Testing complicated HTML with multiple parameter scenarios ===")
+
+        # Create the scrapers once (or you can re-create if needed)
+        original = WebScrapingStrategy()
+        lxml = LXMLWebScrapingStrategy()
+
+        for scenario_name, params in get_test_scenarios().items():
+            print(f"\nScenario: {scenario_name}")
+
+            start = time.time()
+            orig_result = original.scrap("http://test.com", complicated_html, **params)
+            orig_time = time.time() - start
+
+            start = time.time()
+            lxml_result = lxml.scrap("http://test.com", complicated_html, **params)
+            lxml_time = time.time() - start
+
+            diffs = {}
+            link_diff = self.deep_compare_links(
+                orig_result["links"], lxml_result["links"]
+            )
+            if link_diff:
+                diffs["links"] = link_diff
+
+            media_diff = self.deep_compare_media(
+                orig_result["media"], lxml_result["media"]
+            )
+            if media_diff:
+                diffs["media"] = media_diff
+
+            html_diff = self.compare_html_content(
+                orig_result["cleaned_html"], lxml_result["cleaned_html"]
+            )
+            if html_diff:
+                diffs["html"] = html_diff
+
+            test_result = {
+                "case": f"complicated_{scenario_name}",
+                "lxml_mode": {"differences": diffs, "execution_time": lxml_time},
+                "original_time": orig_time,
+            }
+            results["tests"].append(test_result)
+
+            if not diffs:
+                results["summary"]["passed"] += 1
+                print(
+                    f"✅ [OK] No differences found. Time(Orig: {orig_time:.3f}s, LXML: {lxml_time:.3f}s)"
+                )
+            else:
+                results["summary"]["failed"] += 1
+                print("❌ Differences found:")
+                for category, dlist in diffs.items():
+                    print(f"  {category}:")
+                    for d in dlist:
+                        print(f"    - {d}")
+
+        return results
+
+    def print_report(self, results: Dict):
+        """Generate detailed equivalence report"""
+        print("\n=== Scraper Equivalence Test Report ===\n")
+        print(f"Total Cases: {len(results['tests'])}")
+        print(f"Passed: {results['summary']['passed']}")
+        print(f"Failed: {results['summary']['failed']}")
+
+        for test in results["tests"]:
+            print(f"\nTest Case: {test['case']}")
+
+            if not test["lxml_mode"]["differences"]:
+                print("✅ All implementations produced identical results")
+                print(
+                    f"Times - Original: {test['original_time']:.3f}s, "
+                    f"LXML: {test['lxml_mode']['execution_time']:.3f}s"
+                )
+            else:
+                print("❌ Differences found:")
+
+                if test["lxml_mode"]["differences"]:
+                    print("\nLXML Mode Differences:")
+                    for category, diffs in test["lxml_mode"]["differences"].items():
+                        print(f"\n{category}:")
+                        for diff in diffs:
+                            print(f"  - {diff}")
+
+
+def main():
+    tester = ScraperEquivalenceTester()
+    results = tester.run_tests()
+    tester.print_report(results)
+
+    # Save detailed results for debugging
+    with open("scraper_equivalence_results.json", "w") as f:
+        json.dump(results, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/async/test_http_file_download.py b/tests/async/test_http_file_download.py
new file mode 100644
index 0000000..6a9d735
--- /dev/null
+++ b/tests/async/test_http_file_download.py
@@ -0,0 +1,423 @@
+"""
+Tests for HTTP strategy file download detection and handling.
+
+Tests the Content-Type/Content-Disposition detection logic in AsyncHTTPCrawlerStrategy
+that saves non-HTML responses to disk and populates downloaded_files.
+"""
+
+import os
+import sys
+import asyncio
+import tempfile
+import shutil
+import json
+import socket
+
+import pytest
+from aiohttp import web
+
+# Add parent to path so crawl4ai is importable
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, parent_dir)
+
+from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
+from crawl4ai.async_configs import HTTPCrawlerConfig, CrawlerRunConfig
+
+
+# ---------------------------------------------------------------------------
+# Test HTTP server
+# ---------------------------------------------------------------------------
+
+async def handle_html(request):
+    return web.Response(text="Hello", content_type="text/html")
+
+async def handle_csv(request):
+    csv_data = "id,name,value\n1,alpha,100\n2,beta,200\n3,gamma,300\n"
+    return web.Response(
+        text=csv_data,
+        content_type="text/csv",
+        headers={"Content-Disposition": 'attachment; filename="data.csv"'},
+    )
+
+async def handle_csv_no_disposition(request):
+    return web.Response(text="col1,col2\na,b\nc,d\n", content_type="text/csv")
+
+async def handle_json(request):
+    return web.Response(
+        text=json.dumps({"key": "value", "items": [1, 2, 3]}),
+        content_type="application/json",
+    )
+
+async def handle_pdf(request):
+    pdf_bytes = b"%PDF-1.4 fake pdf content " + (b"\x00\xff" * 500)
+    return web.Response(
+        body=pdf_bytes,
+        content_type="application/pdf",
+        headers={"Content-Disposition": 'attachment; filename="report.pdf"'},
+    )
+
+async def handle_binary_no_name(request):
+    return web.Response(body=b"\x89PNG\r\n" + b"\x00" * 100, content_type="image/png")
+
+async def handle_plain_text(request):
+    return web.Response(text="Just plain text content.", content_type="text/plain")
+
+async def handle_xml(request):
+    return web.Response(
+        text='test',
+        content_type="application/xml",
+    )
+
+async def handle_attachment_html(request):
+    return web.Response(
+        text="download me",
+        content_type="text/html",
+        headers={"Content-Disposition": 'attachment; filename="page.html"'},
+    )
+
+async def handle_csv_url_filename(request):
+    return web.Response(text="x,y\n1,2\n", content_type="text/csv")
+
+
+def _find_free_port():
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+class _TestServer:
+    """Minimal test server lifecycle manager."""
+
+    def __init__(self):
+        self.port = _find_free_port()
+        self.runner = None
+
+    @property
+    def base_url(self):
+        return f"http://127.0.0.1:{self.port}"
+
+    def url(self, path):
+        return f"{self.base_url}{path}"
+
+    async def start(self):
+        app = web.Application()
+        app.router.add_get("/page.html", handle_html)
+        app.router.add_get("/data.csv", handle_csv)
+        app.router.add_get("/inline.csv", handle_csv_no_disposition)
+        app.router.add_get("/api/data.json", handle_json)
+        app.router.add_get("/report.pdf", handle_pdf)
+        app.router.add_get("/image", handle_binary_no_name)
+        app.router.add_get("/readme.txt", handle_plain_text)
+        app.router.add_get("/feed.xml", handle_xml)
+        app.router.add_get("/attachment.html", handle_attachment_html)
+        app.router.add_get("/files/export.csv", handle_csv_url_filename)
+
+        self.runner = web.AppRunner(app)
+        await self.runner.setup()
+        site = web.TCPSite(self.runner, "127.0.0.1", self.port)
+        await site.start()
+
+    async def stop(self):
+        if self.runner:
+            await self.runner.cleanup()
+
+
+# ---------------------------------------------------------------------------
+# Helper to run an async crawl test
+# ---------------------------------------------------------------------------
+
+async def _crawl(server, path, downloads_dir=None):
+    config = HTTPCrawlerConfig(downloads_path=downloads_dir)
+    strategy = AsyncHTTPCrawlerStrategy(browser_config=config)
+    return await strategy.crawl(server.url(path))
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+class TestHTMLPassthrough:
+    """Normal HTML responses should behave exactly as before."""
+
+    def test_html_response_unchanged(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/page.html", dl_dir)
+
+                assert "" in result.html
+                assert result.downloaded_files is None
+                assert result.status_code == 200
+                assert len(os.listdir(dl_dir)) == 0
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+
+class TestTextFileDownloads:
+    """Text-based file downloads (CSV, JSON, XML, plain text)."""
+
+    def test_csv_with_disposition(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/data.csv", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert len(result.downloaded_files) == 1
+                filepath = result.downloaded_files[0]
+                assert filepath.endswith("data.csv")
+                assert os.path.isfile(filepath)
+                assert "alpha" in result.html
+                assert "id,name,value" in result.html
+                with open(filepath) as f:
+                    assert "alpha" in f.read()
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_csv_without_disposition(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/inline.csv", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert len(result.downloaded_files) == 1
+                assert result.downloaded_files[0].endswith("inline.csv")
+                assert "col1,col2" in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_json_download(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/api/data.json", dl_dir)
+
+                assert result.downloaded_files is not None
+                filepath = result.downloaded_files[0]
+                assert filepath.endswith("data.json")
+                assert '"key"' in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_plain_text(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/readme.txt", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert "Just plain text content." in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_xml_download(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/feed.xml", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert "" in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_csv_filename_from_url(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/files/export.csv", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert result.downloaded_files[0].endswith("export.csv")
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+
+class TestBinaryFileDownloads:
+    """Binary file downloads (PDF, images) — html should be empty."""
+
+    def test_pdf_download(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/report.pdf", dl_dir)
+
+                assert result.downloaded_files is not None
+                filepath = result.downloaded_files[0]
+                assert filepath.endswith("report.pdf")
+                assert os.path.isfile(filepath)
+                assert result.html == ""
+                with open(filepath, "rb") as f:
+                    data = f.read()
+                    assert data.startswith(b"%PDF")
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_binary_no_filename(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/image", dl_dir)
+
+                assert result.downloaded_files is not None
+                filepath = result.downloaded_files[0]
+                assert filepath.endswith(".png")
+                assert os.path.isfile(filepath)
+                assert result.html == ""
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+
+class TestEdgeCases:
+    """Edge cases and backward compatibility."""
+
+    def test_attachment_html_treated_as_download(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/attachment.html", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert result.downloaded_files[0].endswith("page.html")
+                assert "download me" in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_default_downloads_path(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                config = HTTPCrawlerConfig()  # no downloads_path
+                strategy = AsyncHTTPCrawlerStrategy(browser_config=config)
+                result = await strategy.crawl(srv.url("/data.csv"))
+
+                assert result.downloaded_files is not None
+                filepath = result.downloaded_files[0]
+                assert ".crawl4ai/downloads" in filepath
+                if os.path.isfile(filepath):
+                    os.unlink(filepath)
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_response_headers_contain_content_type(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/data.csv", dl_dir)
+                assert "text/csv" in result.response_headers.get("Content-Type", "")
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_status_code_preserved(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/report.pdf", dl_dir)
+                assert result.status_code == 200
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+
+class TestDetectionHelpers:
+    """Unit tests for the detection helper methods."""
+
+    def test_is_file_download(self):
+        s = AsyncHTTPCrawlerStrategy()
+        assert s._is_file_download("text/csv", "") is True
+        assert s._is_file_download("application/pdf", "") is True
+        assert s._is_file_download("image/png", "") is True
+        assert s._is_file_download("text/html", "") is False
+        assert s._is_file_download("text/html", "attachment; filename=x") is True
+        assert s._is_file_download("", "") is False
+
+    def test_is_text_content(self):
+        s = AsyncHTTPCrawlerStrategy()
+        assert s._is_text_content("text/csv") is True
+        assert s._is_text_content("text/plain") is True
+        assert s._is_text_content("application/json") is True
+        assert s._is_text_content("application/pdf") is False
+        assert s._is_text_content("image/png") is False
+        assert s._is_text_content("text/tab-separated-values") is True
+
+    def test_extract_filename_from_disposition(self):
+        s = AsyncHTTPCrawlerStrategy()
+        assert s._extract_filename('attachment; filename="data.csv"', "http://x/y", "text/csv") == "data.csv"
+        assert s._extract_filename("attachment; filename=report.pdf", "http://x/y", "application/pdf") == "report.pdf"
+
+    def test_extract_filename_from_url(self):
+        s = AsyncHTTPCrawlerStrategy()
+        assert s._extract_filename("", "http://example.com/files/export.csv", "text/csv") == "export.csv"
+
+    def test_extract_filename_fallback(self):
+        s = AsyncHTTPCrawlerStrategy()
+        name = s._extract_filename("", "http://example.com/download", "application/pdf")
+        assert name.startswith("download_")
+        assert name.endswith(".pdf")
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_markdown_genertor.py b/tests/async/test_markdown_genertor.py
new file mode 100644
index 0000000..7eaf5d8
--- /dev/null
+++ b/tests/async/test_markdown_genertor.py
@@ -0,0 +1,181 @@
+# ## Issue #236
+# - **Last Updated:** 2024-11-11 01:42:14
+# - **Title:** [user data crawling opens two windows, unable to control correct user browser](https://github.com/unclecode/crawl4ai/issues/236)
+# - **State:** open
+
+import os, sys, time
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+import os
+import time
+from typing import Dict, Any
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+# Get current directory
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+
+def print_test_result(name: str, result: Dict[str, Any], execution_time: float):
+    """Helper function to print test results."""
+    print(f"\n{'='*20} {name} {'='*20}")
+    print(f"Execution time: {execution_time:.4f} seconds")
+
+    # Save markdown to files
+    for key, content in result.items():
+        if isinstance(content, str):
+            with open(__location__ + f"/output/{name.lower()}_{key}.md", "w") as f:
+                f.write(content)
+
+    # # Print first few lines of each markdown version
+    # for key, content in result.items():
+    #     if isinstance(content, str):
+    #         preview = '\n'.join(content.split('\n')[:3])
+    #         print(f"\n{key} (first 3 lines):")
+    #         print(preview)
+    #         print(f"Total length: {len(content)} characters")
+
+
+def test_basic_markdown_conversion():
+    """Test basic markdown conversion with links."""
+    with open(__location__ + "/data/wikipedia.html", "r") as f:
+        cleaned_html = f.read()
+
+    generator = DefaultMarkdownGenerator()
+
+    start_time = time.perf_counter()
+    result = generator.generate_markdown(
+        cleaned_html=cleaned_html, base_url="https://en.wikipedia.org"
+    )
+    execution_time = time.perf_counter() - start_time
+
+    print_test_result(
+        "Basic Markdown Conversion",
+        {
+            "raw": result.raw_markdown,
+            "with_citations": result.markdown_with_citations,
+            "references": result.references_markdown,
+        },
+        execution_time,
+    )
+
+    # Basic assertions
+    assert result.raw_markdown, "Raw markdown should not be empty"
+    assert result.markdown_with_citations, "Markdown with citations should not be empty"
+    assert result.references_markdown, "References should not be empty"
+    assert "⟨" in result.markdown_with_citations, "Citations should use ⟨⟩ brackets"
+    assert (
+        "## References" in result.references_markdown
+    ), "Should contain references section"
+
+
+def test_relative_links():
+    """Test handling of relative links with base URL."""
+    markdown = """
+    Here's a [relative link](/wiki/Apple) and an [absolute link](https://example.com).
+    Also an [image](/images/test.png) and another [page](/wiki/Banana).
+    """
+
+    generator = DefaultMarkdownGenerator()
+    result = generator.generate_markdown(
+        cleaned_html=markdown, base_url="https://en.wikipedia.org"
+    )
+
+    assert "https://en.wikipedia.org/wiki/Apple" in result.references_markdown
+    assert "https://example.com" in result.references_markdown
+    assert "https://en.wikipedia.org/images/test.png" in result.references_markdown
+
+
+def test_duplicate_links():
+    """Test handling of duplicate links."""
+    markdown = """
+    Here's a [link](/test) and another [link](/test) and a [different link](/other).
+    """
+
+    generator = DefaultMarkdownGenerator()
+    result = generator.generate_markdown(
+        cleaned_html=markdown, base_url="https://example.com"
+    )
+
+    # Count citations in markdown
+    citations = result.markdown_with_citations.count("⟨1⟩")
+    assert citations == 2, "Same link should use same citation number"
+
+
+def test_link_descriptions():
+    """Test handling of link titles and descriptions."""
+    markdown = """
+    Here's a [link with title](/test "Test Title") and a [link with description](/other) to test.
+    """
+
+    generator = DefaultMarkdownGenerator()
+    result = generator.generate_markdown(
+        cleaned_html=markdown, base_url="https://example.com"
+    )
+
+    assert (
+        "Test Title" in result.references_markdown
+    ), "Link title should be in references"
+    assert (
+        "link with description" in result.references_markdown
+    ), "Link text should be in references"
+
+
+def test_performance_large_document():
+    """Test performance with large document."""
+    with open(__location__ + "/data/wikipedia.md", "r") as f:
+        markdown = f.read()
+
+    # Test with multiple iterations
+    iterations = 5
+    times = []
+
+    generator = DefaultMarkdownGenerator()
+
+    for i in range(iterations):
+        start_time = time.perf_counter()
+        result = generator.generate_markdown(
+            cleaned_html=markdown, base_url="https://en.wikipedia.org"
+        )
+        end_time = time.perf_counter()
+        times.append(end_time - start_time)
+
+    avg_time = sum(times) / len(times)
+    print(f"\n{'='*20} Performance Test {'='*20}")
+    print(
+        f"Average execution time over {iterations} iterations: {avg_time:.4f} seconds"
+    )
+    print(f"Min time: {min(times):.4f} seconds")
+    print(f"Max time: {max(times):.4f} seconds")
+
+
+def test_image_links():
+    """Test handling of image links."""
+    markdown = """
+    Here's an ![image](/image.png "Image Title") and another ![image](/other.jpg).
+    And a regular [link](/page).
+    """
+
+    generator = DefaultMarkdownGenerator()
+    result = generator.generate_markdown(
+        cleaned_html=markdown, base_url="https://example.com"
+    )
+
+    assert (
+        "![" in result.markdown_with_citations
+    ), "Image markdown syntax should be preserved"
+    assert (
+        "Image Title" in result.references_markdown
+    ), "Image title should be in references"
+
+
+if __name__ == "__main__":
+    print("Running markdown generation strategy tests...")
+
+    test_basic_markdown_conversion()
+    test_relative_links()
+    test_duplicate_links()
+    test_link_descriptions()
+    test_performance_large_document()
+    test_image_links()
diff --git a/tests/async/test_parameters_and_options.py b/tests/async/test_parameters_and_options.py
new file mode 100644
index 0000000..e153fbd
--- /dev/null
+++ b/tests/async/test_parameters_and_options.py
@@ -0,0 +1,116 @@
+import os
+import sys
+import pytest
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_word_count_threshold():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result_no_threshold = await crawler.arun(
+            url=url, word_count_threshold=0, bypass_cache=True
+        )
+        result_with_threshold = await crawler.arun(
+            url=url, word_count_threshold=50, bypass_cache=True
+        )
+
+        assert len(result_no_threshold.markdown) > len(result_with_threshold.markdown)
+
+
+@pytest.mark.asyncio
+async def test_css_selector():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        css_selector = "h1, h2, h3"
+        result = await crawler.arun(
+            url=url, css_selector=css_selector, bypass_cache=True
+        )
+
+        assert result.success
+        assert (
+            " button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+        ]
+        result_with_more = await crawler.arun(url=url, js=js_code, bypass_cache=True)
+
+        assert result_with_more.success
+        assert len(result_with_more.markdown) > len(result_without_more.markdown)
+
+
+@pytest.mark.asyncio
+async def test_screenshot():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, screenshot=True, bypass_cache=True)
+
+        assert result.success
+        assert result.screenshot
+        assert isinstance(result.screenshot, str)  # Should be a base64 encoded string
+
+
+@pytest.mark.asyncio
+async def test_custom_user_agent():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        custom_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Crawl4AI/1.0"
+        result = await crawler.arun(
+            url=url, user_agent=custom_user_agent, bypass_cache=True
+        )
+
+        assert result.success
+        # Note: We can't directly verify the user agent in the result, but we can check if the crawl was successful
+
+
+@pytest.mark.asyncio
+async def test_extract_media_and_links():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+
+        assert result.success
+        assert result.media
+        assert isinstance(result.media, dict)
+        assert "images" in result.media
+        assert result.links
+        assert isinstance(result.links, dict)
+        assert "internal" in result.links and "external" in result.links
+
+
+@pytest.mark.asyncio
+async def test_metadata_extraction():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+
+        assert result.success
+        assert result.metadata
+        assert isinstance(result.metadata, dict)
+        # Check for common metadata fields
+        assert any(
+            key in result.metadata for key in ["title", "description", "keywords"]
+        )
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_performance.py b/tests/async/test_performance.py
new file mode 100644
index 0000000..a35e2be
--- /dev/null
+++ b/tests/async/test_performance.py
@@ -0,0 +1,79 @@
+import os
+import sys
+import pytest
+import time
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_crawl_speed():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        start_time = time.time()
+        result = await crawler.arun(url=url, bypass_cache=True)
+        end_time = time.time()
+
+        assert result.success
+        crawl_time = end_time - start_time
+        print(f"Crawl time: {crawl_time:.2f} seconds")
+
+        assert crawl_time < 10, f"Crawl took too long: {crawl_time:.2f} seconds"
+
+
+@pytest.mark.asyncio
+async def test_concurrent_crawling_performance():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        urls = [
+            "https://www.nbcnews.com/business",
+            "https://www.example.com",
+            "https://www.python.org",
+            "https://www.github.com",
+            "https://www.stackoverflow.com",
+        ]
+
+        start_time = time.time()
+        results = await crawler.arun_many(urls=urls, bypass_cache=True)
+        end_time = time.time()
+
+        total_time = end_time - start_time
+        print(f"Total time for concurrent crawling: {total_time:.2f} seconds")
+
+        assert all(result.success for result in results)
+        assert len(results) == len(urls)
+
+        assert (
+            total_time < len(urls) * 5
+        ), f"Concurrent crawling not significantly faster: {total_time:.2f} seconds"
+
+
+@pytest.mark.asyncio
+async def test_crawl_speed_with_caching():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        start_time = time.time()
+        result1 = await crawler.arun(url=url, bypass_cache=True)
+        end_time = time.time()
+        first_crawl_time = end_time - start_time
+
+        start_time = time.time()
+        result2 = await crawler.arun(url=url, bypass_cache=False)
+        end_time = time.time()
+        second_crawl_time = end_time - start_time
+
+        assert result1.success and result2.success
+        print(f"First crawl time: {first_crawl_time:.2f} seconds")
+        print(f"Second crawl time (cached): {second_crawl_time:.2f} seconds")
+
+        assert (
+            second_crawl_time < first_crawl_time / 2
+        ), "Cached crawl not significantly faster"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async/test_redirect_url_resolution.py b/tests/async/test_redirect_url_resolution.py
new file mode 100644
index 0000000..cce3e51
--- /dev/null
+++ b/tests/async/test_redirect_url_resolution.py
@@ -0,0 +1,118 @@
+"""Test delayed redirect WITH wait_for - does link resolution use correct URL?"""
+import asyncio
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+class RedirectTestHandler(SimpleHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass
+
+    def do_GET(self):
+        if self.path == "/page-a":
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            content = """
+            
+            
+            Page A
+            
+                
      Page A - Will redirect after 200ms
      
+                
+            
+            
+            """
+            self.wfile.write(content.encode())
+        elif self.path.startswith("/redirect-target"):
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            content = """
+            
+            
+            Redirect Target
+            
+                
      Redirect Target
      
+                
+            
+            
+            """
+            self.wfile.write(content.encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+async def main():
+    import socket
+    class ReuseAddrHTTPServer(HTTPServer):
+        allow_reuse_address = True
+    
+    server = ReuseAddrHTTPServer(("localhost", 8769), RedirectTestHandler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.daemon = True
+    thread.start()
+    
+    try:
+        import sys
+        sys.path.insert(0, '/Users/nasrin/vscode/c4ai-uc/develop')
+        from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+        
+        print("=" * 60)
+        print("TEST: Delayed JS redirect WITH wait_for='css:#target-nav'")
+        print("This waits for the redirect to complete")
+        print("=" * 60)
+        
+        browser_config = BrowserConfig(headless=True, verbose=False)
+        crawl_config = CrawlerRunConfig(
+            cache_mode="bypass",
+            wait_for="css:#target-nav"  # Wait for element on redirect target
+        )
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(
+                url="http://localhost:8769/page-a",
+                config=crawl_config
+            )
+            
+            print(f"Original URL: http://localhost:8769/page-a")
+            print(f"Redirected URL returned: {result.redirected_url}")
+            print(f"HTML contains 'Redirect Target': {'Redirect Target' in result.html}")
+            print()
+            
+            if "/redirect-target" in (result.redirected_url or ""):
+                print("✓ redirected_url is CORRECT")
+            else:
+                print("✗ BUG #1: redirected_url is WRONG - still shows original URL!")
+                
+            # Check links
+            all_links = []
+            if isinstance(result.links, dict):
+                all_links = result.links.get("internal", []) + result.links.get("external", [])
+            
+            print(f"\nLinks found ({len(all_links)} total):")
+            bug_found = False
+            for link in all_links:
+                href = link.get("href", "") if isinstance(link, dict) else getattr(link, 'href', "")
+                if "subpage" in href:
+                    print(f"  {href}")
+                    if "/page-a/" in href:
+                        print("    ^^^ BUG #2: Link resolved with WRONG base URL!")
+                        bug_found = True
+                    elif "/redirect-target/" in href:
+                        print("    ^^^ CORRECT")
+            
+            if not bug_found and all_links:
+                print("\n✓ Link resolution is CORRECT")
+                        
+    finally:
+        server.shutdown()
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/async/test_screenshot.py b/tests/async/test_screenshot.py
new file mode 100644
index 0000000..36c6c0a
--- /dev/null
+++ b/tests/async/test_screenshot.py
@@ -0,0 +1,122 @@
+import os
+import sys
+import pytest
+import base64
+from PIL import Image
+import io
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_basic_screenshot():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://example.com"  # A static website
+        result = await crawler.arun(url=url, bypass_cache=True, screenshot=True)
+
+        assert result.success
+        assert result.screenshot is not None
+
+        # Verify the screenshot is a valid image
+        image_data = base64.b64decode(result.screenshot)
+        image = Image.open(io.BytesIO(image_data))
+        assert image.format == "PNG"
+
+
+@pytest.mark.asyncio
+async def test_screenshot_with_wait_for():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        # Using a website with dynamic content
+        url = "https://www.youtube.com"
+        wait_for = "css:#content"  # Wait for the main content to load
+
+        result = await crawler.arun(
+            url=url, bypass_cache=True, screenshot=True, wait_for=wait_for
+        )
+
+        assert result.success
+        assert result.screenshot is not None
+
+        # Verify the screenshot is a valid image
+        image_data = base64.b64decode(result.screenshot)
+        image = Image.open(io.BytesIO(image_data))
+        assert image.format == "PNG"
+
+        # You might want to add more specific checks here, like image dimensions
+        # or even use image recognition to verify certain elements are present
+
+
+@pytest.mark.asyncio
+async def test_screenshot_with_js_wait_for():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.amazon.com"
+        wait_for = "js:() => document.querySelector('#nav-logo-sprites') !== null"
+
+        result = await crawler.arun(
+            url=url, bypass_cache=True, screenshot=True, wait_for=wait_for
+        )
+
+        assert result.success
+        assert result.screenshot is not None
+
+        image_data = base64.b64decode(result.screenshot)
+        image = Image.open(io.BytesIO(image_data))
+        assert image.format == "PNG"
+
+
+@pytest.mark.asyncio
+async def test_screenshot_without_wait_for():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nytimes.com"  # A website with lots of dynamic content
+
+        result = await crawler.arun(url=url, bypass_cache=True, screenshot=True)
+
+        assert result.success
+        assert result.screenshot is not None
+
+        image_data = base64.b64decode(result.screenshot)
+        image = Image.open(io.BytesIO(image_data))
+        assert image.format == "PNG"
+
+
+@pytest.mark.asyncio
+async def test_screenshot_comparison():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.reddit.com"
+        wait_for = "css:#SHORTCUT_FOCUSABLE_DIV"
+
+        # Take screenshot without wait_for
+        result_without_wait = await crawler.arun(
+            url=url, bypass_cache=True, screenshot=True
+        )
+
+        # Take screenshot with wait_for
+        result_with_wait = await crawler.arun(
+            url=url, bypass_cache=True, screenshot=True, wait_for=wait_for
+        )
+
+        assert result_without_wait.success and result_with_wait.success
+        assert result_without_wait.screenshot is not None
+        assert result_with_wait.screenshot is not None
+
+        # Compare the two screenshots
+        image_without_wait = Image.open(
+            io.BytesIO(base64.b64decode(result_without_wait.screenshot))
+        )
+        image_with_wait = Image.open(
+            io.BytesIO(base64.b64decode(result_with_wait.screenshot))
+        )
+
+        # This is a simple size comparison. In a real-world scenario, you might want to use
+        # more sophisticated image comparison techniques.
+        assert image_with_wait.size[0] >= image_without_wait.size[0]
+        assert image_with_wait.size[1] >= image_without_wait.size[1]
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/async_assistant/test_extract_pipeline.py b/tests/async_assistant/test_extract_pipeline.py
new file mode 100644
index 0000000..719d6ea
--- /dev/null
+++ b/tests/async_assistant/test_extract_pipeline.py
@@ -0,0 +1,381 @@
+"""
+Test implementation of AI Assistant extract pipeline using only Crawl4AI capabilities.
+This follows the exact flow discussed: query enhancement, classification, HTML skimming,
+parent extraction, schema generation, and extraction.
+"""
+
+import asyncio
+import json
+import os
+from typing import List, Dict, Any, Optional, Union
+from lxml import html as lxml_html
+import re
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.async_configs import LLMConfig
+from crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategy
+from crawl4ai.utils import perform_completion_with_backoff
+
+
+async def extract_pipeline(
+    base_url: str,
+    urls: Union[str, List[str], None],
+    query: str,
+    target_json_example: Optional[str] = None,
+    force_llm: bool = False,
+    verbose: bool = True
+) -> Union[Dict, List[Dict]]:
+    """
+    Full implementation of the AI-powered extraction pipeline using only Crawl4AI.
+    
+    Pipeline:
+    1. Quick crawl & HTML skimming
+    2. Classification (structural vs semantic) using LLM
+    3. Parent element extraction using LLM (for structural)
+    4. Schema generation using Crawl4AI's generate_schema
+    5. Extraction execution using Crawl4AI strategies
+    """
+    
+    # Normalize URLs
+    if urls is None:
+        urls = base_url
+    target_urls = [urls] if isinstance(urls, str) else urls
+    single_result = isinstance(urls, str) or urls is None
+    
+    # LLM configs for different tasks
+    llm_small = LLMConfig(
+        provider="openai/gpt-4o-mini",
+        api_token=os.getenv("OPENAI_API_KEY")
+    )
+    llm_small.temperature = 0.3
+    
+    llm_strong = LLMConfig(
+        provider="openai/gpt-4o",
+        api_token=os.getenv("OPENAI_API_KEY")
+    )
+    llm_strong.temperature = 0.5
+    
+    def vprint(msg: str):
+        if verbose:
+            print(f"🔍 {msg}")
+    
+    # Step 1: Starting
+    vprint(f"Query: '{query}'")
+    
+    # Step 2: Quick crawl for analysis
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        vprint(f"Quick crawl: {base_url}")
+        quick_result = await crawler.arun(
+            url=base_url,
+            config=CrawlerRunConfig(
+                cache_mode="bypass",
+                delay_before_return_html=2.0
+            )
+        )
+        
+        if not quick_result.success:
+            raise Exception(f"Failed to crawl {base_url}")
+        
+        # Step 3: HTML Skimming using lxml
+        def skim_html(html: str) -> str:
+            """Remove non-structural elements using lxml."""
+            parser = lxml_html.HTMLParser(remove_comments=True)
+            tree = lxml_html.fromstring(html, parser=parser)
+            
+            # Remove head section entirely
+            for head in tree.xpath('//head'):
+                head.getparent().remove(head)
+            
+            # Remove non-structural elements including SVGs
+            for element in tree.xpath('//script | //style | //noscript | //meta | //link | //svg'):
+                parent = element.getparent()
+                if parent is not None:
+                    parent.remove(element)
+            
+            # Remove base64 images
+            for img in tree.xpath('//img[@src]'):
+                src = img.get('src', '')
+                if 'base64' in src:
+                    img.set('src', 'BASE64_IMAGE')
+            
+            # Remove long class/id attributes
+            for element in tree.xpath('//*[@class or @id]'):
+                if element.get('class') and len(element.get('class')) > 100:
+                    element.set('class', 'LONG_CLASS')
+                if element.get('id') and len(element.get('id')) > 50:
+                    element.set('id', 'LONG_ID')
+            
+            # Truncate text nodes
+            for text_node in tree.xpath('//text()'):
+                if text_node.strip() and len(text_node) > 100:
+                    parent = text_node.getparent()
+                    if parent is not None:
+                        new_text = text_node[:50] + "..." + text_node[-20:]
+                        if text_node.is_text:
+                            parent.text = new_text
+                        elif text_node.is_tail:
+                            parent.tail = new_text
+            
+            return lxml_html.tostring(tree, encoding='unicode')
+        
+        skimmed_html = skim_html(quick_result.html)
+        vprint(f"Skimmed HTML from {len(quick_result.html)} to {len(skimmed_html)} chars")
+        
+        # Step 4: Classification using LLM
+        classification = 'semantic'  # Default
+        
+        if not force_llm:
+            classification_prompt = f"""
+            Analyze this HTML to determine extraction strategy.
+            
+            Query: "{query}"
+            
+            HTML sample:
+            <<<>>
+            {skimmed_html}
+            <<<>>
+            
+            Determine if this can be extracted using CSS/XPath patterns (structural) 
+            or requires semantic understanding (semantic).
+            
+            Look for:
+            - Repeating patterns (lists, cards, tables) → structural
+            - Consistent HTML structure → structural
+            - Need for inference or understanding → semantic
+            
+            Return JSON:
+            {{
+                "strategy": "structural" or "semantic",
+                "confidence": 0.0-1.0,
+                "reasoning": "..."
+            }}
+            """
+            
+            response = perform_completion_with_backoff(
+                provider=llm_small.provider,
+                prompt_with_variables=classification_prompt,
+                api_token=llm_small.api_token,
+                json_response=True,
+                temperature=llm_small.temperature
+            )
+            
+            classification_result = json.loads(response.choices[0].message.content)
+            classification = classification_result['strategy']
+            vprint(f"Classification: {classification} (confidence: {classification_result['confidence']})")
+            vprint(f"Reasoning: {classification_result['reasoning']}")
+        
+        if force_llm:
+            classification = 'semantic'
+            vprint("Forced LLM extraction")
+        
+        # Step 5 & 6: Execute appropriate extraction strategy
+        if classification == 'structural':
+            # Extract parent element using LLM with proper explanation
+            parent_prompt = f"""
+            Identify the CSS selector for the BASE ELEMENT TEMPLATE containing the data to extract.
+            
+            IMPORTANT: The base element template is a repeating pattern in the HTML where each instance
+            contains one item of data (like a product card, article card, issue card, etc.). 
+            
+            The selector should:
+            - Not be too specific (avoid selecting just one item)
+            - Not be too general (avoid selecting unrelated elements)
+            - Select ALL instances of the repeating pattern
+            - Point to the container that holds ONE complete data item
+            
+            For example:
+            - On Amazon: div.s-result-item (each product card)
+            - On GitHub issues: div[id^="issue_"] (each issue card)
+            - On a blog: article.post-card (each article)
+            
+            User query: "{query}"
+            """
+            
+            if target_json_example:
+                parent_prompt += f"""
+            
+            The user expects to extract data in this format:
+            {target_json_example}
+            
+            Find the base element that contains all these fields.
+            """
+            else:
+                parent_prompt += """
+            
+            Also provide a JSON example of what data can be extracted from one instance of this base element.
+            """
+            
+            parent_prompt += f"""
+            
+            HTML (first 8000 chars):
+            <<<>>
+            {skimmed_html}
+            <<<>>
+            
+            Return JSON:
+            {{
+                "parent_selector": "css_selector_here",
+                "explanation": "why this selector is appropriate","""
+            
+            if not target_json_example:
+                parent_prompt += """
+                "suggested_json_example": {
+                    "field1": "example value",
+                    "field2": "example value"
+                }"""
+            
+            parent_prompt += """
+            }}
+            """
+            
+            response = perform_completion_with_backoff(
+                provider=llm_small.provider,
+                prompt_with_variables=parent_prompt,
+                api_token=llm_small.api_token,
+                json_response=True,
+                temperature=llm_small.temperature
+            )
+            
+            parent_data = json.loads(response.choices[0].message.content)
+            parent_selector = parent_data['parent_selector']
+            vprint(f"Parent selector: {parent_selector}")
+            vprint(f"Explanation: {parent_data['explanation']}")
+            
+            # Use suggested JSON example if no target provided
+            if not target_json_example and 'suggested_json_example' in parent_data:
+                target_json_example = json.dumps(parent_data['suggested_json_example'])
+                vprint(f"Using LLM suggested example: {target_json_example}")
+            
+            # Get the actual parent HTML for schema generation
+            tree = lxml_html.fromstring(quick_result.html)
+            parent_elements = tree.cssselect(parent_selector)
+            
+            if not parent_elements:
+                vprint("Parent selector not found, falling back to semantic")
+                classification = 'semantic'
+            else:
+                # Use the first instance as sample
+                sample_html = lxml_html.tostring(parent_elements[0], encoding='unicode')
+                vprint(f"Generating schema from sample HTML ({len(sample_html)} chars)")
+                
+                # Generate schema using Crawl4AI
+                schema_params = {
+                    "html": sample_html,
+                    "query": query,
+                    "llm_config": llm_strong
+                }
+                
+                if target_json_example:
+                    schema_params["target_json_example"] = target_json_example
+                
+                schema = JsonCssExtractionStrategy.generate_schema(**schema_params)
+                
+                vprint(f"Generated schema with {len(schema.get('fields', []))} fields")
+                
+                # Extract from all URLs
+                extraction_strategy = JsonCssExtractionStrategy(schema)
+                results = []
+                
+                for url in target_urls:
+                    vprint(f"Extracting from: {url}")
+                    result = await crawler.arun(
+                        url=url,
+                        config=CrawlerRunConfig(
+                            extraction_strategy=extraction_strategy,
+                            cache_mode="bypass"
+                        )
+                    )
+                    
+                    if result.success and result.extracted_content:
+                        data = json.loads(result.extracted_content)
+                        results.append({
+                            'url': url,
+                            'data': data,
+                            'count': len(data) if isinstance(data, list) else 1,
+                            'method': 'JsonCssExtraction',
+                            'schema': schema
+                        })
+                
+                return results[0] if single_result else results
+        
+        # Semantic extraction (LLM)
+        if classification == 'semantic':
+            vprint("Using LLM extraction")
+            
+            # Build instruction from query
+            instruction = f"""
+            {query}
+            
+            Return structured JSON data.
+            """
+            
+            extraction_strategy = LLMExtractionStrategy(
+                llm_config=llm_strong,
+                instruction=instruction
+            )
+            
+            results = []
+            for url in target_urls:
+                vprint(f"LLM extracting from: {url}")
+                result = await crawler.arun(
+                    url=url,
+                    config=CrawlerRunConfig(
+                        extraction_strategy=extraction_strategy,
+                        cache_mode="bypass"
+                    )
+                )
+                
+                if result.success and result.extracted_content:
+                    data = json.loads(result.extracted_content)
+                    results.append({
+                        'url': url,
+                        'data': data,
+                        'count': len(data) if isinstance(data, list) else 1,
+                        'method': 'LLMExtraction'
+                    })
+            
+            return results[0] if single_result else results
+
+
+async def main():
+    """Test the extraction pipeline."""
+    
+    print("\n🚀 CRAWL4AI EXTRACTION PIPELINE TEST")
+    print("="*50)
+    
+    # Test structural extraction
+    try:
+        result = await extract_pipeline(
+            base_url="https://github.com/unclecode/crawl4ai/issues",
+            urls=None,
+            query="I want to extract all issue titles, numbers, and who opened them",
+            verbose=True
+        )
+        
+        print(f"\n✅ Success! Extracted {result.get('count', 0)} items")
+        print(f"Method used: {result.get('method')}")
+        
+        if result.get('data'):
+            print("\nFirst few items:")
+            data = result['data']
+            items_to_show = data[:3] if isinstance(data, list) else data
+            print(json.dumps(items_to_show, indent=2))
+            
+            if result.get('schema'):
+                print(f"\nGenerated schema fields: {[f['name'] for f in result['schema'].get('fields', [])]}")
+    
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    # Check for API key
+    if not os.getenv("OPENAI_API_KEY"):
+        print("⚠️  Error: OPENAI_API_KEY environment variable not set")
+        exit(1)
+    
+    asyncio.run(main())
+    
+    
diff --git a/tests/async_assistant/test_extract_pipeline_v2.py b/tests/async_assistant/test_extract_pipeline_v2.py
new file mode 100644
index 0000000..bb65df8
--- /dev/null
+++ b/tests/async_assistant/test_extract_pipeline_v2.py
@@ -0,0 +1,386 @@
+"""
+Test implementation v2: Combined classification and preparation in one LLM call.
+More efficient approach that reduces token usage and LLM calls.
+"""
+
+import asyncio
+import json
+import os
+from typing import List, Dict, Any, Optional, Union
+from lxml import html as lxml_html
+import re
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.async_configs import LLMConfig
+from crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategy
+from crawl4ai.utils import perform_completion_with_backoff
+
+
+async def extract_pipeline_v2(
+    base_url: str,
+    urls: Union[str, List[str], None],
+    query: str,
+    target_json_example: Optional[str] = None,
+    force_llm: bool = False,
+    verbose: bool = True
+) -> Union[Dict, List[Dict]]:
+    """
+    Improved extraction pipeline with combined classification and preparation.
+    
+    Pipeline:
+    1. Quick crawl & HTML skimming
+    2. Combined LLM call for classification + preparation
+    3. Execute appropriate extraction strategy
+    """
+    
+    # Normalize URLs
+    if urls is None:
+        urls = base_url
+    target_urls = [urls] if isinstance(urls, str) else urls
+    single_result = isinstance(urls, str) or urls is None
+    
+    # LLM configs
+    llm_small = LLMConfig(
+        provider="openai/gpt-4o-mini",
+        api_token=os.getenv("OPENAI_API_KEY")
+    )
+    llm_small.temperature = 0.3
+    
+    llm_strong = LLMConfig(
+        provider="openai/gpt-4o",
+        api_token=os.getenv("OPENAI_API_KEY")
+    )
+    llm_strong.temperature = 0.5
+    
+    def vprint(msg: str):
+        if verbose:
+            print(f"🔍 {msg}")
+    
+    vprint(f"Query: '{query}'")
+    if target_json_example:
+        vprint(f"Target format provided: {target_json_example[:100]}...")
+    
+    # Step 1: Quick crawl for analysis
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        vprint(f"Quick crawl: {base_url}")
+        quick_result = await crawler.arun(
+            url=base_url,
+            config=CrawlerRunConfig(
+                cache_mode="bypass",
+                delay_before_return_html=2.0
+            )
+        )
+        
+        if not quick_result.success:
+            raise Exception(f"Failed to crawl {base_url}")
+        
+        # HTML Skimming
+        def skim_html(html: str) -> str:
+            """Remove non-structural elements using lxml."""
+            parser = lxml_html.HTMLParser(remove_comments=True)
+            tree = lxml_html.fromstring(html, parser=parser)
+            
+            # Remove head section entirely
+            for head in tree.xpath('//head'):
+                head.getparent().remove(head)
+            
+            # Remove non-structural elements including SVGs
+            for element in tree.xpath('//script | //style | //noscript | //meta | //link | //svg'):
+                parent = element.getparent()
+                if parent is not None:
+                    parent.remove(element)
+            
+            # Remove base64 images
+            for img in tree.xpath('//img[@src]'):
+                src = img.get('src', '')
+                if 'base64' in src:
+                    img.set('src', 'BASE64_IMAGE')
+            
+            # Remove long class/id attributes
+            for element in tree.xpath('//*[@class or @id]'):
+                if element.get('class') and len(element.get('class')) > 100:
+                    element.set('class', 'LONG_CLASS')
+                if element.get('id') and len(element.get('id')) > 50:
+                    element.set('id', 'LONG_ID')
+            
+            # Truncate text nodes
+            for text_node in tree.xpath('//text()'):
+                if text_node.strip() and len(text_node) > 100:
+                    parent = text_node.getparent()
+                    if parent is not None:
+                        new_text = text_node[:50] + "..." + text_node[-20:]
+                        if text_node.is_text:
+                            parent.text = new_text
+                        elif text_node.is_tail:
+                            parent.tail = new_text
+            
+            return lxml_html.tostring(tree, encoding='unicode')
+        
+        skimmed_html = skim_html(quick_result.html)
+        vprint(f"Skimmed HTML from {len(quick_result.html)} to {len(skimmed_html)} chars")
+        
+        # Step 2: Combined classification and preparation
+        if force_llm:
+            classification_data = {"classification": "semantic"}
+            vprint("Forced LLM extraction")
+        else:
+            combined_prompt = f"""
+            Analyze this HTML and prepare for data extraction.
+            
+            User query: "{query}"
+            """
+            
+            if target_json_example:
+                combined_prompt += f"""
+            Target format: {target_json_example}
+            """
+            
+            combined_prompt += f"""
+            
+            HTML:
+            <<<>>>
+            {skimmed_html}
+            <<<>>>
+            
+            STEP 1: Determine extraction strategy
+            - If data follows repeating HTML patterns (lists, tables, cards) → "structural"
+            - If data requires understanding/inference → "semantic"
+            
+            STEP 2A: If STRUCTURAL extraction is appropriate:
+            - Find the CSS selector for the BASE ELEMENT (repeating pattern)
+            - Base element = container holding ONE data item (e.g., product card, table row)
+            - Selector should select ALL instances, not too specific, not too general
+            - Count approximate number of these elements
+            """
+            
+            if not target_json_example:
+                combined_prompt += """
+            - Suggest what JSON structure can be extracted from one element
+            """
+            
+            combined_prompt += """
+            
+            STEP 2B: If SEMANTIC extraction is needed:
+            - Write a detailed instruction for what to extract
+            - Be specific about the data needed
+            """
+            
+            if not target_json_example:
+                combined_prompt += """
+            - Suggest expected JSON output structure
+            """
+            
+            combined_prompt += """
+            
+            Return JSON with ONLY the relevant fields based on classification:
+            {
+                "classification": "structural" or "semantic",
+                "confidence": 0.0-1.0,
+                "reasoning": "brief explanation",
+                
+                // Include ONLY if classification is "structural":
+                "base_selector": "css selector",
+                "element_count": approximate number,
+                
+                // Include ONLY if classification is "semantic":
+                "extraction_instruction": "detailed instruction",
+                
+                // Include if no target_json_example was provided:
+                "suggested_json_example": { ... }
+            }
+            """
+            
+            response = perform_completion_with_backoff(
+                provider=llm_small.provider,
+                prompt_with_variables=combined_prompt,
+                api_token=llm_small.api_token,
+                json_response=True,
+                temperature=llm_small.temperature
+            )
+            
+            classification_data = json.loads(response.choices[0].message.content)
+            vprint(f"Classification: {classification_data['classification']} (confidence: {classification_data['confidence']})")
+            vprint(f"Reasoning: {classification_data['reasoning']}")
+        
+        # Use suggested JSON example if needed
+        if not target_json_example and 'suggested_json_example' in classification_data:
+            target_json_example = json.dumps(classification_data['suggested_json_example'])
+            vprint(f"Using suggested example: {target_json_example}")
+        
+        # Step 3: Execute extraction based on classification
+        if classification_data['classification'] == 'structural':
+            vprint(f"Base selector: {classification_data['base_selector']}")
+            vprint(f"Found ~{classification_data['element_count']} elements")
+            
+            # Get sample HTML for schema generation
+            tree = lxml_html.fromstring(quick_result.html)
+            parent_elements = tree.cssselect(classification_data['base_selector'])
+            
+            if not parent_elements:
+                vprint("Base selector not found, falling back to semantic")
+                classification_data['classification'] = 'semantic'
+            else:
+                # Use first element as sample
+                sample_html = lxml_html.tostring(parent_elements[0], encoding='unicode')
+                vprint(f"Generating schema from sample ({len(sample_html)} chars)")
+                
+                # Generate schema
+                schema_params = {
+                    "html": sample_html,
+                    "query": query,
+                    "llm_config": llm_strong
+                }
+                
+                if target_json_example:
+                    schema_params["target_json_example"] = target_json_example
+                
+                schema = JsonCssExtractionStrategy.generate_schema(**schema_params)
+                vprint(f"Generated schema with {len(schema.get('fields', []))} fields")
+                
+                # Extract from all URLs
+                extraction_strategy = JsonCssExtractionStrategy(schema)
+                results = []
+                
+                for idx, url in enumerate(target_urls):
+                    vprint(f"Extracting from: {url}")
+                    
+                    # Use already crawled HTML for base_url, crawl others
+                    if idx == 0 and url == base_url:
+                        # We already have this HTML, use raw:// to avoid re-crawling
+                        raw_url = f"raw://{quick_result.html}"
+                        vprint("Using cached HTML with raw:// scheme")
+                    else:
+                        # Need to crawl this URL
+                        raw_url = url
+                    
+                    result = await crawler.arun(
+                        url=raw_url,
+                        config=CrawlerRunConfig(
+                            extraction_strategy=extraction_strategy,
+                            cache_mode="bypass"
+                        )
+                    )
+                    
+                    if result.success and result.extracted_content:
+                        data = json.loads(result.extracted_content)
+                        results.append({
+                            'url': url,  # Keep original URL for reference
+                            'data': data,
+                            'count': len(data) if isinstance(data, list) else 1,
+                            'method': 'JsonCssExtraction',
+                            'schema': schema
+                        })
+                
+                return results[0] if single_result else results
+        
+        # Semantic extraction
+        if classification_data['classification'] == 'semantic':
+            vprint("Using LLM extraction")
+            
+            # Use generated instruction or create simple one
+            if 'extraction_instruction' in classification_data:
+                instruction = classification_data['extraction_instruction']
+                vprint(f"Generated instruction: {instruction[:100]}...")
+            else:
+                instruction = f"{query}\n\nReturn structured JSON data."
+            
+            extraction_strategy = LLMExtractionStrategy(
+                llm_config=llm_strong,
+                instruction=instruction
+            )
+            
+            results = []
+            for idx, url in enumerate(target_urls):
+                vprint(f"LLM extracting from: {url}")
+                
+                # Use already crawled HTML for base_url, crawl others
+                if idx == 0 and url == base_url:
+                    # We already have this HTML, use raw:// to avoid re-crawling
+                    raw_url = f"raw://{quick_result.html}"
+                    vprint("Using cached HTML with raw:// scheme")
+                else:
+                    # Need to crawl this URL
+                    raw_url = url
+                
+                result = await crawler.arun(
+                    url=raw_url,
+                    config=CrawlerRunConfig(
+                        extraction_strategy=extraction_strategy,
+                        cache_mode="bypass"
+                    )
+                )
+                
+                if result.success and result.extracted_content:
+                    data = json.loads(result.extracted_content)
+                    results.append({
+                        'url': url,  # Keep original URL for reference
+                        'data': data,
+                        'count': len(data) if isinstance(data, list) else 1,
+                        'method': 'LLMExtraction'
+                    })
+            
+            return results[0] if single_result else results
+
+
+async def main():
+    """Test the improved extraction pipeline."""
+    
+    print("\n🚀 CRAWL4AI EXTRACTION PIPELINE V2 TEST")
+    print("="*50)
+    
+    try:
+        # Test 1: Structural extraction (GitHub issues)
+        print("\nTest 1: GitHub Issues (should use structural)")
+        result = await extract_pipeline_v2(
+            base_url="https://github.com/unclecode/crawl4ai/issues",
+            urls=None,
+            query="Extract all issue titles, numbers, and authors",
+            verbose=True
+        )
+        
+        print(f"\n✅ Extracted {result.get('count', 0)} items using {result.get('method')}")
+        if result.get('data'):
+            print("Sample:", json.dumps(result['data'][:2] if isinstance(result['data'], list) else result['data'], indent=2))
+        
+        # Test 2: With target JSON example
+        print("\n\nTest 2: With target JSON example")
+        target_example = json.dumps({
+            "title": "Issue title here",
+            "number": "#123",
+            "author": "username"
+        })
+        
+        result2 = await extract_pipeline_v2(
+            base_url="https://github.com/unclecode/crawl4ai/issues",
+            urls=None,
+            query="Extract GitHub issues",
+            target_json_example=target_example,
+            verbose=True
+        )
+        
+        print(f"\n✅ Extracted {result2.get('count', 0)} items")
+        
+        # Test 3: Semantic extraction (force LLM)
+        print("\n\nTest 3: Force semantic extraction")
+        result3 = await extract_pipeline_v2(
+            base_url="https://en.wikipedia.org/wiki/Artificial_intelligence",
+            urls=None,
+            query="Extract key concepts and their relationships in AI field",
+            force_llm=True,
+            verbose=True
+        )
+        
+        print(f"\n✅ Extracted using {result3.get('method')}")
+        
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    if not os.getenv("OPENAI_API_KEY"):
+        print("⚠️  Error: OPENAI_API_KEY environment variable not set")
+        exit(1)
+    
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/browser/docker/__init__.py b/tests/browser/docker/__init__.py
new file mode 100644
index 0000000..b86e573
--- /dev/null
+++ b/tests/browser/docker/__init__.py
@@ -0,0 +1,4 @@
+"""Docker browser strategy tests.
+
+This package contains tests for the Docker browser strategy implementation.
+"""
\ No newline at end of file
diff --git a/tests/browser/docker/test_docker_browser.py b/tests/browser/docker/test_docker_browser.py
new file mode 100644
index 0000000..2ec64a6
--- /dev/null
+++ b/tests/browser/docker/test_docker_browser.py
@@ -0,0 +1,651 @@
+"""Test examples for Docker Browser Strategy.
+
+These examples demonstrate the functionality of Docker Browser Strategy
+and serve as functional tests.
+"""
+
+import asyncio
+import os
+import sys
+import shutil
+import uuid
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../..')))
+
+from crawl4ai.browser import BrowserManager
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+from crawl4ai.browser import DockerConfig
+from crawl4ai.browser import DockerRegistry
+from crawl4ai.browser import DockerUtils
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+# Global Docker utils instance
+docker_utils = DockerUtils(logger)
+
+async def test_docker_components():
+    """Test Docker utilities, registry, and image building.
+    
+    This function tests the core Docker components before running the browser tests.
+    It validates DockerRegistry, DockerUtils, and builds test images to ensure
+    everything is functioning correctly.
+    """
+    logger.info("Testing Docker components", tag="SETUP")
+    
+    # Create a test registry directory
+    registry_dir = os.path.join(os.path.dirname(__file__), "test_registry")
+    registry_file = os.path.join(registry_dir, "test_registry.json")
+    os.makedirs(registry_dir, exist_ok=True)
+    
+    try:
+        # 1. Test DockerRegistry
+        logger.info("Testing DockerRegistry...", tag="SETUP")
+        registry = DockerRegistry(registry_file)
+        
+        # Test saving and loading registry
+        test_container_id = "test-container-123"
+        registry.register_container(test_container_id, 9876, "test-hash-123")
+        registry.save()
+        
+        # Create a new registry instance that loads from the file
+        registry2 = DockerRegistry(registry_file)
+        port = registry2.get_container_host_port(test_container_id)
+        hash_value = registry2.get_container_config_hash(test_container_id)
+        
+        if port != 9876 or hash_value != "test-hash-123":
+            logger.error("DockerRegistry persistence failed", tag="SETUP")
+            return False
+            
+        # Clean up test container from registry
+        registry2.unregister_container(test_container_id)
+        logger.success("DockerRegistry works correctly", tag="SETUP")
+        
+        # 2. Test DockerUtils
+        logger.info("Testing DockerUtils...", tag="SETUP")
+        
+        # Test port detection
+        in_use = docker_utils.is_port_in_use(22)  # SSH port is usually in use
+        logger.info(f"Port 22 in use: {in_use}", tag="SETUP")
+        
+        # Get next available port
+        available_port = docker_utils.get_next_available_port(9000)
+        logger.info(f"Next available port: {available_port}", tag="SETUP")
+        
+        # Test config hash generation
+        config_dict = {"mode": "connect", "headless": True}
+        config_hash = docker_utils.generate_config_hash(config_dict)
+        logger.info(f"Generated config hash: {config_hash[:8]}...", tag="SETUP")
+        
+        # 3. Test Docker is available
+        logger.info("Checking Docker availability...", tag="SETUP")
+        if not await check_docker_available():
+            logger.error("Docker is not available - cannot continue tests", tag="SETUP")
+            return False
+            
+        # 4. Test building connect image
+        logger.info("Building connect mode Docker image...", tag="SETUP")
+        connect_image = await docker_utils.ensure_docker_image_exists(None, "connect")
+        if not connect_image:
+            logger.error("Failed to build connect mode image", tag="SETUP")
+            return False
+        logger.success(f"Successfully built connect image: {connect_image}", tag="SETUP")
+        
+        # 5. Test building launch image
+        logger.info("Building launch mode Docker image...", tag="SETUP")
+        launch_image = await docker_utils.ensure_docker_image_exists(None, "launch")
+        if not launch_image:
+            logger.error("Failed to build launch mode image", tag="SETUP")
+            return False
+        logger.success(f"Successfully built launch image: {launch_image}", tag="SETUP")
+        
+        # 6. Test creating and removing container
+        logger.info("Testing container creation and removal...", tag="SETUP")
+        container_id = await docker_utils.create_container(
+            image_name=launch_image,
+            host_port=available_port,
+            container_name="crawl4ai-test-container"
+        )
+        
+        if not container_id:
+            logger.error("Failed to create test container", tag="SETUP")
+            return False
+            
+        logger.info(f"Created test container: {container_id[:12]}", tag="SETUP")
+        
+        # Verify container is running
+        running = await docker_utils.is_container_running(container_id)
+        if not running:
+            logger.error("Test container is not running", tag="SETUP")
+            await docker_utils.remove_container(container_id)
+            return False
+            
+        # Test commands in container
+        logger.info("Testing command execution in container...", tag="SETUP")
+        returncode, stdout, stderr = await docker_utils.exec_in_container(
+            container_id, ["ls", "-la", "/"]
+        )
+        
+        if returncode != 0:
+            logger.error(f"Command execution failed: {stderr}", tag="SETUP")
+            await docker_utils.remove_container(container_id)
+            return False
+            
+        # Verify Chrome is installed in the container
+        returncode, stdout, stderr = await docker_utils.exec_in_container(
+            container_id, ["which", "chromium"]
+        )
+        
+        if returncode != 0:
+            logger.error("Chrome not found in container", tag="SETUP")
+            await docker_utils.remove_container(container_id)
+            return False
+            
+        chrome_path = stdout.strip()
+        logger.info(f"Chrome found at: {chrome_path}", tag="SETUP")
+        
+        # Test Chrome version
+        returncode, stdout, stderr = await docker_utils.exec_in_container(
+            container_id, ["chromium", "--version"]
+        )
+        
+        if returncode != 0:
+            logger.error(f"Failed to get Chrome version: {stderr}", tag="SETUP")
+            await docker_utils.remove_container(container_id)
+            return False
+            
+        logger.info(f"Chrome version: {stdout.strip()}", tag="SETUP")
+        
+        # Remove test container
+        removed = await docker_utils.remove_container(container_id)
+        if not removed:
+            logger.error("Failed to remove test container", tag="SETUP")
+            return False
+            
+        logger.success("Test container removed successfully", tag="SETUP")
+        
+        # All components tested successfully
+        logger.success("All Docker components tested successfully", tag="SETUP")
+        return True
+        
+    except Exception as e:
+        logger.error(f"Docker component tests failed: {str(e)}", tag="SETUP")
+        return False
+    finally:
+        # Clean up registry test directory
+        if os.path.exists(registry_dir):
+            shutil.rmtree(registry_dir)
+
+async def test_docker_connect_mode():
+    """Test Docker browser in connect mode.
+    
+    This tests the basic functionality of creating a browser in Docker
+    connect mode and using it for navigation.
+    """
+    logger.info("Testing Docker browser in connect mode", tag="TEST")
+    
+    # Create temp directory for user data
+    temp_dir = os.path.join(os.path.dirname(__file__), "tmp_user_data")
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    try:
+        # Create Docker configuration
+        docker_config = DockerConfig(
+            mode="connect",
+            persistent=False,
+            remove_on_exit=True,
+            user_data_dir=temp_dir
+        )
+        
+        # Create browser configuration
+        browser_config = BrowserConfig(
+            browser_mode="docker",
+            headless=True,
+            docker_config=docker_config
+        )
+        
+        # Create browser manager
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Create crawler config
+        crawler_config = CrawlerRunConfig(url="https://example.com")
+        
+        # Get a page
+        page, context = await manager.get_page(crawler_config)
+        logger.info("Got page successfully", tag="TEST")
+        
+        # Navigate to a website
+        await page.goto("https://example.com")
+        logger.info("Navigated to example.com", tag="TEST")
+        
+        # Get page title
+        title = await page.title()
+        logger.info(f"Page title: {title}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        return True
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+    finally:
+        # Clean up the temp directory
+        if os.path.exists(temp_dir):
+            shutil.rmtree(temp_dir)
+
+async def test_docker_launch_mode():
+    """Test Docker browser in launch mode.
+    
+    This tests launching a Chrome browser within a Docker container
+    on demand with custom settings.
+    """
+    logger.info("Testing Docker browser in launch mode", tag="TEST")
+    
+    # Create temp directory for user data
+    temp_dir = os.path.join(os.path.dirname(__file__), "tmp_user_data_launch")
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    try:
+        # Create Docker configuration
+        docker_config = DockerConfig(
+            mode="launch",
+            persistent=False,
+            remove_on_exit=True,
+            user_data_dir=temp_dir
+        )
+        
+        # Create browser configuration
+        browser_config = BrowserConfig(
+            browser_mode="docker",
+            headless=True,
+            text_mode=True,  # Enable text mode for faster operation
+            docker_config=docker_config
+        )
+        
+        # Create browser manager
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Create crawler config
+        crawler_config = CrawlerRunConfig(url="https://example.com")
+        
+        # Get a page
+        page, context = await manager.get_page(crawler_config)
+        logger.info("Got page successfully", tag="TEST")
+        
+        # Navigate to a website
+        await page.goto("https://example.com")
+        logger.info("Navigated to example.com", tag="TEST")
+        
+        # Get page title
+        title = await page.title()
+        logger.info(f"Page title: {title}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        return True
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+    finally:
+        # Clean up the temp directory
+        if os.path.exists(temp_dir):
+            shutil.rmtree(temp_dir)
+
+async def test_docker_persistent_storage():
+    """Test Docker browser with persistent storage.
+    
+    This tests creating localStorage data in one session and verifying
+    it persists to another session when using persistent storage.
+    """
+    logger.info("Testing Docker browser with persistent storage", tag="TEST")
+    
+    # Create a unique temp directory
+    test_id = uuid.uuid4().hex[:8]
+    temp_dir = os.path.join(os.path.dirname(__file__), f"tmp_user_data_persist_{test_id}")
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    manager1 = None
+    manager2 = None
+    
+    try:
+        # Create Docker configuration with persistence
+        docker_config = DockerConfig(
+            mode="connect",
+            persistent=True,  # Keep container running between sessions
+            user_data_dir=temp_dir,
+            container_user_data_dir="/data"
+        )
+        
+        # Create browser configuration
+        browser_config = BrowserConfig(
+            browser_mode="docker",
+            headless=True,
+            docker_config=docker_config
+        )
+        
+        # Create first browser manager
+        manager1 = BrowserManager(browser_config=browser_config, logger=logger)
+        
+        # Start the browser
+        await manager1.start()
+        logger.info("First browser started successfully", tag="TEST")
+        
+        # Create crawler config
+        crawler_config = CrawlerRunConfig()
+        
+        # Get a page
+        page1, context1 = await manager1.get_page(crawler_config)
+        
+        # Navigate to example.com
+        await page1.goto("https://example.com")
+        
+        # Set localStorage item
+        test_value = f"test_value_{test_id}"
+        await page1.evaluate(f"localStorage.setItem('test_key', '{test_value}')")
+        logger.info(f"Set localStorage test_key = {test_value}", tag="TEST")
+        
+        # Close the first browser manager
+        await manager1.close()
+        logger.info("First browser closed", tag="TEST")
+        
+        # Create second browser manager with same config
+        manager2 = BrowserManager(browser_config=browser_config, logger=logger)
+        
+        # Start the browser
+        await manager2.start()
+        logger.info("Second browser started successfully", tag="TEST")
+        
+        # Get a page
+        page2, context2 = await manager2.get_page(crawler_config)
+        
+        # Navigate to same site
+        await page2.goto("https://example.com")
+        
+        # Get localStorage item
+        value = await page2.evaluate("localStorage.getItem('test_key')")
+        logger.info(f"Retrieved localStorage test_key = {value}", tag="TEST")
+        
+        # Check if persistence worked
+        if value == test_value:
+            logger.success("Storage persistence verified!", tag="TEST")
+        else:
+            logger.error(f"Storage persistence failed! Expected {test_value}, got {value}", tag="TEST")
+        
+        # Clean up
+        await manager2.close()
+        logger.info("Second browser closed successfully", tag="TEST")
+        
+        return value == test_value
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            if manager1:
+                await manager1.close()
+            if manager2:
+                await manager2.close()
+        except:
+            pass
+        return False
+    finally:
+        # Clean up the temp directory
+        if os.path.exists(temp_dir):
+            shutil.rmtree(temp_dir)
+
+async def test_docker_parallel_pages():
+    """Test Docker browser with parallel page creation.
+    
+    This tests the ability to create and use multiple pages in parallel
+    from a single Docker browser instance.
+    """
+    logger.info("Testing Docker browser with parallel pages", tag="TEST")
+    
+    try:
+        # Create Docker configuration
+        docker_config = DockerConfig(
+            mode="connect",
+            persistent=False,
+            remove_on_exit=True
+        )
+        
+        # Create browser configuration
+        browser_config = BrowserConfig(
+            browser_mode="docker",
+            headless=True,
+            docker_config=docker_config
+        )
+        
+        # Create browser manager
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Create crawler config
+        crawler_config = CrawlerRunConfig()
+        
+        # Get multiple pages
+        page_count = 3
+        pages = await manager.get_pages(crawler_config, count=page_count)
+        logger.info(f"Got {len(pages)} pages successfully", tag="TEST")
+        
+        if len(pages) != page_count:
+            logger.error(f"Expected {page_count} pages, got {len(pages)}", tag="TEST")
+            await manager.close()
+            return False
+            
+        # Navigate to different sites with each page
+        tasks = []
+        for i, (page, _) in enumerate(pages):
+            tasks.append(page.goto(f"https://example.com?page={i}"))
+            
+        # Wait for all navigations to complete
+        await asyncio.gather(*tasks)
+        logger.info("All pages navigated successfully", tag="TEST")
+        
+        # Get titles from all pages
+        titles = []
+        for i, (page, _) in enumerate(pages):
+            title = await page.title()
+            titles.append(title)
+            logger.info(f"Page {i+1} title: {title}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        return True
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_docker_registry_reuse():
+    """Test Docker container reuse via registry.
+    
+    This tests that containers with matching configurations
+    are reused rather than creating new ones.
+    """
+    logger.info("Testing Docker container reuse via registry", tag="TEST")
+    
+    # Create registry for this test
+    registry_dir = os.path.join(os.path.dirname(__file__), "registry_reuse_test")
+    registry_file = os.path.join(registry_dir, "registry.json")
+    os.makedirs(registry_dir, exist_ok=True)
+    
+    manager1 = None
+    manager2 = None
+    container_id1 = None
+    
+    try:
+        # Create identical Docker configurations with custom registry
+        docker_config1 = DockerConfig(
+            mode="connect",
+            persistent=True,  # Keep container running after closing
+            registry_file=registry_file
+        )
+        
+        # Create first browser configuration
+        browser_config1 = BrowserConfig(
+            browser_mode="docker",
+            headless=True,
+            docker_config=docker_config1
+        )
+        
+        # Create first browser manager
+        manager1 = BrowserManager(browser_config=browser_config1, logger=logger)
+        
+        # Start the first browser
+        await manager1.start()
+        logger.info("First browser started successfully", tag="TEST")
+        
+        # Get container ID from the strategy
+        docker_strategy1 = manager1.strategy
+        container_id1 = docker_strategy1.container_id
+        logger.info(f"First browser container ID: {container_id1[:12]}", tag="TEST")
+        
+        # Close the first manager but keep container running
+        await manager1.close()
+        logger.info("First browser closed", tag="TEST")
+        
+        # Create second Docker configuration identical to first
+        docker_config2 = DockerConfig(
+            mode="connect",
+            persistent=True,
+            registry_file=registry_file
+        )
+        
+        # Create second browser configuration
+        browser_config2 = BrowserConfig(
+            browser_mode="docker",
+            headless=True,
+            docker_config=docker_config2
+        )
+        
+        # Create second browser manager
+        manager2 = BrowserManager(browser_config=browser_config2, logger=logger)
+        
+        # Start the second browser - should reuse existing container
+        await manager2.start()
+        logger.info("Second browser started successfully", tag="TEST")
+        
+        # Get container ID from the second strategy
+        docker_strategy2 = manager2.strategy
+        container_id2 = docker_strategy2.container_id
+        logger.info(f"Second browser container ID: {container_id2[:12]}", tag="TEST")
+        
+        # Verify container reuse
+        if container_id1 == container_id2:
+            logger.success("Container reuse successful - using same container!", tag="TEST")
+        else:
+            logger.error("Container reuse failed - new container created!", tag="TEST")
+        
+        # Clean up
+        docker_strategy2.docker_config.persistent = False
+        docker_strategy2.docker_config.remove_on_exit = True
+        await manager2.close()
+        logger.info("Second browser closed and container removed", tag="TEST")
+        
+        return container_id1 == container_id2
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            if manager1:
+                await manager1.close()
+            if manager2:
+                await manager2.close()
+            # Make sure container is removed
+            if container_id1:
+                await docker_utils.remove_container(container_id1, force=True)
+        except:
+            pass
+        return False
+    finally:
+        # Clean up registry directory
+        if os.path.exists(registry_dir):
+            shutil.rmtree(registry_dir)
+
+async def run_tests():
+    """Run all tests sequentially."""
+    results = []
+    
+    logger.info("Starting Docker Browser Strategy tests", tag="TEST")
+    
+    # Check if Docker is available
+    if not await check_docker_available():
+        logger.error("Docker is not available - skipping tests", tag="TEST")
+        return
+    
+    # First test Docker components
+    # setup_result = await test_docker_components()
+    # if not setup_result:
+    #     logger.error("Docker component tests failed - skipping browser tests", tag="TEST")
+    #     return
+    
+    # Run browser tests
+    results.append(await test_docker_connect_mode())
+    results.append(await test_docker_launch_mode())
+    results.append(await test_docker_persistent_storage())
+    results.append(await test_docker_parallel_pages())
+    results.append(await test_docker_registry_reuse())
+    
+    # Print summary
+    total = len(results)
+    passed = sum(1 for r in results if r)
+    logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
+    
+    if passed == total:
+        logger.success("All tests passed!", tag="SUMMARY")
+    else:
+        logger.error(f"{total - passed} tests failed", tag="SUMMARY")
+
+async def check_docker_available() -> bool:
+    """Check if Docker is available on the system.
+    
+    Returns:
+        bool: True if Docker is available, False otherwise
+    """
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "docker", "--version",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, _ = await proc.communicate()
+        return proc.returncode == 0 and stdout
+    except:
+        return False
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
\ No newline at end of file
diff --git a/tests/browser/manager/demo_browser_manager.py b/tests/browser/manager/demo_browser_manager.py
new file mode 100644
index 0000000..2fde7e8
--- /dev/null
+++ b/tests/browser/manager/demo_browser_manager.py
@@ -0,0 +1,525 @@
+"""Demo script for testing the enhanced BrowserManager.
+
+This script demonstrates the browser pooling capabilities of the enhanced
+BrowserManager with various configurations and usage patterns.
+"""
+
+import asyncio
+import time
+import random
+
+from crawl4ai.browser.manager import BrowserManager, UnavailableBehavior
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+import playwright
+
+SAFE_URLS = [
+    "https://example.com",
+    "https://example.com/page1",
+    "https://httpbin.org/get",
+    "https://httpbin.org/html",
+    "https://httpbin.org/ip",
+    "https://httpbin.org/user-agent",
+    "https://httpbin.org/headers",
+    "https://httpbin.org/cookies",
+    "https://httpstat.us/200",
+    "https://httpstat.us/301",
+    "https://httpstat.us/404",
+    "https://httpstat.us/500",
+    "https://jsonplaceholder.typicode.com/posts/1",
+    "https://jsonplaceholder.typicode.com/posts/2",
+    "https://jsonplaceholder.typicode.com/posts/3",
+    "https://jsonplaceholder.typicode.com/posts/4",
+    "https://jsonplaceholder.typicode.com/posts/5",
+    "https://jsonplaceholder.typicode.com/comments/1",
+    "https://jsonplaceholder.typicode.com/comments/2",
+    "https://jsonplaceholder.typicode.com/users/1",
+    "https://jsonplaceholder.typicode.com/users/2",
+    "https://jsonplaceholder.typicode.com/albums/1",
+    "https://jsonplaceholder.typicode.com/albums/2",
+    "https://jsonplaceholder.typicode.com/photos/1",
+    "https://jsonplaceholder.typicode.com/photos/2",
+    "https://jsonplaceholder.typicode.com/todos/1",
+    "https://jsonplaceholder.typicode.com/todos/2",
+    "https://www.iana.org",
+    "https://www.iana.org/domains",
+    "https://www.iana.org/numbers",
+    "https://www.iana.org/protocols",
+    "https://www.iana.org/about",
+    "https://www.iana.org/time-zones",
+    "https://www.data.gov",
+    "https://catalog.data.gov/dataset",
+    "https://www.archives.gov",
+    "https://www.usa.gov",
+    "https://www.loc.gov",
+    "https://www.irs.gov",
+    "https://www.census.gov",
+    "https://www.bls.gov",
+    "https://www.gpo.gov",
+    "https://www.w3.org",
+    "https://www.w3.org/standards",
+    "https://www.w3.org/WAI",
+    "https://www.rfc-editor.org",
+    "https://www.ietf.org",
+    "https://www.icann.org",
+    "https://www.internetsociety.org",
+    "https://www.python.org"
+]
+
+async def basic_pooling_demo():
+    """Demonstrate basic browser pooling functionality."""
+    print("\n=== Basic Browser Pooling Demo ===")
+    
+    # Create logger
+    logger = AsyncLogger(verbose=True)
+    
+    # Create browser configurations
+    config1 = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        browser_mode="playwright"
+    )
+    
+    config2 = BrowserConfig(
+        browser_type="chromium", 
+        headless=True,
+        browser_mode="cdp"
+    )
+    
+    # Create browser manager with on-demand behavior
+    manager = BrowserManager(
+        browser_config=config1,
+        logger=logger,
+        unavailable_behavior=UnavailableBehavior.ON_DEMAND,
+        max_browsers_per_config=3
+    )
+    
+    try:
+        # Initialize pool with both configurations
+        print("Initializing browser pool...")
+        await manager.initialize_pool(
+            browser_configs=[config1, config2],
+            browsers_per_config=2
+        )
+        
+        # Display initial pool status
+        status = await manager.get_pool_status()
+        print(f"Initial pool status: {status}")
+        
+        # Create crawler run configurations
+        run_config1 = CrawlerRunConfig()
+        run_config2 = CrawlerRunConfig()
+        
+        # Simulate concurrent page requests
+        print("\nGetting pages for parallel crawling...")
+        
+        # Function to simulate crawling
+        async def simulate_crawl(index: int, config: BrowserConfig, run_config: CrawlerRunConfig):
+            print(f"Crawler {index}: Requesting page...")
+            page, context, strategy = await manager.get_page(run_config, config)
+            print(f"Crawler {index}: Got page, navigating to example.com...")
+            
+            try:
+                await page.goto("https://example.com")
+                title = await page.title()
+                print(f"Crawler {index}: Page title: {title}")
+                
+                # Simulate work
+                await asyncio.sleep(random.uniform(1, 3))
+                print(f"Crawler {index}: Work completed, releasing page...")
+                
+                # Check dynamic page content
+                content = await page.content()
+                content_length = len(content)
+                print(f"Crawler {index}: Page content length: {content_length}")
+                
+            except Exception as e:
+                print(f"Crawler {index}: Error: {str(e)}")
+            finally:
+                # Release the page
+                await manager.release_page(page, strategy, config)
+                print(f"Crawler {index}: Page released")
+        
+        # Create 5 parallel crawls
+        crawl_tasks = []
+        for i in range(5):
+            # Alternate between configurations
+            config = config1 if i % 2 == 0 else config2
+            run_config = run_config1 if i % 2 == 0 else run_config2
+            
+            task = asyncio.create_task(simulate_crawl(i+1, config, run_config))
+            crawl_tasks.append(task)
+        
+        # Wait for all crawls to complete
+        await asyncio.gather(*crawl_tasks)
+        
+        # Display final pool status
+        status = await manager.get_pool_status()
+        print(f"\nFinal pool status: {status}")
+        
+    finally:
+        # Clean up
+        print("\nClosing browser manager...")
+        await manager.close()
+        print("Browser manager closed")
+
+
+async def prewarm_pages_demo():
+    """Demonstrate page pre-warming functionality."""
+    print("\n=== Page Pre-warming Demo ===")
+    
+    # Create logger
+    logger = AsyncLogger(verbose=True)
+    
+    # Create browser configuration
+    config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        browser_mode="playwright"
+    )
+    
+    # Create crawler run configurations for pre-warming
+    run_config1 = CrawlerRunConfig(
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    )
+    
+    run_config2 = CrawlerRunConfig(
+        user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15"
+    )
+    
+    # Create page pre-warm configurations
+    page_configs = [
+        (config, run_config1, 2),  # 2 pages with run_config1
+        (config, run_config2, 3)   # 3 pages with run_config2
+    ]
+    
+    # Create browser manager
+    manager = BrowserManager(
+        browser_config=config,
+        logger=logger,
+        unavailable_behavior=UnavailableBehavior.EXCEPTION
+    )
+    
+    try:
+        # Initialize pool with pre-warmed pages
+        print("Initializing browser pool with pre-warmed pages...")
+        await manager.initialize_pool(
+            browser_configs=[config],
+            browsers_per_config=2,
+            page_configs=page_configs
+        )
+        
+        # Display pool status
+        status = await manager.get_pool_status()
+        print(f"Pool status after pre-warming: {status}")
+        
+        # Simulate using pre-warmed pages
+        print("\nUsing pre-warmed pages...")
+        
+        async def use_prewarm_page(index: int, run_config: CrawlerRunConfig):
+            print(f"Task {index}: Requesting pre-warmed page...")
+            page, context, strategy = await manager.get_page(run_config, config)
+            
+            try:
+                print(f"Task {index}: Got page, navigating to example.com...")
+                await page.goto("https://example.com")
+                
+                # Verify user agent was applied correctly
+                user_agent = await page.evaluate("() => navigator.userAgent")
+                print(f"Task {index}: User agent: {user_agent}")
+                
+                # Get page title
+                title = await page.title()
+                print(f"Task {index}: Page title: {title}")
+                
+                # Simulate work
+                await asyncio.sleep(1)
+            finally:
+                # Release the page
+                print(f"Task {index}: Releasing page...")
+                await manager.release_page(page, strategy, config)
+        
+        # Create tasks to use pre-warmed pages
+        tasks = []
+        # Use run_config1 pages
+        for i in range(2):
+            tasks.append(asyncio.create_task(use_prewarm_page(i+1, run_config1)))
+        
+        # Use run_config2 pages
+        for i in range(3):
+            tasks.append(asyncio.create_task(use_prewarm_page(i+3, run_config2)))
+        
+        # Wait for all tasks to complete
+        await asyncio.gather(*tasks)
+        
+        # Try to use more pages than we pre-warmed (should raise exception)
+        print("\nTrying to use more pages than pre-warmed...")
+        try:
+            page, context, strategy = await manager.get_page(run_config1, config)
+            try:
+                print("Got extra page (unexpected)")
+                await page.goto("https://example.com")
+            finally:
+                await manager.release_page(page, strategy, config)
+        except Exception as e:
+            print(f"Expected exception when requesting more pages: {str(e)}")
+        
+    finally:
+        # Clean up
+        print("\nClosing browser manager...")
+        await manager.close()
+        print("Browser manager closed")
+
+
+async def prewarm_on_demand_demo():
+    """Demonstrate pre-warming with on-demand browser creation."""
+    print("\n=== Pre-warming with On-Demand Browser Creation Demo ===")
+    
+    # Create logger
+    logger = AsyncLogger(verbose=True)
+    
+    # Create browser configuration
+    config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        browser_mode="playwright"
+    )
+    
+    # Create crawler run configurations
+    run_config = CrawlerRunConfig(
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    )
+    
+    # Create page pre-warm configurations - just pre-warm 2 pages
+    page_configs = [
+        (config, run_config, 2)
+    ]
+    
+    # Create browser manager with ON_DEMAND behavior
+    manager = BrowserManager(
+        browser_config=config,
+        logger=logger,
+        unavailable_behavior=UnavailableBehavior.ON_DEMAND,
+        max_browsers_per_config=5  # Allow up to 5 browsers
+    )
+    
+    try:
+        # Initialize pool with pre-warmed pages
+        print("Initializing browser pool with pre-warmed pages...")
+        await manager.initialize_pool(
+            browser_configs=[config],
+            browsers_per_config=1,  # Start with just 1 browser
+            page_configs=page_configs
+        )
+        
+        # Display initial pool status
+        status = await manager.get_pool_status()
+        print(f"Initial pool status: {status}")
+        
+        # Simulate using more pages than pre-warmed - should create browsers on demand
+        print("\nUsing more pages than pre-warmed (should create on demand)...")
+        
+        async def use_page(index: int):
+            print(f"Task {index}: Requesting page...")
+            page, context, strategy = await manager.get_page(run_config, config)
+            
+            try:
+                print(f"Task {index}: Got page, navigating to example.com...")
+                await page.goto("https://example.com")
+                
+                # Get page title
+                title = await page.title()
+                print(f"Task {index}: Page title: {title}")
+                
+                # Simulate work for a varying amount of time
+                work_time = 1 + (index * 0.5)  # Stagger completion times
+                print(f"Task {index}: Working for {work_time} seconds...")
+                await asyncio.sleep(work_time)
+                print(f"Task {index}: Work completed")
+            finally:
+                # Release the page
+                print(f"Task {index}: Releasing page...")
+                await manager.release_page(page, strategy, config)
+        
+        # Create more tasks than pre-warmed pages
+        tasks = []
+        for i in range(5):  # Try to use 5 pages when only 2 are pre-warmed
+            tasks.append(asyncio.create_task(use_page(i+1)))
+        
+        # Wait for all tasks to complete
+        await asyncio.gather(*tasks)
+        
+        # Display final pool status - should show on-demand created browsers
+        status = await manager.get_pool_status()
+        print(f"\nFinal pool status: {status}")
+        
+    finally:
+        # Clean up
+        print("\nClosing browser manager...")
+        await manager.close()
+        print("Browser manager closed")
+
+
+async def high_volume_demo():
+    """Demonstrate high-volume access to pre-warmed pages."""
+    print("\n=== High Volume Pre-warmed Pages Demo ===")
+    
+    # Create logger
+    logger = AsyncLogger(verbose=True)
+    
+    # Create browser configuration
+    config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        browser_mode="playwright"
+    )
+    
+    # Create crawler run configuration
+    run_config = CrawlerRunConfig(
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    )
+    
+    # Set up dimensions
+    browser_count = 10
+    pages_per_browser = 5
+    total_pages = browser_count * pages_per_browser
+    
+    # Create page pre-warm configuration
+    page_configs = [
+        (config, run_config, total_pages)
+    ]
+    
+    print(f"Preparing {browser_count} browsers with {pages_per_browser} pages each ({total_pages} total pages)")
+    
+    # Create browser manager with ON_DEMAND behavior as fallback
+    # No need to specify max_browsers_per_config as it will be calculated automatically
+    manager = BrowserManager(
+        browser_config=config,
+        logger=logger,
+        unavailable_behavior=UnavailableBehavior.ON_DEMAND
+    )
+    
+    try:
+        # Initialize pool with browsers and pre-warmed pages
+        print(f"Pre-warming {total_pages} pages...")
+        start_time = time.time()
+        await manager.initialize_pool(
+            browser_configs=[config],
+            browsers_per_config=browser_count,
+            page_configs=page_configs
+        )
+        warmup_time = time.time() - start_time
+        print(f"Pre-warming completed in {warmup_time:.2f} seconds")
+        
+        # Display pool status
+        status = await manager.get_pool_status()
+        print(f"Pool status after pre-warming: {status}")
+        
+        # Simulate using all pre-warmed pages simultaneously
+        print(f"\nSending {total_pages} crawl requests simultaneously...")
+        
+        async def crawl_page(index: int):
+            # url = f"https://example.com/page{index}"
+            url = SAFE_URLS[index % len(SAFE_URLS)]
+            print(f"Page {index}: Requesting page...")            
+            # Measure time to acquire page
+            page_start = time.time()
+            page, context, strategy = await manager.get_page(run_config, config)
+            page_acquisition_time = time.time() - page_start
+            
+            try:
+                # Navigate to the URL
+                nav_start = time.time()
+                await page.goto(url, timeout=5000)
+                navigation_time = time.time() - nav_start
+                
+                # Get the page title
+                title = await page.title()
+                
+                return {
+                    "index": index,
+                    "url": url,
+                    "title": title,
+                    "page_acquisition_time": page_acquisition_time,
+                    "navigation_time": navigation_time
+                }
+            except playwright._impl._errors.TimeoutError as e:
+                # print(f"Page {index}: Navigation timed out - {e}")
+                return {
+                    "index": index,
+                    "url": url,
+                    "title": "Navigation timed out",
+                    "page_acquisition_time": page_acquisition_time,
+                    "navigation_time": 0
+                }
+            finally:
+                # Release the page
+                await manager.release_page(page, strategy, config)
+        
+        # Create and execute all tasks simultaneously
+        start_time = time.time()
+
+        # Non-parallel way
+        # for i in range(total_pages):
+        #     await crawl_page(i+1)
+
+        tasks = [crawl_page(i+1) for i in range(total_pages)]
+        results = await asyncio.gather(*tasks)
+        total_time = time.time() - start_time
+        
+        # # Print all titles
+        # for result in results:
+        #     print(f"Page {result['index']} ({result['url']}): Title: {result['title']}")
+        #     print(f"  Page acquisition time: {result['page_acquisition_time']:.4f}s")
+        #     print(f"  Navigation time: {result['navigation_time']:.4f}s")
+        #     print(f"  Total time: {result['page_acquisition_time'] + result['navigation_time']:.4f}s")
+        #     print("-" * 40)
+        
+        # Report results
+        print(f"\nAll {total_pages} crawls completed in {total_time:.2f} seconds")
+        
+        # Calculate statistics
+        acquisition_times = [r["page_acquisition_time"] for r in results]
+        navigation_times = [r["navigation_time"] for r in results]
+        
+        avg_acquisition = sum(acquisition_times) / len(acquisition_times)
+        max_acquisition = max(acquisition_times)
+        min_acquisition = min(acquisition_times)
+        
+        avg_navigation = sum(navigation_times) / len(navigation_times)
+        max_navigation = max(navigation_times)
+        min_navigation = min(navigation_times)
+        
+        print("\nPage acquisition times:")
+        print(f"  Average: {avg_acquisition:.4f}s")
+        print(f"  Min: {min_acquisition:.4f}s")
+        print(f"  Max: {max_acquisition:.4f}s")
+        
+        print("\nPage navigation times:")
+        print(f"  Average: {avg_navigation:.4f}s")
+        print(f"  Min: {min_navigation:.4f}s")
+        print(f"  Max: {max_navigation:.4f}s")
+        
+        # Display final pool status
+        status = await manager.get_pool_status()
+        print(f"\nFinal pool status: {status}")
+        
+    finally:
+        # Clean up
+        print("\nClosing browser manager...")
+        await manager.close()
+        print("Browser manager closed")
+
+
+async def main():
+    """Run all demos."""
+    # await basic_pooling_demo()
+    # await prewarm_pages_demo()
+    # await prewarm_on_demand_demo()
+    await high_volume_demo()
+    # Additional demo functions can be added here
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/browser/test_browser_context_id.py b/tests/browser/test_browser_context_id.py
new file mode 100644
index 0000000..0ee0b1b
--- /dev/null
+++ b/tests/browser/test_browser_context_id.py
@@ -0,0 +1,489 @@
+"""Test for browser_context_id and target_id parameters.
+
+These tests verify that Crawl4AI can connect to and use pre-created
+browser contexts, which is essential for cloud browser services that
+pre-create isolated contexts for each user.
+
+The flow being tested:
+1. Start a browser with CDP
+2. Create a context via raw CDP commands (simulating cloud service)
+3. Create a page/target in that context
+4. Have Crawl4AI connect using browser_context_id and target_id
+5. Verify Crawl4AI uses the existing context/page instead of creating new ones
+"""
+
+import asyncio
+import json
+import os
+import sys
+import websockets
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai.browser_manager import BrowserManager, ManagedBrowser
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+
+class CDPContextCreator:
+    """
+    Helper class to create browser contexts via raw CDP commands.
+    This simulates what a cloud browser service would do.
+    """
+
+    def __init__(self, cdp_url: str):
+        self.cdp_url = cdp_url
+        self._message_id = 0
+        self._ws = None
+        self._pending_responses = {}
+        self._receiver_task = None
+
+    async def connect(self):
+        """Establish WebSocket connection to browser."""
+        # Convert HTTP URL to WebSocket URL if needed
+        ws_url = self.cdp_url.replace("http://", "ws://").replace("https://", "wss://")
+        if not ws_url.endswith("/devtools/browser"):
+            # Get the browser websocket URL from /json/version
+            import aiohttp
+            async with aiohttp.ClientSession() as session:
+                async with session.get(f"{self.cdp_url}/json/version") as response:
+                    data = await response.json()
+                    ws_url = data.get("webSocketDebuggerUrl", ws_url)
+
+        self._ws = await websockets.connect(ws_url, max_size=None, ping_interval=None)
+        self._receiver_task = asyncio.create_task(self._receive_messages())
+        logger.info(f"Connected to CDP at {ws_url}", tag="CDP")
+
+    async def disconnect(self):
+        """Close WebSocket connection."""
+        if self._receiver_task:
+            self._receiver_task.cancel()
+            try:
+                await self._receiver_task
+            except asyncio.CancelledError:
+                pass
+        if self._ws:
+            await self._ws.close()
+            self._ws = None
+
+    async def _receive_messages(self):
+        """Background task to receive CDP messages."""
+        try:
+            async for message in self._ws:
+                data = json.loads(message)
+                msg_id = data.get('id')
+                if msg_id is not None and msg_id in self._pending_responses:
+                    self._pending_responses[msg_id].set_result(data)
+        except asyncio.CancelledError:
+            pass
+        except Exception as e:
+            logger.error(f"CDP receiver error: {e}", tag="CDP")
+
+    async def _send_command(self, method: str, params: dict = None) -> dict:
+        """Send CDP command and wait for response."""
+        self._message_id += 1
+        msg_id = self._message_id
+
+        message = {
+            "id": msg_id,
+            "method": method,
+            "params": params or {}
+        }
+
+        future = asyncio.get_event_loop().create_future()
+        self._pending_responses[msg_id] = future
+
+        try:
+            await self._ws.send(json.dumps(message))
+            response = await asyncio.wait_for(future, timeout=30.0)
+
+            if 'error' in response:
+                raise Exception(f"CDP error: {response['error']}")
+
+            return response.get('result', {})
+        finally:
+            self._pending_responses.pop(msg_id, None)
+
+    async def create_context(self) -> dict:
+        """
+        Create an isolated browser context with a blank page.
+
+        Returns:
+            dict with browser_context_id, target_id, and cdp_session_id
+        """
+        await self.connect()
+
+        # 1. Create isolated browser context
+        result = await self._send_command("Target.createBrowserContext", {
+            "disposeOnDetach": False  # Keep context alive
+        })
+        browser_context_id = result["browserContextId"]
+        logger.info(f"Created browser context: {browser_context_id}", tag="CDP")
+
+        # 2. Create a new page (target) in the context
+        result = await self._send_command("Target.createTarget", {
+            "url": "about:blank",
+            "browserContextId": browser_context_id
+        })
+        target_id = result["targetId"]
+        logger.info(f"Created target: {target_id}", tag="CDP")
+
+        # 3. Attach to the target to get a session ID
+        result = await self._send_command("Target.attachToTarget", {
+            "targetId": target_id,
+            "flatten": True
+        })
+        cdp_session_id = result["sessionId"]
+        logger.info(f"Attached to target, sessionId: {cdp_session_id}", tag="CDP")
+
+        return {
+            "browser_context_id": browser_context_id,
+            "target_id": target_id,
+            "cdp_session_id": cdp_session_id
+        }
+
+    async def get_targets(self) -> list:
+        """Get list of all targets in the browser."""
+        result = await self._send_command("Target.getTargets")
+        return result.get("targetInfos", [])
+
+    async def dispose_context(self, browser_context_id: str):
+        """Dispose of a browser context."""
+        try:
+            await self._send_command("Target.disposeBrowserContext", {
+                "browserContextId": browser_context_id
+            })
+            logger.info(f"Disposed browser context: {browser_context_id}", tag="CDP")
+        except Exception as e:
+            logger.warning(f"Error disposing context: {e}", tag="CDP")
+
+
+async def test_browser_context_id_basic():
+    """
+    Test that BrowserConfig accepts browser_context_id and target_id parameters.
+    """
+    logger.info("Testing BrowserConfig browser_context_id parameter", tag="TEST")
+
+    try:
+        # Test that BrowserConfig accepts the new parameters
+        config = BrowserConfig(
+            cdp_url="http://localhost:9222",
+            browser_context_id="test-context-id",
+            target_id="test-target-id",
+            headless=True
+        )
+
+        # Verify parameters are set correctly
+        assert config.browser_context_id == "test-context-id", "browser_context_id not set"
+        assert config.target_id == "test-target-id", "target_id not set"
+
+        # Test from_kwargs
+        config2 = BrowserConfig.from_kwargs({
+            "cdp_url": "http://localhost:9222",
+            "browser_context_id": "test-context-id-2",
+            "target_id": "test-target-id-2"
+        })
+
+        assert config2.browser_context_id == "test-context-id-2", "browser_context_id not set via from_kwargs"
+        assert config2.target_id == "test-target-id-2", "target_id not set via from_kwargs"
+
+        # Test to_dict
+        config_dict = config.to_dict()
+        assert config_dict.get("browser_context_id") == "test-context-id", "browser_context_id not in to_dict"
+        assert config_dict.get("target_id") == "test-target-id", "target_id not in to_dict"
+
+        logger.success("BrowserConfig browser_context_id test passed", tag="TEST")
+        return True
+
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        return False
+
+
+async def test_pre_created_context_usage():
+    """
+    Test that Crawl4AI uses a pre-created browser context instead of creating a new one.
+
+    This simulates the cloud browser service flow:
+    1. Start browser with CDP
+    2. Create context via raw CDP (simulating cloud service)
+    3. Have Crawl4AI connect with browser_context_id
+    4. Verify it uses existing context
+    """
+    logger.info("Testing pre-created context usage", tag="TEST")
+
+    # Start a managed browser first
+    browser_config_initial = BrowserConfig(
+        use_managed_browser=True,
+        headless=True,
+        debugging_port=9226,  # Use unique port
+        verbose=True
+    )
+
+    managed_browser = ManagedBrowser(browser_config=browser_config_initial, logger=logger)
+    cdp_creator = None
+    manager = None
+    context_info = None
+
+    try:
+        # Start the browser
+        cdp_url = await managed_browser.start()
+        logger.info(f"Browser started at {cdp_url}", tag="TEST")
+
+        # Create a context via raw CDP (simulating cloud service)
+        cdp_creator = CDPContextCreator(cdp_url)
+        context_info = await cdp_creator.create_context()
+
+        logger.info(f"Pre-created context: {context_info['browser_context_id']}", tag="TEST")
+        logger.info(f"Pre-created target: {context_info['target_id']}", tag="TEST")
+
+        # Get initial target count
+        targets_before = await cdp_creator.get_targets()
+        initial_target_count = len(targets_before)
+        logger.info(f"Initial target count: {initial_target_count}", tag="TEST")
+
+        # Now create BrowserManager with browser_context_id and target_id
+        browser_config = BrowserConfig(
+            cdp_url=cdp_url,
+            browser_context_id=context_info['browser_context_id'],
+            target_id=context_info['target_id'],
+            headless=True,
+            verbose=True
+        )
+
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        await manager.start()
+
+        logger.info("BrowserManager started with pre-created context", tag="TEST")
+
+        # Get a page
+        crawler_config = CrawlerRunConfig()
+        page, context = await manager.get_page(crawler_config)
+
+        # Navigate to a test page
+        await page.goto("https://example.com", wait_until="domcontentloaded")
+        title = await page.title()
+
+        logger.info(f"Page title: {title}", tag="TEST")
+
+        # Get target count after
+        targets_after = await cdp_creator.get_targets()
+        final_target_count = len(targets_after)
+        logger.info(f"Final target count: {final_target_count}", tag="TEST")
+
+        # Verify: target count should not have increased significantly
+        # (allow for 1 extra target for internal use, but not many more)
+        target_diff = final_target_count - initial_target_count
+        logger.info(f"Target count difference: {target_diff}", tag="TEST")
+
+        # Success criteria:
+        # 1. Page navigation worked
+        # 2. Target count didn't explode (reused existing context)
+        success = title == "Example Domain" and target_diff <= 1
+
+        if success:
+            logger.success("Pre-created context usage test passed", tag="TEST")
+        else:
+            logger.error(f"Test failed - Title: {title}, Target diff: {target_diff}", tag="TEST")
+
+        return success
+
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        import traceback
+        traceback.print_exc()
+        return False
+
+    finally:
+        # Cleanup
+        if manager:
+            try:
+                await manager.close()
+            except:
+                pass
+
+        if cdp_creator and context_info:
+            try:
+                await cdp_creator.dispose_context(context_info['browser_context_id'])
+                await cdp_creator.disconnect()
+            except:
+                pass
+
+        if managed_browser:
+            try:
+                await managed_browser.cleanup()
+            except:
+                pass
+
+
+async def test_context_isolation():
+    """
+    Test that using browser_context_id actually provides isolation.
+    Create two contexts and verify they don't share state.
+    """
+    logger.info("Testing context isolation with browser_context_id", tag="TEST")
+
+    browser_config_initial = BrowserConfig(
+        use_managed_browser=True,
+        headless=True,
+        debugging_port=9227,
+        verbose=True
+    )
+
+    managed_browser = ManagedBrowser(browser_config=browser_config_initial, logger=logger)
+    cdp_creator = None
+    manager1 = None
+    manager2 = None
+    context_info_1 = None
+    context_info_2 = None
+
+    try:
+        # Start the browser
+        cdp_url = await managed_browser.start()
+        logger.info(f"Browser started at {cdp_url}", tag="TEST")
+
+        # Create two separate contexts
+        cdp_creator = CDPContextCreator(cdp_url)
+        context_info_1 = await cdp_creator.create_context()
+        logger.info(f"Context 1: {context_info_1['browser_context_id']}", tag="TEST")
+
+        # Need to reconnect for second context (or use same connection)
+        await cdp_creator.disconnect()
+        cdp_creator2 = CDPContextCreator(cdp_url)
+        context_info_2 = await cdp_creator2.create_context()
+        logger.info(f"Context 2: {context_info_2['browser_context_id']}", tag="TEST")
+
+        # Verify contexts are different
+        assert context_info_1['browser_context_id'] != context_info_2['browser_context_id'], \
+            "Contexts should have different IDs"
+
+        # Connect with first context
+        browser_config_1 = BrowserConfig(
+            cdp_url=cdp_url,
+            browser_context_id=context_info_1['browser_context_id'],
+            target_id=context_info_1['target_id'],
+            headless=True
+        )
+
+        manager1 = BrowserManager(browser_config=browser_config_1, logger=logger)
+        await manager1.start()
+
+        # Set a cookie in context 1
+        page1, ctx1 = await manager1.get_page(CrawlerRunConfig())
+        await page1.goto("https://example.com", wait_until="domcontentloaded")
+        await ctx1.add_cookies([{
+            "name": "test_isolation",
+            "value": "context_1_value",
+            "domain": "example.com",
+            "path": "/"
+        }])
+
+        cookies1 = await ctx1.cookies(["https://example.com"])
+        cookie1_value = next((c["value"] for c in cookies1 if c["name"] == "test_isolation"), None)
+        logger.info(f"Cookie in context 1: {cookie1_value}", tag="TEST")
+
+        # Connect with second context
+        browser_config_2 = BrowserConfig(
+            cdp_url=cdp_url,
+            browser_context_id=context_info_2['browser_context_id'],
+            target_id=context_info_2['target_id'],
+            headless=True
+        )
+
+        manager2 = BrowserManager(browser_config=browser_config_2, logger=logger)
+        await manager2.start()
+
+        # Check cookies in context 2 - should not have the cookie from context 1
+        page2, ctx2 = await manager2.get_page(CrawlerRunConfig())
+        await page2.goto("https://example.com", wait_until="domcontentloaded")
+
+        cookies2 = await ctx2.cookies(["https://example.com"])
+        cookie2_value = next((c["value"] for c in cookies2 if c["name"] == "test_isolation"), None)
+        logger.info(f"Cookie in context 2: {cookie2_value}", tag="TEST")
+
+        # Verify isolation
+        isolation_works = cookie1_value == "context_1_value" and cookie2_value is None
+
+        if isolation_works:
+            logger.success("Context isolation test passed", tag="TEST")
+        else:
+            logger.error(f"Isolation failed - Cookie1: {cookie1_value}, Cookie2: {cookie2_value}", tag="TEST")
+
+        return isolation_works
+
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        import traceback
+        traceback.print_exc()
+        return False
+
+    finally:
+        # Cleanup
+        for mgr in [manager1, manager2]:
+            if mgr:
+                try:
+                    await mgr.close()
+                except:
+                    pass
+
+        for ctx_info, creator in [(context_info_1, cdp_creator), (context_info_2, cdp_creator2 if 'cdp_creator2' in dir() else None)]:
+            if ctx_info and creator:
+                try:
+                    await creator.dispose_context(ctx_info['browser_context_id'])
+                    await creator.disconnect()
+                except:
+                    pass
+
+        if managed_browser:
+            try:
+                await managed_browser.cleanup()
+            except:
+                pass
+
+
+async def run_tests():
+    """Run all browser_context_id tests."""
+    results = []
+
+    logger.info("Running browser_context_id tests", tag="SUITE")
+
+    # Basic parameter test
+    results.append(("browser_context_id_basic", await test_browser_context_id_basic()))
+
+    # Pre-created context usage test
+    results.append(("pre_created_context_usage", await test_pre_created_context_usage()))
+
+    # Note: Context isolation test is commented out because isolation is enforced
+    # at the CDP level by the cloud browser service, not at the Playwright level.
+    # When multiple BrowserManagers connect to the same browser, Playwright sees
+    # all contexts. In production, each worker gets exactly one pre-created context.
+    # results.append(("context_isolation", await test_context_isolation()))
+
+    # Print summary
+    total = len(results)
+    passed = sum(1 for _, r in results if r)
+
+    logger.info("=" * 50, tag="SUMMARY")
+    logger.info(f"Test Results: {passed}/{total} passed", tag="SUMMARY")
+    logger.info("=" * 50, tag="SUMMARY")
+
+    for name, result in results:
+        status = "PASSED" if result else "FAILED"
+        logger.info(f"  {name}: {status}", tag="SUMMARY")
+
+    if passed == total:
+        logger.success("All tests passed!", tag="SUMMARY")
+        return True
+    else:
+        logger.error(f"{total - passed} tests failed", tag="SUMMARY")
+        return False
+
+
+if __name__ == "__main__":
+    success = asyncio.run(run_tests())
+    sys.exit(0 if success else 1)
diff --git a/tests/browser/test_browser_manager.py b/tests/browser/test_browser_manager.py
new file mode 100644
index 0000000..d8f9376
--- /dev/null
+++ b/tests/browser/test_browser_manager.py
@@ -0,0 +1,190 @@
+"""Test examples for BrowserManager.
+
+These examples demonstrate the functionality of BrowserManager
+and serve as functional tests.
+"""
+
+import asyncio
+import os
+import sys
+from typing import List
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai.browser import BrowserManager
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+async def test_basic_browser_manager():
+    """Test basic BrowserManager functionality with default configuration."""
+    logger.info("Starting test_basic_browser_manager", tag="TEST")
+    
+    try:
+        # Create a browser manager with default config
+        manager = BrowserManager(logger=logger)
+        
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Get a page
+        crawler_config = CrawlerRunConfig(url="https://example.com")
+        page, context = await manager.get_page(crawler_config)
+        logger.info("Page created successfully", tag="TEST")
+        
+        # Navigate to a website
+        await page.goto("https://example.com")
+        title = await page.title()
+        logger.info(f"Page title: {title}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.success("test_basic_browser_manager completed successfully", tag="TEST")
+        return True
+    except Exception as e:
+        logger.error(f"test_basic_browser_manager failed: {str(e)}", tag="TEST")
+        return False
+
+async def test_custom_browser_config():
+    """Test BrowserManager with custom browser configuration."""
+    logger.info("Starting test_custom_browser_config", tag="TEST")
+    
+    try:
+        # Create a custom browser config
+        browser_config = BrowserConfig(
+            browser_type="chromium",
+            headless=True,
+            viewport_width=1280,
+            viewport_height=800,
+            light_mode=True
+        )
+        
+        # Create browser manager with the config
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully with custom config", tag="TEST")
+        
+        # Get a page
+        crawler_config = CrawlerRunConfig(url="https://example.com")
+        page, context = await manager.get_page(crawler_config)
+        
+        # Navigate to a website
+        await page.goto("https://example.com")
+        title = await page.title()
+        logger.info(f"Page title: {title}", tag="TEST")
+        
+        # Verify viewport size
+        viewport_size = await page.evaluate("() => ({ width: window.innerWidth, height: window.innerHeight })")
+        logger.info(f"Viewport size: {viewport_size}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.success("test_custom_browser_config completed successfully", tag="TEST")
+        return True
+    except Exception as e:
+        logger.error(f"test_custom_browser_config failed: {str(e)}", tag="TEST")
+        return False
+
+async def test_multiple_pages():
+    """Test BrowserManager with multiple pages."""
+    logger.info("Starting test_multiple_pages", tag="TEST")
+    
+    try:
+        # Create browser manager
+        manager = BrowserManager(logger=logger)
+        
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Create multiple pages
+        pages = []
+        urls = ["https://example.com", "https://example.org", "https://mozilla.org"]
+        
+        for i, url in enumerate(urls):
+            crawler_config = CrawlerRunConfig(url=url)
+            page, context = await manager.get_page(crawler_config)
+            await page.goto(url)
+            pages.append((page, url))
+            logger.info(f"Created page {i+1} for {url}", tag="TEST")
+        
+        # Verify all pages are loaded correctly
+        for i, (page, url) in enumerate(pages):
+            title = await page.title()
+            logger.info(f"Page {i+1} title: {title}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.success("test_multiple_pages completed successfully", tag="TEST")
+        return True
+    except Exception as e:
+        logger.error(f"test_multiple_pages failed: {str(e)}", tag="TEST")
+        return False
+
+async def test_session_management():
+    """Test session management in BrowserManager."""
+    logger.info("Starting test_session_management", tag="TEST")
+    
+    try:
+        # Create browser manager
+        manager = BrowserManager(logger=logger)
+        
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Create a session
+        session_id = "test_session_1"
+        crawler_config = CrawlerRunConfig(url="https://example.com", session_id=session_id)
+        page1, context1 = await manager.get_page(crawler_config)
+        await page1.goto("https://example.com")
+        logger.info(f"Created session with ID: {session_id}", tag="TEST")
+        
+        # Get the same session again
+        page2, context2 = await manager.get_page(crawler_config)
+        
+        # Verify it's the same page/context
+        is_same_page = page1 == page2
+        is_same_context = context1 == context2
+        logger.info(f"Same page: {is_same_page}, Same context: {is_same_context}", tag="TEST")
+        
+        # Kill the session
+        await manager.kill_session(session_id)
+        logger.info(f"Killed session with ID: {session_id}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.success("test_session_management completed successfully", tag="TEST")
+        return True
+    except Exception as e:
+        logger.error(f"test_session_management failed: {str(e)}", tag="TEST")
+        return False
+
+async def run_tests():
+    """Run all tests sequentially."""
+    results = []
+    
+    results.append(await test_basic_browser_manager())
+    results.append(await test_custom_browser_config())
+    results.append(await test_multiple_pages())
+    results.append(await test_session_management())
+    
+    # Print summary
+    total = len(results)
+    passed = sum(results)
+    logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
+    
+    if passed == total:
+        logger.success("All tests passed!", tag="SUMMARY")
+    else:
+        logger.error(f"{total - passed} tests failed", tag="SUMMARY")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
diff --git a/tests/browser/test_browser_manager_close.py b/tests/browser/test_browser_manager_close.py
new file mode 100644
index 0000000..c451995
--- /dev/null
+++ b/tests/browser/test_browser_manager_close.py
@@ -0,0 +1,35 @@
+from types import SimpleNamespace
+
+import pytest
+
+from crawl4ai.browser_manager import BrowserManager
+
+
+class _Context:
+    def __init__(self, manager=None):
+        self.manager = manager
+        self.closed = False
+
+    async def close(self):
+        self.closed = True
+        if self.manager:
+            self.manager.contexts_by_config.pop("second", None)
+
+
+@pytest.mark.asyncio
+async def test_close_iterates_over_context_snapshot():
+    manager = BrowserManager.__new__(BrowserManager)
+    manager.config = SimpleNamespace(cdp_url=None, sleep_on_close=False)
+    manager.sessions = {}
+    manager.browser = manager.managed_browser = manager.playwright = None
+    manager.logger = SimpleNamespace(error=lambda **_: None)
+    manager._using_cached_cdp = manager._launched_persistent = False
+    manager._context_refcounts = manager._context_last_used = manager._page_to_sig = {}
+    first, second = _Context(manager), _Context()
+    manager.contexts_by_config = {"first": first, "second": second}
+
+    await manager.close()
+
+    assert first.closed
+    assert second.closed
+    assert manager.contexts_by_config == {}
diff --git a/tests/browser/test_builtin_browser.py b/tests/browser/test_builtin_browser.py
new file mode 100644
index 0000000..4797648
--- /dev/null
+++ b/tests/browser/test_builtin_browser.py
@@ -0,0 +1,809 @@
+"""
+Test script for builtin browser functionality in the browser module.
+
+This script tests:
+1. Creating a builtin browser
+2. Getting browser information
+3. Killing the browser
+4. Restarting the browser
+5. Testing operations with different browser strategies
+6. Testing edge cases
+"""
+
+import asyncio
+import os
+import sys
+import time
+from typing import List, Dict, Any
+from colorama import Fore, Style, init
+
+# Add the project root to the path for imports
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
+
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.text import Text
+from rich.box import Box, SIMPLE
+
+from crawl4ai.browser import BrowserManager
+from crawl4ai.browser.strategies import BuiltinBrowserStrategy
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Initialize colorama for cross-platform colored terminal output
+init()
+
+# Define colors for pretty output
+SUCCESS = Fore.GREEN
+WARNING = Fore.YELLOW
+ERROR = Fore.RED
+INFO = Fore.CYAN
+RESET = Fore.RESET
+
+# Create logger
+logger = AsyncLogger(verbose=True)
+
+
+async def test_builtin_browser_creation():
+    """Test creating a builtin browser using the BrowserManager with BuiltinBrowserStrategy"""
+    print(f"\n{INFO}========== Testing Builtin Browser Creation =========={RESET}")
+
+    # Step 1: Create a BrowserManager with builtin mode
+    print(f"\n{INFO}1. Creating BrowserManager with builtin mode{RESET}")
+    browser_config = BrowserConfig(browser_mode="builtin", headless=True, verbose=True)
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+
+    # Step 2: Check if we have a BuiltinBrowserStrategy
+    print(f"\n{INFO}2. Checking if we have a BuiltinBrowserStrategy{RESET}")
+    if isinstance(manager.strategy, BuiltinBrowserStrategy):
+        print(
+            f"{SUCCESS}Correct strategy type: {manager.strategy.__class__.__name__}{RESET}"
+        )
+    else:
+        print(
+            f"{ERROR}Wrong strategy type: {manager.strategy.__class__.__name__}{RESET}"
+        )
+        return None
+
+    # Step 3: Start the manager to launch or connect to builtin browser
+    print(f"\n{INFO}3. Starting the browser manager{RESET}")
+    try:
+        await manager.start()
+        print(f"{SUCCESS}Browser manager started successfully{RESET}")
+    except Exception as e:
+        print(f"{ERROR}Failed to start browser manager: {str(e)}{RESET}")
+        return None
+
+    # Step 4: Get browser info from the strategy
+    print(f"\n{INFO}4. Getting browser information{RESET}")
+    browser_info = manager.strategy.get_browser_info()
+    if browser_info:
+        print(f"{SUCCESS}Browser info retrieved:{RESET}")
+        for key, value in browser_info.items():
+            if key != "config":  # Skip the verbose config section
+                print(f"  {key}: {value}")
+
+        cdp_url = browser_info.get("cdp_url")
+        print(f"{SUCCESS}CDP URL: {cdp_url}{RESET}")
+    else:
+        print(f"{ERROR}Failed to get browser information{RESET}")
+        cdp_url = None
+
+    # Save manager for later tests
+    return manager, cdp_url
+
+
+async def test_page_operations(manager: BrowserManager):
+    """Test page operations with the builtin browser"""
+    print(
+        f"\n{INFO}========== Testing Page Operations with Builtin Browser =========={RESET}"
+    )
+
+    # Step 1: Get a single page
+    print(f"\n{INFO}1. Getting a single page{RESET}")
+    try:
+        crawler_config = CrawlerRunConfig()
+        page, context = await manager.get_page(crawler_config)
+        print(f"{SUCCESS}Got page successfully{RESET}")
+
+        # Navigate to a test URL
+        await page.goto("https://example.com")
+        title = await page.title()
+        print(f"{SUCCESS}Page title: {title}{RESET}")
+
+        # Close the page
+        await page.close()
+        print(f"{SUCCESS}Page closed successfully{RESET}")
+    except Exception as e:
+        print(f"{ERROR}Page operation failed: {str(e)}{RESET}")
+        return False
+
+    # Step 2: Get multiple pages
+    print(f"\n{INFO}2. Getting multiple pages with get_pages(){RESET}")
+    try:
+        # Request 3 pages
+        crawler_config = CrawlerRunConfig()
+        pages = await manager.get_pages(crawler_config, count=3)
+        print(f"{SUCCESS}Got {len(pages)} pages{RESET}")
+
+        # Test each page
+        for i, (page, context) in enumerate(pages):
+            await page.goto(f"https://example.com?test={i}")
+            title = await page.title()
+            print(f"{SUCCESS}Page {i + 1} title: {title}{RESET}")
+            await page.close()
+
+        print(f"{SUCCESS}All pages tested and closed successfully{RESET}")
+    except Exception as e:
+        print(f"{ERROR}Multiple page operation failed: {str(e)}{RESET}")
+        return False
+
+    return True
+
+
+async def test_browser_status_management(manager: BrowserManager):
+    """Test browser status and management operations"""
+    print(f"\n{INFO}========== Testing Browser Status and Management =========={RESET}")
+
+    # Step 1: Get browser status
+    print(f"\n{INFO}1. Getting browser status{RESET}")
+    try:
+        status = await manager.strategy.get_builtin_browser_status()
+        print(f"{SUCCESS}Browser status:{RESET}")
+        print(f"  Running: {status['running']}")
+        print(f"  CDP URL: {status['cdp_url']}")
+    except Exception as e:
+        print(f"{ERROR}Failed to get browser status: {str(e)}{RESET}")
+        return False
+
+    # Step 2: Test killing the browser
+    print(f"\n{INFO}2. Testing killing the browser{RESET}")
+    try:
+        result = await manager.strategy.kill_builtin_browser()
+        if result:
+            print(f"{SUCCESS}Browser killed successfully{RESET}")
+        else:
+            print(f"{ERROR}Failed to kill browser{RESET}")
+    except Exception as e:
+        print(f"{ERROR}Browser kill operation failed: {str(e)}{RESET}")
+        return False
+
+    # Step 3: Check status after kill
+    print(f"\n{INFO}3. Checking status after kill{RESET}")
+    try:
+        status = await manager.strategy.get_builtin_browser_status()
+        if not status["running"]:
+            print(f"{SUCCESS}Browser is correctly reported as not running{RESET}")
+        else:
+            print(f"{ERROR}Browser is incorrectly reported as still running{RESET}")
+    except Exception as e:
+        print(f"{ERROR}Failed to get browser status: {str(e)}{RESET}")
+        return False
+
+    # Step 4: Launch a new browser
+    print(f"\n{INFO}4. Launching a new browser{RESET}")
+    try:
+        cdp_url = await manager.strategy.launch_builtin_browser(
+            browser_type="chromium", headless=True
+        )
+        if cdp_url:
+            print(f"{SUCCESS}New browser launched at: {cdp_url}{RESET}")
+        else:
+            print(f"{ERROR}Failed to launch new browser{RESET}")
+            return False
+    except Exception as e:
+        print(f"{ERROR}Browser launch failed: {str(e)}{RESET}")
+        return False
+
+    return True
+
+
+async def test_multiple_managers():
+    """Test creating multiple BrowserManagers that use the same builtin browser"""
+    print(f"\n{INFO}========== Testing Multiple Browser Managers =========={RESET}")
+
+    # Step 1: Create first manager
+    print(f"\n{INFO}1. Creating first browser manager{RESET}")
+    browser_config1 = BrowserConfig(browser_mode="builtin", headless=True)
+    manager1 = BrowserManager(browser_config=browser_config1, logger=logger)
+
+    # Step 2: Create second manager
+    print(f"\n{INFO}2. Creating second browser manager{RESET}")
+    browser_config2 = BrowserConfig(browser_mode="builtin", headless=True)
+    manager2 = BrowserManager(browser_config=browser_config2, logger=logger)
+
+    # Step 3: Start both managers (should connect to the same builtin browser)
+    print(f"\n{INFO}3. Starting both managers{RESET}")
+    try:
+        await manager1.start()
+        print(f"{SUCCESS}First manager started{RESET}")
+
+        await manager2.start()
+        print(f"{SUCCESS}Second manager started{RESET}")
+
+        # Check if they got the same CDP URL
+        cdp_url1 = manager1.strategy.config.cdp_url
+        cdp_url2 = manager2.strategy.config.cdp_url
+
+        if cdp_url1 == cdp_url2:
+            print(
+                f"{SUCCESS}Both managers connected to the same browser: {cdp_url1}{RESET}"
+            )
+        else:
+            print(
+                f"{WARNING}Managers connected to different browsers: {cdp_url1} and {cdp_url2}{RESET}"
+            )
+    except Exception as e:
+        print(f"{ERROR}Failed to start managers: {str(e)}{RESET}")
+        return False
+
+    # Step 4: Test using both managers
+    print(f"\n{INFO}4. Testing operations with both managers{RESET}")
+    try:
+        # First manager creates a page
+        page1, ctx1 = await manager1.get_page(CrawlerRunConfig())
+        await page1.goto("https://example.com")
+        title1 = await page1.title()
+        print(f"{SUCCESS}Manager 1 page title: {title1}{RESET}")
+
+        # Second manager creates a page
+        page2, ctx2 = await manager2.get_page(CrawlerRunConfig())
+        await page2.goto("https://example.org")
+        title2 = await page2.title()
+        print(f"{SUCCESS}Manager 2 page title: {title2}{RESET}")
+
+        # Clean up
+        await page1.close()
+        await page2.close()
+    except Exception as e:
+        print(f"{ERROR}Failed to use both managers: {str(e)}{RESET}")
+        return False
+
+    # Step 5: Close both managers
+    print(f"\n{INFO}5. Closing both managers{RESET}")
+    try:
+        await manager1.close()
+        print(f"{SUCCESS}First manager closed{RESET}")
+
+        await manager2.close()
+        print(f"{SUCCESS}Second manager closed{RESET}")
+    except Exception as e:
+        print(f"{ERROR}Failed to close managers: {str(e)}{RESET}")
+        return False
+
+    return True
+
+
+async def test_edge_cases():
+    """Test edge cases like multiple starts, killing browser during operations, etc."""
+    print(f"\n{INFO}========== Testing Edge Cases =========={RESET}")
+
+    # Step 1: Test multiple starts with the same manager
+    print(f"\n{INFO}1. Testing multiple starts with the same manager{RESET}")
+    browser_config = BrowserConfig(browser_mode="builtin", headless=True)
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+
+    try:
+        await manager.start()
+        print(f"{SUCCESS}First start successful{RESET}")
+
+        # Try to start again
+        await manager.start()
+        print(f"{SUCCESS}Second start completed without errors{RESET}")
+
+        # Test if it's still functional
+        page, context = await manager.get_page(CrawlerRunConfig())
+        await page.goto("https://example.com")
+        title = await page.title()
+        print(
+            f"{SUCCESS}Page operations work after multiple starts. Title: {title}{RESET}"
+        )
+        await page.close()
+    except Exception as e:
+        print(f"{ERROR}Multiple starts test failed: {str(e)}{RESET}")
+        return False
+    finally:
+        await manager.close()
+
+    # Step 2: Test killing the browser while manager is active
+    print(f"\n{INFO}2. Testing killing the browser while manager is active{RESET}")
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+
+    try:
+        await manager.start()
+        print(f"{SUCCESS}Manager started{RESET}")
+
+        # Kill the browser directly
+        print(f"{INFO}Killing the browser...{RESET}")
+        await manager.strategy.kill_builtin_browser()
+        print(f"{SUCCESS}Browser killed{RESET}")
+
+        # Try to get a page (should fail or launch a new browser)
+        try:
+            page, context = await manager.get_page(CrawlerRunConfig())
+            print(
+                f"{WARNING}Page request succeeded despite killed browser (might have auto-restarted){RESET}"
+            )
+            title = await page.title()
+            print(f"{SUCCESS}Got page title: {title}{RESET}")
+            await page.close()
+        except Exception as e:
+            print(
+                f"{SUCCESS}Page request failed as expected after browser was killed: {str(e)}{RESET}"
+            )
+    except Exception as e:
+        print(f"{ERROR}Kill during operation test failed: {str(e)}{RESET}")
+        return False
+    finally:
+        await manager.close()
+
+    return True
+
+
+async def cleanup_browsers():
+    """Clean up any remaining builtin browsers"""
+    print(f"\n{INFO}========== Cleaning Up Builtin Browsers =========={RESET}")
+
+    browser_config = BrowserConfig(browser_mode="builtin", headless=True)
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+
+    try:
+        # No need to start, just access the strategy directly
+        strategy = manager.strategy
+        if isinstance(strategy, BuiltinBrowserStrategy):
+            result = await strategy.kill_builtin_browser()
+            if result:
+                print(f"{SUCCESS}Successfully killed all builtin browsers{RESET}")
+            else:
+                print(f"{WARNING}No builtin browsers found to kill{RESET}")
+        else:
+            print(f"{ERROR}Wrong strategy type: {strategy.__class__.__name__}{RESET}")
+    except Exception as e:
+        print(f"{ERROR}Cleanup failed: {str(e)}{RESET}")
+    finally:
+        # Just to be safe
+        try:
+            await manager.close()
+        except:
+            pass
+
+
+async def test_performance_scaling():
+    """Test performance with multiple browsers and pages.
+
+    This test creates multiple browsers on different ports,
+    spawns multiple pages per browser, and measures performance metrics.
+    """
+    print(f"\n{INFO}========== Testing Performance Scaling =========={RESET}")
+
+    # Configuration parameters
+    num_browsers = 10
+    pages_per_browser = 10
+    total_pages = num_browsers * pages_per_browser
+    base_port = 9222
+
+    # Set up a measuring mechanism for memory
+    import psutil
+    import gc
+
+    # Force garbage collection before starting
+    gc.collect()
+    process = psutil.Process()
+    initial_memory = process.memory_info().rss / 1024 / 1024  # in MB
+    peak_memory = initial_memory
+
+    # Report initial configuration
+    print(
+        f"{INFO}Test configuration: {num_browsers} browsers × {pages_per_browser} pages = {total_pages} total crawls{RESET}"
+    )
+
+    # List to track managers
+    managers: List[BrowserManager] = []
+    all_pages = []
+
+
+
+    # Get crawl4ai home directory
+    crawl4ai_home = os.path.expanduser("~/.crawl4ai")
+    temp_dir = os.path.join(crawl4ai_home, "temp")
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # Create all managers but don't start them yet
+    manager_configs = []
+    for i in range(num_browsers):
+        port = base_port + i
+        browser_config = BrowserConfig(
+            browser_mode="builtin",
+            headless=True,
+            debugging_port=port,
+            user_data_dir=os.path.join(temp_dir, f"browser_profile_{i}"),
+        )
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        manager.strategy.shutting_down = True
+        manager_configs.append((manager, i, port))
+
+    # Define async function to start a single manager
+    async def start_manager(manager, index, port):
+        try:
+            await manager.start()
+            return manager
+        except Exception as e:
+            print(
+                f"{ERROR}Failed to start browser {index + 1} on port {port}: {str(e)}{RESET}"
+            )
+            return None
+
+    # Start all managers in parallel
+    start_tasks = [
+        start_manager(manager, i, port) for manager, i, port in manager_configs
+    ]
+    started_managers = await asyncio.gather(*start_tasks)
+
+    # Filter out None values (failed starts) and add to managers list
+    managers = [m for m in started_managers if m is not None]
+
+    if len(managers) == 0:
+        print(f"{ERROR}All browser managers failed to start. Aborting test.{RESET}")
+        return False
+
+    if len(managers) < num_browsers:
+        print(
+            f"{WARNING}Only {len(managers)} out of {num_browsers} browser managers started successfully{RESET}"
+        )
+
+    # Create pages for each browser
+    for i, manager in enumerate(managers):
+        try:
+            pages = await manager.get_pages(CrawlerRunConfig(), count=pages_per_browser)
+            all_pages.extend(pages)
+        except Exception as e:
+            print(f"{ERROR}Failed to create pages for browser {i + 1}: {str(e)}{RESET}")
+
+    # Check memory after page creation
+    gc.collect()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    peak_memory = max(peak_memory, current_memory)
+
+    # Ask for confirmation before loading
+    confirmation = input(
+        f"{WARNING}Do you want to proceed with loading pages? (y/n): {RESET}"
+    )
+    # Step 1: Create and start multiple browser managers in parallel
+    start_time = time.time()
+    
+    if confirmation.lower() == "y":
+        load_start_time = time.time()
+
+        # Function to load a single page
+        async def load_page(page_ctx, index):
+            page, _ = page_ctx
+            try:
+                await page.goto(f"https://example.com/page{index}", timeout=30000)
+                title = await page.title()
+                return title
+            except Exception as e:
+                return f"Error: {str(e)}"
+
+        # Load all pages concurrently
+        load_tasks = [load_page(page_ctx, i) for i, page_ctx in enumerate(all_pages)]
+        load_results = await asyncio.gather(*load_tasks, return_exceptions=True)
+
+        # Count successes and failures
+        successes = sum(
+            1 for r in load_results if isinstance(r, str) and not r.startswith("Error")
+        )
+        failures = len(load_results) - successes
+
+        load_time = time.time() - load_start_time
+        total_test_time = time.time() - start_time
+
+        # Check memory after loading (peak memory)
+        gc.collect()
+        current_memory = process.memory_info().rss / 1024 / 1024
+        peak_memory = max(peak_memory, current_memory)
+
+        # Calculate key metrics
+        memory_per_page = peak_memory / successes if successes > 0 else 0
+        time_per_crawl = total_test_time / successes if successes > 0 else 0
+        crawls_per_second = successes / total_test_time if total_test_time > 0 else 0
+        crawls_per_minute = crawls_per_second * 60
+        crawls_per_hour = crawls_per_minute * 60
+
+        # Print simplified performance summary
+        from rich.console import Console
+        from rich.table import Table
+
+        console = Console()
+
+        # Create a simple summary table
+        table = Table(title="CRAWL4AI PERFORMANCE SUMMARY")
+
+        table.add_column("Metric", style="cyan")
+        table.add_column("Value", style="green")
+
+        table.add_row("Total Crawls Completed", f"{successes}")
+        table.add_row("Total Time", f"{total_test_time:.2f} seconds")
+        table.add_row("Time Per Crawl", f"{time_per_crawl:.2f} seconds")
+        table.add_row("Crawling Speed", f"{crawls_per_second:.2f} crawls/second")
+        table.add_row("Projected Rate (1 minute)", f"{crawls_per_minute:.0f} crawls")
+        table.add_row("Projected Rate (1 hour)", f"{crawls_per_hour:.0f} crawls")
+        table.add_row("Peak Memory Usage", f"{peak_memory:.2f} MB")
+        table.add_row("Memory Per Crawl", f"{memory_per_page:.2f} MB")
+
+        # Display the table
+        console.print(table)
+
+    # Ask confirmation before cleanup
+    confirmation = input(
+        f"{WARNING}Do you want to proceed with cleanup? (y/n): {RESET}"
+    )
+    if confirmation.lower() != "y":
+        print(f"{WARNING}Cleanup aborted by user{RESET}")
+        return False
+
+    # Close all pages
+    for page, _ in all_pages:
+        try:
+            await page.close()
+        except:
+            pass
+
+    # Close all managers
+    for manager in managers:
+        try:
+            await manager.close()
+        except:
+            pass
+
+    # Remove the temp directory
+    import shutil
+
+    if os.path.exists(temp_dir):
+        shutil.rmtree(temp_dir)
+
+    return True
+
+
+async def test_performance_scaling_lab( num_browsers: int = 10, pages_per_browser: int = 10):
+    """Test performance with multiple browsers and pages.
+
+    This test creates multiple browsers on different ports,
+    spawns multiple pages per browser, and measures performance metrics.
+    """
+    print(f"\n{INFO}========== Testing Performance Scaling =========={RESET}")
+
+    # Configuration parameters
+    num_browsers = num_browsers
+    pages_per_browser = pages_per_browser
+    total_pages = num_browsers * pages_per_browser
+    base_port = 9222
+
+    # Set up a measuring mechanism for memory
+    import psutil
+    import gc
+
+    # Force garbage collection before starting
+    gc.collect()
+    process = psutil.Process()
+    initial_memory = process.memory_info().rss / 1024 / 1024  # in MB
+    peak_memory = initial_memory
+
+    # Report initial configuration
+    print(
+        f"{INFO}Test configuration: {num_browsers} browsers × {pages_per_browser} pages = {total_pages} total crawls{RESET}"
+    )
+
+    # List to track managers
+    managers: List[BrowserManager] = []
+    all_pages = []
+
+    # Get crawl4ai home directory
+    crawl4ai_home = os.path.expanduser("~/.crawl4ai")
+    temp_dir = os.path.join(crawl4ai_home, "temp")
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # Create all managers but don't start them yet
+    manager_configs = []
+    for i in range(num_browsers):
+        port = base_port + i
+        browser_config = BrowserConfig(
+            browser_mode="builtin",
+            headless=True,
+            debugging_port=port,
+            user_data_dir=os.path.join(temp_dir, f"browser_profile_{i}"),
+        )
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        manager.strategy.shutting_down = True
+        manager_configs.append((manager, i, port))
+
+    # Define async function to start a single manager
+    async def start_manager(manager, index, port):
+        try:
+            await manager.start()
+            return manager
+        except Exception as e:
+            print(
+                f"{ERROR}Failed to start browser {index + 1} on port {port}: {str(e)}{RESET}"
+            )
+            return None
+
+    # Start all managers in parallel
+    start_tasks = [
+        start_manager(manager, i, port) for manager, i, port in manager_configs
+    ]
+    started_managers = await asyncio.gather(*start_tasks)
+
+    # Filter out None values (failed starts) and add to managers list
+    managers = [m for m in started_managers if m is not None]
+
+    if len(managers) == 0:
+        print(f"{ERROR}All browser managers failed to start. Aborting test.{RESET}")
+        return False
+
+    if len(managers) < num_browsers:
+        print(
+            f"{WARNING}Only {len(managers)} out of {num_browsers} browser managers started successfully{RESET}"
+        )
+
+    # Create pages for each browser
+    for i, manager in enumerate(managers):
+        try:
+            pages = await manager.get_pages(CrawlerRunConfig(), count=pages_per_browser)
+            all_pages.extend(pages)
+        except Exception as e:
+            print(f"{ERROR}Failed to create pages for browser {i + 1}: {str(e)}{RESET}")
+
+    # Check memory after page creation
+    gc.collect()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    peak_memory = max(peak_memory, current_memory)
+
+    # Ask for confirmation before loading
+    confirmation = input(
+        f"{WARNING}Do you want to proceed with loading pages? (y/n): {RESET}"
+    )
+    # Step 1: Create and start multiple browser managers in parallel
+    start_time = time.time()
+    
+    if confirmation.lower() == "y":
+        load_start_time = time.time()
+
+        # Function to load a single page
+        async def load_page(page_ctx, index):
+            page, _ = page_ctx
+            try:
+                await page.goto(f"https://example.com/page{index}", timeout=30000)
+                title = await page.title()
+                return title
+            except Exception as e:
+                return f"Error: {str(e)}"
+
+        # Load all pages concurrently
+        load_tasks = [load_page(page_ctx, i) for i, page_ctx in enumerate(all_pages)]
+        load_results = await asyncio.gather(*load_tasks, return_exceptions=True)
+
+        # Count successes and failures
+        successes = sum(
+            1 for r in load_results if isinstance(r, str) and not r.startswith("Error")
+        )
+        failures = len(load_results) - successes
+
+        load_time = time.time() - load_start_time
+        total_test_time = time.time() - start_time
+
+        # Check memory after loading (peak memory)
+        gc.collect()
+        current_memory = process.memory_info().rss / 1024 / 1024
+        peak_memory = max(peak_memory, current_memory)
+
+        # Calculate key metrics
+        memory_per_page = peak_memory / successes if successes > 0 else 0
+        time_per_crawl = total_test_time / successes if successes > 0 else 0
+        crawls_per_second = successes / total_test_time if total_test_time > 0 else 0
+        crawls_per_minute = crawls_per_second * 60
+        crawls_per_hour = crawls_per_minute * 60
+
+        # Print simplified performance summary
+        from rich.console import Console
+        from rich.table import Table
+
+        console = Console()
+
+        # Create a simple summary table
+        table = Table(title="CRAWL4AI PERFORMANCE SUMMARY")
+
+        table.add_column("Metric", style="cyan")
+        table.add_column("Value", style="green")
+
+        table.add_row("Total Crawls Completed", f"{successes}")
+        table.add_row("Total Time", f"{total_test_time:.2f} seconds")
+        table.add_row("Time Per Crawl", f"{time_per_crawl:.2f} seconds")
+        table.add_row("Crawling Speed", f"{crawls_per_second:.2f} crawls/second")
+        table.add_row("Projected Rate (1 minute)", f"{crawls_per_minute:.0f} crawls")
+        table.add_row("Projected Rate (1 hour)", f"{crawls_per_hour:.0f} crawls")
+        table.add_row("Peak Memory Usage", f"{peak_memory:.2f} MB")
+        table.add_row("Memory Per Crawl", f"{memory_per_page:.2f} MB")
+
+        # Display the table
+        console.print(table)
+
+    # Ask confirmation before cleanup
+    confirmation = input(
+        f"{WARNING}Do you want to proceed with cleanup? (y/n): {RESET}"
+    )
+    if confirmation.lower() != "y":
+        print(f"{WARNING}Cleanup aborted by user{RESET}")
+        return False
+
+    # Close all pages
+    for page, _ in all_pages:
+        try:
+            await page.close()
+        except:
+            pass
+
+    # Close all managers
+    for manager in managers:
+        try:
+            await manager.close()
+        except:
+            pass
+
+    # Remove the temp directory
+    import shutil
+
+    if os.path.exists(temp_dir):
+        shutil.rmtree(temp_dir)
+
+    return True
+
+
+
+async def main():
+    """Run all tests"""
+    try:
+        print(f"{INFO}Starting builtin browser tests with browser module{RESET}")
+
+        # # Run browser creation test
+        # manager, cdp_url = await test_builtin_browser_creation()
+        # if not manager:
+        #     print(f"{ERROR}Browser creation failed, cannot continue tests{RESET}")
+        #     return
+
+        # # Run page operations test
+        # await test_page_operations(manager)
+
+        # # Run browser status and management test
+        # await test_browser_status_management(manager)
+
+        # # Close manager before multiple manager test
+        # await manager.close()
+
+        # Run multiple managers test
+        await test_multiple_managers()
+
+        # Run performance scaling test
+        await test_performance_scaling()
+
+        # Run cleanup test
+        await cleanup_browsers()
+
+        # Run edge cases test
+        await test_edge_cases()
+
+        print(f"\n{SUCCESS}All tests completed!{RESET}")
+
+    except Exception as e:
+        print(f"\n{ERROR}Test failed with error: {str(e)}{RESET}")
+        import traceback
+
+        traceback.print_exc()
+    finally:
+        # Clean up: kill any remaining builtin browsers
+        await cleanup_browsers()
+        print(f"{SUCCESS}Test cleanup complete{RESET}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/browser/test_builtin_strategy.py b/tests/browser/test_builtin_strategy.py
new file mode 100644
index 0000000..7c435b3
--- /dev/null
+++ b/tests/browser/test_builtin_strategy.py
@@ -0,0 +1,160 @@
+"""Test examples for BuiltinBrowserStrategy.
+
+These examples demonstrate the functionality of BuiltinBrowserStrategy
+and serve as functional tests.
+"""
+
+import asyncio
+import os
+import sys
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai.browser import BrowserManager
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+async def test_builtin_browser():
+    """Test using a builtin browser that persists between sessions."""
+    logger.info("Testing builtin browser", tag="TEST")
+    
+    browser_config = BrowserConfig(
+        browser_mode="builtin",
+        headless=True
+    )
+    
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        # Start should connect to existing builtin browser or create one
+        await manager.start()
+        logger.info("Connected to builtin browser", tag="TEST")
+        
+        # Test page creation
+        crawler_config = CrawlerRunConfig()
+        page, context = await manager.get_page(crawler_config)
+        
+        # Test navigation
+        await page.goto("https://example.com")
+        title = await page.title()
+        logger.info(f"Page title: {title}", tag="TEST")
+        
+        # Close manager (should not close the builtin browser)
+        await manager.close()
+        logger.info("First session closed", tag="TEST")
+        
+        # Create a second manager to verify browser persistence
+        logger.info("Creating second session to verify persistence", tag="TEST")
+        manager2 = BrowserManager(browser_config=browser_config, logger=logger)
+        
+        await manager2.start()
+        logger.info("Connected to existing builtin browser", tag="TEST")
+        
+        page2, context2 = await manager2.get_page(crawler_config)
+        await page2.goto("https://example.org")
+        title2 = await page2.title()
+        logger.info(f"Second session page title: {title2}", tag="TEST")
+        
+        await manager2.close()
+        logger.info("Second session closed successfully", tag="TEST")
+        
+        return True
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_builtin_browser_status():
+    """Test getting status of the builtin browser."""
+    logger.info("Testing builtin browser status", tag="TEST")
+    
+    from crawl4ai.browser.strategies import BuiltinBrowserStrategy
+    
+    browser_config = BrowserConfig(
+        browser_mode="builtin",
+        headless=True
+    )
+    
+    # Create strategy directly to access its status methods
+    strategy = BuiltinBrowserStrategy(browser_config, logger)
+    
+    try:
+        # Get status before starting (should be not running)
+        status_before = await strategy.get_builtin_browser_status()
+        logger.info(f"Initial status: {status_before}", tag="TEST")
+        
+        # Start the browser
+        await strategy.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Get status after starting
+        status_after = await strategy.get_builtin_browser_status()
+        logger.info(f"Status after start: {status_after}", tag="TEST")
+        
+        # Create a page to verify functionality
+        crawler_config = CrawlerRunConfig()
+        page, context = await strategy.get_page(crawler_config)
+        await page.goto("https://example.com")
+        title = await page.title()
+        logger.info(f"Page title: {title}", tag="TEST")
+        
+        # Close strategy (should not kill the builtin browser)
+        await strategy.close()
+        logger.info("Strategy closed successfully", tag="TEST")
+        
+        # Create a new strategy object
+        strategy2 = BuiltinBrowserStrategy(browser_config, logger)
+        
+        # Get status again (should still be running)
+        status_final = await strategy2.get_builtin_browser_status()
+        logger.info(f"Final status: {status_final}", tag="TEST")
+        
+        # Verify that the status shows the browser is running
+        is_running = status_final.get('running', False)
+        logger.info(f"Builtin browser persistence confirmed: {is_running}", tag="TEST")
+        
+        # Kill the builtin browser to clean up
+        logger.info("Killing builtin browser", tag="TEST")
+        success = await strategy2.kill_builtin_browser()
+        logger.info(f"Killed builtin browser successfully: {success}", tag="TEST")
+        
+        return is_running and success
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        try:
+            await strategy.close()
+            
+            # Try to kill the builtin browser to clean up
+            strategy2 = BuiltinBrowserStrategy(browser_config, logger)
+            await strategy2.kill_builtin_browser()
+        except:
+            pass
+        return False
+
+async def run_tests():
+    """Run all tests sequentially."""
+    results = []
+    
+    results.append(await test_builtin_browser())
+    results.append(await test_builtin_browser_status())
+    
+    # Print summary
+    total = len(results)
+    passed = sum(results)
+    logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
+    
+    if passed == total:
+        logger.success("All tests passed!", tag="SUMMARY")
+    else:
+        logger.error(f"{total - passed} tests failed", tag="SUMMARY")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
diff --git a/tests/browser/test_cdp_cleanup_reuse.py b/tests/browser/test_cdp_cleanup_reuse.py
new file mode 100644
index 0000000..dc5a93f
--- /dev/null
+++ b/tests/browser/test_cdp_cleanup_reuse.py
@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+"""
+Tests for CDP connection cleanup and browser reuse.
+
+These tests verify that:
+1. WebSocket URLs are properly handled (skip HTTP verification)
+2. cdp_cleanup_on_close properly disconnects without terminating the browser
+3. The same browser can be reused by multiple sequential connections
+
+Requirements:
+- A CDP-compatible browser pool service running (e.g., chromepoold)
+- Service should be accessible at CDP_SERVICE_URL (default: http://localhost:11235)
+
+Usage:
+    pytest tests/browser/test_cdp_cleanup_reuse.py -v
+
+Or run directly:
+    python tests/browser/test_cdp_cleanup_reuse.py
+"""
+
+import asyncio
+import os
+import pytest
+import requests
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+# Configuration
+CDP_SERVICE_URL = os.getenv("CDP_SERVICE_URL", "http://localhost:11235")
+
+
+def is_cdp_service_available():
+    """Check if CDP service is running."""
+    try:
+        resp = requests.get(f"{CDP_SERVICE_URL}/health", timeout=2)
+        return resp.status_code == 200
+    except:
+        return False
+
+
+def create_browser():
+    """Create a browser via CDP service API."""
+    resp = requests.post(
+        f"{CDP_SERVICE_URL}/v1/browsers",
+        json={"headless": True},
+        timeout=10
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+def get_browser_info(browser_id):
+    """Get browser info from CDP service."""
+    resp = requests.get(f"{CDP_SERVICE_URL}/v1/browsers", timeout=5)
+    for browser in resp.json():
+        if browser["id"] == browser_id:
+            return browser
+    return None
+
+
+def delete_browser(browser_id):
+    """Delete a browser via CDP service API."""
+    try:
+        requests.delete(f"{CDP_SERVICE_URL}/v1/browsers/{browser_id}", timeout=5)
+    except:
+        pass
+
+
+# Skip all tests if CDP service is not available
+pytestmark = pytest.mark.skipif(
+    not is_cdp_service_available(),
+    reason=f"CDP service not available at {CDP_SERVICE_URL}"
+)
+
+
+class TestCDPWebSocketURL:
+    """Tests for WebSocket URL handling."""
+
+    @pytest.mark.asyncio
+    async def test_websocket_url_skips_http_verification(self):
+        """WebSocket URLs should skip HTTP /json/version verification."""
+        browser = create_browser()
+        try:
+            ws_url = browser["ws_url"]
+            assert ws_url.startswith("ws://") or ws_url.startswith("wss://")
+
+            async with AsyncWebCrawler(
+                config=BrowserConfig(
+                    browser_mode="cdp",
+                    cdp_url=ws_url,
+                    headless=True,
+                    cdp_cleanup_on_close=True,
+                )
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://example.com",
+                    config=CrawlerRunConfig(verbose=False),
+                )
+                assert result.success
+                assert "Example Domain" in result.metadata.get("title", "")
+        finally:
+            delete_browser(browser["browser_id"])
+
+
+class TestCDPCleanupOnClose:
+    """Tests for cdp_cleanup_on_close behavior."""
+
+    @pytest.mark.asyncio
+    async def test_browser_survives_after_cleanup_close(self):
+        """Browser should remain alive after close with cdp_cleanup_on_close=True."""
+        browser = create_browser()
+        browser_id = browser["browser_id"]
+        ws_url = browser["ws_url"]
+
+        try:
+            # Verify browser exists
+            info_before = get_browser_info(browser_id)
+            assert info_before is not None
+            pid_before = info_before["pid"]
+
+            # Connect, crawl, and close with cleanup
+            async with AsyncWebCrawler(
+                config=BrowserConfig(
+                    browser_mode="cdp",
+                    cdp_url=ws_url,
+                    headless=True,
+                    cdp_cleanup_on_close=True,
+                )
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://example.com",
+                    config=CrawlerRunConfig(verbose=False),
+                )
+                assert result.success
+
+            # Browser should still exist with same PID
+            info_after = get_browser_info(browser_id)
+            assert info_after is not None, "Browser was terminated but should only disconnect"
+            assert info_after["pid"] == pid_before, "Browser PID changed unexpectedly"
+        finally:
+            delete_browser(browser_id)
+
+
+class TestCDPBrowserReuse:
+    """Tests for reusing the same browser with multiple connections."""
+
+    @pytest.mark.asyncio
+    async def test_sequential_connections_same_browser(self):
+        """Multiple sequential connections to the same browser should work."""
+        browser = create_browser()
+        browser_id = browser["browser_id"]
+        ws_url = browser["ws_url"]
+
+        try:
+            urls = [
+                "https://example.com",
+                "https://httpbin.org/ip",
+                "https://httpbin.org/headers",
+            ]
+
+            for i, url in enumerate(urls, 1):
+                # Each connection uses cdp_cleanup_on_close=True
+                async with AsyncWebCrawler(
+                    config=BrowserConfig(
+                        browser_mode="cdp",
+                        cdp_url=ws_url,
+                        headless=True,
+                        cdp_cleanup_on_close=True,
+                    )
+                ) as crawler:
+                    result = await crawler.arun(
+                        url=url,
+                        config=CrawlerRunConfig(verbose=False),
+                    )
+                    assert result.success, f"Connection {i} failed for {url}"
+
+                # Verify browser is still healthy
+                info = get_browser_info(browser_id)
+                assert info is not None, f"Browser died after connection {i}"
+
+        finally:
+            delete_browser(browser_id)
+
+    @pytest.mark.asyncio
+    async def test_no_user_wait_needed_between_connections(self):
+        """With cdp_cleanup_on_close=True, no user wait should be needed."""
+        browser = create_browser()
+        browser_id = browser["browser_id"]
+        ws_url = browser["ws_url"]
+
+        try:
+            # Rapid-fire connections with NO sleep between them
+            for i in range(3):
+                async with AsyncWebCrawler(
+                    config=BrowserConfig(
+                        browser_mode="cdp",
+                        cdp_url=ws_url,
+                        headless=True,
+                        cdp_cleanup_on_close=True,
+                    )
+                ) as crawler:
+                    result = await crawler.arun(
+                        url="https://example.com",
+                        config=CrawlerRunConfig(verbose=False),
+                    )
+                    assert result.success, f"Rapid connection {i+1} failed"
+                # NO asyncio.sleep() here - internal delay should be sufficient
+        finally:
+            delete_browser(browser_id)
+
+
+class TestCDPBackwardCompatibility:
+    """Tests for backward compatibility with existing CDP usage."""
+
+    @pytest.mark.asyncio
+    async def test_http_url_with_browser_id_works(self):
+        """HTTP URL with browser_id query param should work (backward compatibility)."""
+        browser = create_browser()
+        browser_id = browser["browser_id"]
+        try:
+            # Use HTTP URL with browser_id query parameter
+            http_url = f"{CDP_SERVICE_URL}?browser_id={browser_id}"
+
+            async with AsyncWebCrawler(
+                config=BrowserConfig(
+                    browser_mode="cdp",
+                    cdp_url=http_url,
+                    headless=True,
+                    cdp_cleanup_on_close=True,
+                )
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://example.com",
+                    config=CrawlerRunConfig(verbose=False),
+                )
+                assert result.success
+        finally:
+            delete_browser(browser_id)
+
+
+# Allow running directly
+if __name__ == "__main__":
+    if not is_cdp_service_available():
+        print(f"CDP service not available at {CDP_SERVICE_URL}")
+        print("Please start a CDP-compatible browser pool service first.")
+        exit(1)
+
+    async def run_tests():
+        print("=" * 60)
+        print("CDP Cleanup and Browser Reuse Tests")
+        print("=" * 60)
+
+        tests = [
+            ("WebSocket URL handling", TestCDPWebSocketURL().test_websocket_url_skips_http_verification),
+            ("Browser survives after cleanup", TestCDPCleanupOnClose().test_browser_survives_after_cleanup_close),
+            ("Sequential connections", TestCDPBrowserReuse().test_sequential_connections_same_browser),
+            ("No user wait needed", TestCDPBrowserReuse().test_no_user_wait_needed_between_connections),
+            ("HTTP URL with browser_id", TestCDPBackwardCompatibility().test_http_url_with_browser_id_works),
+        ]
+
+        results = []
+        for name, test_func in tests:
+            print(f"\n--- {name} ---")
+            try:
+                await test_func()
+                print(f"PASS")
+                results.append((name, True))
+            except Exception as e:
+                print(f"FAIL: {e}")
+                results.append((name, False))
+
+        print("\n" + "=" * 60)
+        print("SUMMARY")
+        print("=" * 60)
+        for name, passed in results:
+            print(f"  {name}: {'PASS' if passed else 'FAIL'}")
+
+        all_passed = all(r[1] for r in results)
+        print(f"\nOverall: {'ALL TESTS PASSED' if all_passed else 'SOME TESTS FAILED'}")
+        return 0 if all_passed else 1
+
+    exit(asyncio.run(run_tests()))
diff --git a/tests/browser/test_cdp_strategy.py b/tests/browser/test_cdp_strategy.py
new file mode 100644
index 0000000..b4b9021
--- /dev/null
+++ b/tests/browser/test_cdp_strategy.py
@@ -0,0 +1,470 @@
+"""Test examples for CDPBrowserStrategy.
+
+These examples demonstrate the functionality of CDPBrowserStrategy
+and serve as functional tests.
+"""
+
+import asyncio
+import os
+import sys
+import time
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai.browser_manager import BrowserManager
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+async def test_cdp_launch_connect():
+    """Test launching a browser and connecting via CDP."""
+    logger.info("Testing launch and connect via CDP", tag="TEST")
+    
+    browser_config = BrowserConfig(
+        browser_mode="cdp",
+        use_managed_browser=True,
+        headless=True
+    )
+    
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        await manager.start()
+        logger.info("Browser launched and connected via CDP", tag="TEST")
+        
+        # Test with multiple pages
+        pages = []
+        for i in range(3):
+            crawler_config = CrawlerRunConfig()
+            page, context = await manager.get_page(crawler_config)
+            await page.goto(f"https://example.com?test={i}")
+            pages.append(page)
+            logger.info(f"Created page {i+1}", tag="TEST")
+        
+        # Verify all pages are working
+        for i, page in enumerate(pages):
+            title = await page.title()
+            logger.info(f"Page {i+1} title: {title}", tag="TEST")
+        
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        return True
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_cdp_with_user_data_dir():
+    """Test CDP browser with a user data directory and storage state."""
+    logger.info("Testing CDP browser with user data directory", tag="TEST")
+    
+    # Create a temporary user data directory
+    import tempfile
+    user_data_dir = tempfile.mkdtemp(prefix="crawl4ai-test-")
+    storage_state_file = os.path.join(user_data_dir, "storage_state.json")
+    logger.info(f"Created temporary user data directory: {user_data_dir}", tag="TEST")
+    
+    browser_config = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        user_data_dir=user_data_dir
+    )
+    
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        await manager.start()
+        logger.info("Browser launched with user data directory", tag="TEST")
+        
+        # Navigate to a page and store some data
+        crawler_config = CrawlerRunConfig()
+        page, context = await manager.get_page(crawler_config)
+        
+        # Visit the site first
+        await page.goto("https://example.com", wait_until="domcontentloaded")
+        
+        # Set a cookie via JavaScript (more reliable for persistence)
+        await page.evaluate("""
+            document.cookie = 'test_cookie=test_value; path=/; max-age=86400';
+        """)
+        
+        # Also set via context API for double coverage
+        await context.add_cookies([{
+            "name": "test_cookie_api",
+            "value": "test_value_api",
+            "domain": "example.com",
+            "path": "/"
+        }])
+        
+        # Verify cookies were set
+        cookies = await context.cookies(["https://example.com"])
+        has_test_cookie = any(cookie["name"] in ["test_cookie", "test_cookie_api"] for cookie in cookies)
+        logger.info(f"Cookie set successfully: {has_test_cookie}", tag="TEST")
+        
+        # Save storage state before closing
+        await context.storage_state(path=storage_state_file)
+        logger.info(f"Storage state saved to: {storage_state_file}", tag="TEST")
+        
+        # Close the browser
+        await manager.close()
+        logger.info("First browser session closed", tag="TEST")
+        
+        # Wait a moment for clean shutdown
+        await asyncio.sleep(1.0)
+        
+        # Start a new browser with the same user data directory and storage state
+        logger.info("Starting second browser session with same user data directory", tag="TEST")
+        browser_config2 = BrowserConfig(
+            headless=True,
+            use_managed_browser=True,
+            user_data_dir=user_data_dir,
+            storage_state=storage_state_file
+        )
+        
+        manager2 = BrowserManager(browser_config=browser_config2, logger=logger)
+        await manager2.start()
+        
+        # Get a new page and check if the cookie persists
+        page2, context2 = await manager2.get_page(crawler_config)
+        await page2.goto("https://example.com", wait_until="domcontentloaded")
+        
+        # Verify cookie persisted
+        cookies2 = await context2.cookies(["https://example.com"])
+        has_test_cookie2 = any(cookie["name"] in ["test_cookie", "test_cookie_api"] for cookie in cookies2)
+        logger.info(f"Cookie persisted across sessions: {has_test_cookie2}", tag="TEST")
+        logger.info(f"Cookies found: {[c['name'] for c in cookies2]}", tag="TEST")
+        
+        # Clean up
+        await manager2.close()
+        
+        # Remove temporary directory
+        import shutil
+        shutil.rmtree(user_data_dir, ignore_errors=True)
+        logger.info(f"Removed temporary user data directory", tag="TEST")
+        
+        return has_test_cookie and has_test_cookie2
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+        except:
+            pass
+        try:
+            await manager2.close()
+        except:
+            pass
+            
+        # Clean up temporary directory
+        try:
+            import shutil
+            shutil.rmtree(user_data_dir, ignore_errors=True)
+        except:
+            pass
+            
+        return False
+
+async def test_cdp_session_management():
+    """Test session management with CDP browser - focused on session tracking."""
+    logger.info("Testing session management with CDP browser", tag="TEST")
+    
+    browser_config = BrowserConfig(
+        use_managed_browser=True,
+        headless=True
+    )
+    
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        await manager.start()
+        logger.info("Browser launched successfully", tag="TEST")
+        
+        # Test session tracking and lifecycle management
+        session1_id = "test_session_1"
+        session2_id = "test_session_2"
+        
+        # Set up first session
+        crawler_config1 = CrawlerRunConfig(session_id=session1_id)
+        page1, context1 = await manager.get_page(crawler_config1)
+        await page1.goto("https://example.com", wait_until="domcontentloaded")
+        
+        # Get page URL and title for verification
+        page1_url = page1.url
+        page1_title = await page1.title()
+        logger.info(f"Session 1 setup - URL: {page1_url}, Title: {page1_title}", tag="TEST")
+        
+        # Set up second session  
+        crawler_config2 = CrawlerRunConfig(session_id=session2_id)
+        page2, context2 = await manager.get_page(crawler_config2)
+        await page2.goto("https://httpbin.org/html", wait_until="domcontentloaded")
+        
+        page2_url = page2.url
+        page2_title = await page2.title()
+        logger.info(f"Session 2 setup - URL: {page2_url}, Title: {page2_title}", tag="TEST")
+        
+        # Verify sessions exist in manager
+        session1_exists = session1_id in manager.sessions
+        session2_exists = session2_id in manager.sessions
+        logger.info(f"Sessions in manager - S1: {session1_exists}, S2: {session2_exists}", tag="TEST")
+        
+        # Test session reuse
+        page1_again, context1_again = await manager.get_page(crawler_config1)
+        is_same_page = page1 == page1_again
+        is_same_context = context1 == context1_again
+        
+        logger.info(f"Session 1 reuse - Same page: {is_same_page}, Same context: {is_same_context}", tag="TEST")
+        
+        # Test that sessions are properly tracked with timestamps
+        session1_info = manager.sessions.get(session1_id)
+        session2_info = manager.sessions.get(session2_id)
+        
+        session1_has_timestamp = session1_info and len(session1_info) == 3
+        session2_has_timestamp = session2_info and len(session2_info) == 3
+        
+        logger.info(f"Session tracking - S1 complete: {session1_has_timestamp}, S2 complete: {session2_has_timestamp}", tag="TEST")
+        
+        # In managed browser mode, pages might be shared. Let's test what actually happens
+        pages_same_or_different = page1 == page2
+        logger.info(f"Pages same object: {pages_same_or_different}", tag="TEST")
+        
+        # Test that we can distinguish sessions by their stored info
+        session1_context, session1_page, session1_time = session1_info
+        session2_context, session2_page, session2_time = session2_info
+        
+        sessions_have_different_timestamps = session1_time != session2_time
+        logger.info(f"Sessions have different timestamps: {sessions_have_different_timestamps}", tag="TEST")
+        
+        # Test session killing
+        await manager.kill_session(session1_id)
+        logger.info(f"Killed session 1", tag="TEST")
+        
+        # Verify session was removed
+        session1_removed = session1_id not in manager.sessions
+        session2_still_exists = session2_id in manager.sessions
+        logger.info(f"After kill - S1 removed: {session1_removed}, S2 exists: {session2_still_exists}", tag="TEST")
+        
+        # Test page state after killing session
+        page1_closed = page1.is_closed()
+        logger.info(f"Page1 closed after kill: {page1_closed}", tag="TEST")
+        
+        # Clean up remaining session
+        try:
+            await manager.kill_session(session2_id)
+            logger.info("Killed session 2", tag="TEST")
+            session2_removed = session2_id not in manager.sessions
+        except Exception as e:
+            logger.info(f"Session 2 cleanup: {e}", tag="TEST")
+            session2_removed = False
+        
+        # Clean up
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        # Success criteria for managed browser sessions:
+        # 1. Sessions can be created and tracked with proper info
+        # 2. Same page/context returned for same session ID  
+        # 3. Sessions have proper timestamp tracking
+        # 4. Sessions can be killed and removed from tracking
+        # 5. Session cleanup works properly
+        success = (session1_exists and 
+                  session2_exists and 
+                  is_same_page and 
+                  session1_has_timestamp and 
+                  session2_has_timestamp and
+                  sessions_have_different_timestamps and
+                  session1_removed and 
+                  session2_removed)
+        
+        logger.info(f"Test success: {success}", tag="TEST")
+        return success
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_cdp_timing_fix_fast_startup():
+    """
+    Test that the CDP timing fix handles fast browser startup correctly.
+    This should work without any delays or retries.
+    """
+    logger.info("Testing CDP timing fix with fast startup", tag="TEST")
+    
+    browser_config = BrowserConfig(
+        use_managed_browser=True,
+        browser_mode="cdp",
+        headless=True,
+        debugging_port=9223,  # Use different port to avoid conflicts
+        verbose=True
+    )
+    
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        start_time = time.time()
+        await manager.start()
+        startup_time = time.time() - start_time
+        
+        logger.info(f"Browser started successfully in {startup_time:.2f}s", tag="TEST")
+        
+        # Test basic functionality
+        crawler_config = CrawlerRunConfig(url="https://example.com")
+        page, context = await manager.get_page(crawler_config)
+        
+        await page.goto("https://example.com", wait_until="domcontentloaded")
+        title = await page.title()
+        
+        logger.info(f"Successfully navigated to page: {title}", tag="TEST")
+        
+        await manager.close()
+        logger.success("test_cdp_timing_fix_fast_startup completed successfully", tag="TEST")
+        return True
+        
+    except Exception as e:
+        logger.error(f"test_cdp_timing_fix_fast_startup failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+
+async def test_cdp_timing_fix_delayed_browser_start():
+    """
+    Test CDP timing fix by actually delaying the browser startup process.
+    This simulates a real scenario where the browser takes time to expose CDP.
+    """
+    logger.info("Testing CDP timing fix with delayed browser startup", tag="TEST")
+    
+    browser_config = BrowserConfig(
+        use_managed_browser=True,
+        browser_mode="cdp",
+        headless=True,
+        debugging_port=9224,
+        verbose=True
+    )
+    
+    # Start the managed browser separately to control timing
+    from crawl4ai.browser_manager import ManagedBrowser
+    managed_browser = ManagedBrowser(browser_config=browser_config, logger=logger)
+    
+    try:
+        # Start browser process but it will take time for CDP to be ready
+        cdp_url = await managed_browser.start()
+        logger.info(f"Managed browser started at {cdp_url}", tag="TEST")
+        
+        # Small delay to simulate the browser needing time to fully initialize CDP
+        await asyncio.sleep(1.0)
+        
+        # Now create BrowserManager and connect - this should use the CDP verification fix
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        manager.config.cdp_url = cdp_url  # Use the CDP URL from managed browser
+        
+        start_time = time.time()
+        await manager.start()
+        startup_time = time.time() - start_time
+        
+        logger.info(f"BrowserManager connected successfully in {startup_time:.2f}s", tag="TEST")
+        
+        # Test basic functionality
+        crawler_config = CrawlerRunConfig(url="https://example.com")
+        page, context = await manager.get_page(crawler_config)
+        await page.goto("https://example.com", wait_until="domcontentloaded")
+        title = await page.title()
+        
+        logger.info(f"Successfully navigated to page: {title}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        await managed_browser.cleanup()
+        
+        logger.success("test_cdp_timing_fix_delayed_browser_start completed successfully", tag="TEST")
+        return True
+        
+    except Exception as e:
+        logger.error(f"test_cdp_timing_fix_delayed_browser_start failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+            await managed_browser.cleanup()
+        except:
+            pass
+        return False
+
+
+async def test_cdp_verification_backoff_behavior():
+    """
+    Test the exponential backoff behavior of CDP verification in isolation.
+    """
+    logger.info("Testing CDP verification exponential backoff behavior", tag="TEST")
+    
+    browser_config = BrowserConfig(
+        use_managed_browser=True,
+        debugging_port=9225,  # Use different port
+        verbose=True
+    )
+    
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        # Test with a non-existent CDP URL to trigger retries
+        fake_cdp_url = "http://localhost:19999"  # This should not exist
+        
+        start_time = time.time()
+        result = await manager._verify_cdp_ready(fake_cdp_url)
+        elapsed_time = time.time() - start_time
+        
+        # Should return False after all retries
+        assert result is False, "Expected CDP verification to fail with non-existent endpoint"
+        
+        # Should take some time due to retries and backoff
+        assert elapsed_time > 2.0, f"Expected backoff delays, but took only {elapsed_time:.2f}s"
+        
+        logger.info(f"CDP verification correctly failed after {elapsed_time:.2f}s with exponential backoff", tag="TEST")
+        logger.success("test_cdp_verification_backoff_behavior completed successfully", tag="TEST")
+        return True
+        
+    except Exception as e:
+        logger.error(f"test_cdp_verification_backoff_behavior failed: {str(e)}", tag="TEST")
+        return False
+
+
+
+async def run_tests():
+    """Run all tests sequentially."""
+    import time
+    
+    results = []
+    
+    # Original CDP strategy tests
+    logger.info("Running original CDP strategy tests", tag="SUITE")
+    # results.append(await test_cdp_launch_connect())
+    results.append(await test_cdp_with_user_data_dir())
+    results.append(await test_cdp_session_management())
+    
+    # CDP timing fix tests
+    logger.info("Running CDP timing fix tests", tag="SUITE")
+    results.append(await test_cdp_timing_fix_fast_startup())
+    results.append(await test_cdp_timing_fix_delayed_browser_start())
+    results.append(await test_cdp_verification_backoff_behavior())
+    
+    # Print summary
+    total = len(results)
+    passed = sum(results)
+    logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
+    
+    if passed == total:
+        logger.success("All tests passed!", tag="SUMMARY")
+    else:
+        logger.error(f"{total - passed} tests failed", tag="SUMMARY")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
diff --git a/tests/browser/test_combined.py b/tests/browser/test_combined.py
new file mode 100644
index 0000000..b5bce3c
--- /dev/null
+++ b/tests/browser/test_combined.py
@@ -0,0 +1,77 @@
+"""Combined test runner for all browser module tests.
+
+This script runs all the browser module tests in sequence and
+provides a comprehensive summary.
+"""
+
+import asyncio
+import os
+import sys
+import time
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+async def run_test_module(module_name, header):
+    """Run all tests in a module and return results."""
+    logger.info(f"\n{'-'*30}", tag="TEST")
+    logger.info(f"RUNNING: {header}", tag="TEST")
+    logger.info(f"{'-'*30}", tag="TEST")
+    
+    # Import the module dynamically
+    module = __import__(f"tests.browser.{module_name}", fromlist=["run_tests"])
+    
+    # Track time for performance measurement
+    start_time = time.time()
+    
+    # Run the tests
+    await module.run_tests()
+    
+    # Calculate time taken
+    time_taken = time.time() - start_time
+    logger.info(f"Time taken: {time_taken:.2f} seconds", tag="TIMING")
+    
+    return time_taken
+
+async def main():
+    """Run all test modules."""
+    logger.info("STARTING COMPREHENSIVE BROWSER MODULE TESTS", tag="MAIN")
+    
+    # List of test modules to run
+    test_modules = [
+        ("test_browser_manager", "Browser Manager Tests"),
+        ("test_playwright_strategy", "Playwright Strategy Tests"),
+        ("test_cdp_strategy", "CDP Strategy Tests"),
+        ("test_builtin_strategy", "Builtin Browser Strategy Tests"),
+        ("test_profiles", "Profile Management Tests")
+    ]
+    
+    # Run each test module
+    timings = {}
+    for module_name, header in test_modules:
+        try:
+            time_taken = await run_test_module(module_name, header)
+            timings[module_name] = time_taken
+        except Exception as e:
+            logger.error(f"Error running {module_name}: {str(e)}", tag="ERROR")
+    
+    # Print summary
+    logger.info("\n\nTEST SUMMARY:", tag="SUMMARY")
+    logger.info(f"{'-'*50}", tag="SUMMARY")
+    for module_name, header in test_modules:
+        if module_name in timings:
+            logger.info(f"{header}: {timings[module_name]:.2f} seconds", tag="SUMMARY")
+        else:
+            logger.error(f"{header}: FAILED TO RUN", tag="SUMMARY")
+    logger.info(f"{'-'*50}", tag="SUMMARY")
+    total_time = sum(timings.values())
+    logger.info(f"Total time: {total_time:.2f} seconds", tag="SUMMARY")
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/browser/test_context_leak_fix.py b/tests/browser/test_context_leak_fix.py
new file mode 100644
index 0000000..69b3313
--- /dev/null
+++ b/tests/browser/test_context_leak_fix.py
@@ -0,0 +1,358 @@
+"""
+Integration tests for the browser context memory leak fix.
+
+Tests:
+1. Signature shrink: non-context fields produce same hash
+2. Signature correctness: context-affecting fields produce different hashes
+3. Refcount lifecycle: increment on get_page, decrement on release
+4. LRU eviction: oldest idle context is evicted when over limit
+5. Eviction respects active refcounts
+6. Real browser: contexts don't leak under varying configs
+7. Real browser: batch crawl reuses same context
+8. Storage state path: temporary context is closed
+"""
+import asyncio
+import time
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_configs import ProxyConfig, GeolocationConfig
+from crawl4ai.browser_manager import BrowserManager
+
+
+# ── Unit tests (no browser needed) ──────────────────────────────────────
+
+class TestSignatureShrink:
+    """Verify the whitelist signature only considers context-affecting fields."""
+
+    def _bm(self):
+        return BrowserManager(BrowserConfig(), logger=None)
+
+    def test_non_context_fields_same_signature(self):
+        """Fields that don't affect browser context must produce identical sigs."""
+        bm = self._bm()
+        configs = [
+            CrawlerRunConfig(word_count_threshold=200),
+            CrawlerRunConfig(word_count_threshold=50),
+            CrawlerRunConfig(css_selector=".main"),
+            CrawlerRunConfig(screenshot=True),
+            CrawlerRunConfig(pdf=True, verbose=False),
+            CrawlerRunConfig(scan_full_page=True, scroll_delay=0.5),
+            CrawlerRunConfig(only_text=True),
+            CrawlerRunConfig(wait_until="networkidle", page_timeout=30000),
+            CrawlerRunConfig(capture_network_requests=True),
+            CrawlerRunConfig(exclude_external_links=True),
+        ]
+        sigs = [bm._make_config_signature(c) for c in configs]
+        assert len(set(sigs)) == 1, (
+            f"Expected all same sig, got {len(set(sigs))} unique: {sigs[:3]}"
+        )
+
+    def test_proxy_changes_signature(self):
+        bm = self._bm()
+        c1 = CrawlerRunConfig()
+        c2 = CrawlerRunConfig(proxy_config=ProxyConfig(server="http://p1:8080"))
+        c3 = CrawlerRunConfig(proxy_config=ProxyConfig(server="http://p2:8080"))
+        s1 = bm._make_config_signature(c1)
+        s2 = bm._make_config_signature(c2)
+        s3 = bm._make_config_signature(c3)
+        assert s1 != s2, "proxy vs no-proxy should differ"
+        assert s2 != s3, "different proxies should differ"
+
+    def test_locale_changes_signature(self):
+        bm = self._bm()
+        s1 = bm._make_config_signature(CrawlerRunConfig())
+        s2 = bm._make_config_signature(CrawlerRunConfig(locale="en-US"))
+        s3 = bm._make_config_signature(CrawlerRunConfig(locale="fr-FR"))
+        assert s1 != s2
+        assert s2 != s3
+
+    def test_timezone_changes_signature(self):
+        bm = self._bm()
+        s1 = bm._make_config_signature(CrawlerRunConfig())
+        s2 = bm._make_config_signature(CrawlerRunConfig(timezone_id="America/New_York"))
+        assert s1 != s2
+
+    def test_geolocation_changes_signature(self):
+        bm = self._bm()
+        s1 = bm._make_config_signature(CrawlerRunConfig())
+        s2 = bm._make_config_signature(CrawlerRunConfig(
+            geolocation=GeolocationConfig(latitude=40.7, longitude=-74.0)
+        ))
+        assert s1 != s2
+
+    def test_navigator_overrides_change_signature(self):
+        bm = self._bm()
+        base = bm._make_config_signature(CrawlerRunConfig())
+        s_nav = bm._make_config_signature(CrawlerRunConfig(override_navigator=True))
+        s_sim = bm._make_config_signature(CrawlerRunConfig(simulate_user=True))
+        s_mag = bm._make_config_signature(CrawlerRunConfig(magic=True))
+        assert base != s_nav
+        assert base != s_sim
+        assert base != s_mag
+
+    def test_signature_stability(self):
+        """Same config always produces the same hash."""
+        bm = self._bm()
+        c = CrawlerRunConfig(locale="ja-JP", override_navigator=True)
+        assert bm._make_config_signature(c) == bm._make_config_signature(c)
+
+    def test_proxy_config_with_credentials(self):
+        """ProxyConfig with username/password produces distinct stable sigs."""
+        bm = self._bm()
+        c1 = CrawlerRunConfig(proxy_config=ProxyConfig(
+            server="http://proxy:8080", username="user1", password="pass1"
+        ))
+        c2 = CrawlerRunConfig(proxy_config=ProxyConfig(
+            server="http://proxy:8080", username="user2", password="pass2"
+        ))
+        s1 = bm._make_config_signature(c1)
+        s2 = bm._make_config_signature(c2)
+        assert s1 != s2, "different credentials should differ"
+        assert s1 == bm._make_config_signature(c1), "should be stable"
+
+
+class TestLRUEviction:
+    """Verify eviction logic (no browser needed)."""
+
+    def _bm(self, max_ctx=3):
+        bm = BrowserManager(BrowserConfig(), logger=None)
+        bm._max_contexts = max_ctx
+        return bm
+
+    def test_no_eviction_under_limit(self):
+        bm = self._bm(max_ctx=5)
+        for i in range(5):
+            sig = f"sig_{i}"
+            bm.contexts_by_config[sig] = f"ctx_{i}"
+            bm._context_refcounts[sig] = 0
+            bm._context_last_used[sig] = time.monotonic()
+        assert bm._evict_lru_context_locked() is None
+
+    def test_evicts_oldest_idle(self):
+        bm = self._bm(max_ctx=3)
+        for i in range(5):
+            sig = f"sig_{i}"
+            bm.contexts_by_config[sig] = f"ctx_{i}"
+            bm._context_refcounts[sig] = 0
+            bm._context_last_used[sig] = time.monotonic()
+            time.sleep(0.002)
+
+        evicted = bm._evict_lru_context_locked()
+        assert evicted == "ctx_0", f"expected oldest ctx_0, got {evicted}"
+        assert "sig_0" not in bm.contexts_by_config
+        assert "sig_0" not in bm._context_refcounts
+        assert "sig_0" not in bm._context_last_used
+
+    def test_skips_active_contexts(self):
+        bm = self._bm(max_ctx=2)
+        # sig_0: old but active
+        bm.contexts_by_config["sig_0"] = "ctx_0"
+        bm._context_refcounts["sig_0"] = 3
+        bm._context_last_used["sig_0"] = 0  # very old
+
+        # sig_1: newer, idle
+        bm.contexts_by_config["sig_1"] = "ctx_1"
+        bm._context_refcounts["sig_1"] = 0
+        bm._context_last_used["sig_1"] = time.monotonic()
+
+        # sig_2: newest, idle
+        bm.contexts_by_config["sig_2"] = "ctx_2"
+        bm._context_refcounts["sig_2"] = 0
+        bm._context_last_used["sig_2"] = time.monotonic()
+
+        evicted = bm._evict_lru_context_locked()
+        # sig_0 is oldest but active (refcount=3) — must skip it
+        assert evicted == "ctx_1", f"expected ctx_1 (oldest idle), got {evicted}"
+        assert "sig_0" in bm.contexts_by_config, "active context must NOT be evicted"
+
+    def test_all_active_no_eviction(self):
+        bm = self._bm(max_ctx=1)
+        for i in range(3):
+            sig = f"sig_{i}"
+            bm.contexts_by_config[sig] = f"ctx_{i}"
+            bm._context_refcounts[sig] = 1  # all active
+            bm._context_last_used[sig] = time.monotonic()
+
+        evicted = bm._evict_lru_context_locked()
+        assert evicted is None, "cannot evict when all are active"
+        assert len(bm.contexts_by_config) == 3, "all contexts should remain"
+
+    def test_eviction_cleans_page_to_sig(self):
+        bm = self._bm(max_ctx=1)
+        bm.contexts_by_config["sig_old"] = "ctx_old"
+        bm._context_refcounts["sig_old"] = 0
+        bm._context_last_used["sig_old"] = 0
+
+        bm.contexts_by_config["sig_new"] = "ctx_new"
+        bm._context_refcounts["sig_new"] = 0
+        bm._context_last_used["sig_new"] = time.monotonic()
+
+        # Simulate a stale page mapping for the old context
+        mock_page = object()
+        bm._page_to_sig[mock_page] = "sig_old"
+
+        evicted = bm._evict_lru_context_locked()
+        assert evicted == "ctx_old"
+        assert mock_page not in bm._page_to_sig, "stale page mapping should be cleaned"
+
+
+# ── Integration tests (real browser) ────────────────────────────────────
+
+@pytest.fixture
+def event_loop():
+    loop = asyncio.new_event_loop()
+    yield loop
+    loop.close()
+
+
+def run(coro):
+    """Run an async function synchronously."""
+    loop = asyncio.new_event_loop()
+    try:
+        return loop.run_until_complete(coro)
+    finally:
+        loop.close()
+
+
+class TestRealBrowserContextLifecycle:
+    """Real browser tests — verify contexts aren't leaked."""
+
+    def test_varying_configs_same_context(self):
+        """Different non-context fields should reuse the same context."""
+        async def _test():
+            async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+                bm = crawler.crawler_strategy.browser_manager
+
+                # Crawl with different non-context configs
+                html = "
      Hello World with enough words to pass threshold
      "
+                for wct in [10, 50, 200]:
+                    config = CrawlerRunConfig(word_count_threshold=wct)
+                    result = await crawler.arun(f"raw:{html}", config=config)
+                    assert result.success
+
+                # Should have at most 1 context (all configs hash the same)
+                ctx_count = len(bm.contexts_by_config)
+                assert ctx_count <= 1, (
+                    f"Expected 1 context for identical browser config, got {ctx_count}"
+                )
+        run(_test())
+
+    def test_batch_crawl_reuses_context(self):
+        """Multiple URLs with same config should reuse a single context."""
+        async def _test():
+            async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+                bm = crawler.crawler_strategy.browser_manager
+
+                html1 = "
      Page one content here
      "
+                html2 = "
      Page two content here
      "
+                html3 = "
      Page three content here
      "
+
+                config = CrawlerRunConfig()
+                for h in [html1, html2, html3]:
+                    result = await crawler.arun(f"raw:{h}", config=config)
+                    assert result.success
+
+                ctx_count = len(bm.contexts_by_config)
+                assert ctx_count <= 1, f"Batch should reuse context, got {ctx_count}"
+        run(_test())
+
+    def test_refcount_drops_to_zero_after_crawl(self):
+        """After a crawl completes, the context refcount should be 0."""
+        async def _test():
+            async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+                bm = crawler.crawler_strategy.browser_manager
+                html = "
      Test content
      "
+                config = CrawlerRunConfig()
+                result = await crawler.arun(f"raw:{html}", config=config)
+                assert result.success
+
+                # All refcounts should be 0 after crawl completes
+                for sig, count in bm._context_refcounts.items():
+                    assert count == 0, (
+                        f"Refcount for {sig[:8]} should be 0 after crawl, got {count}"
+                    )
+        run(_test())
+
+    def test_page_to_sig_cleaned_after_crawl(self):
+        """After crawl, the page->sig mapping should be empty (pages released)."""
+        async def _test():
+            async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+                bm = crawler.crawler_strategy.browser_manager
+                html = "
      Test
      "
+                result = await crawler.arun(f"raw:{html}", config=CrawlerRunConfig())
+                assert result.success
+
+                assert len(bm._page_to_sig) == 0, (
+                    f"Expected empty _page_to_sig after crawl, got {len(bm._page_to_sig)} entries"
+                )
+        run(_test())
+
+    def test_concurrent_crawls_refcount_tracking(self):
+        """Concurrent crawls should all properly increment/decrement refcounts."""
+        async def _test():
+            async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+                bm = crawler.crawler_strategy.browser_manager
+                config = CrawlerRunConfig()
+
+                htmls = [
+                    f"raw:
      Concurrent page {i}
      "
+                    for i in range(5)
+                ]
+                tasks = [crawler.arun(h, config=config) for h in htmls]
+                results = await asyncio.gather(*tasks)
+                for r in results:
+                    assert r.success
+
+                # All done — refcounts should be 0
+                for sig, count in bm._context_refcounts.items():
+                    assert count == 0, (
+                        f"After concurrent crawls, refcount for {sig[:8]} = {count}"
+                    )
+                assert len(bm._page_to_sig) == 0
+        run(_test())
+
+    def test_lru_eviction_real_browser(self):
+        """Verify LRU eviction actually closes contexts when limit exceeded."""
+        async def _test():
+            async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+                bm = crawler.crawler_strategy.browser_manager
+                bm._max_contexts = 2  # Low limit to trigger eviction
+
+                html = "
      Test
      "
+
+                # Crawl with 4 different locales → 4 different context signatures
+                for locale in ["en-US", "fr-FR", "de-DE", "ja-JP"]:
+                    config = CrawlerRunConfig(locale=locale)
+                    result = await crawler.arun(f"raw:{html}", config=config)
+                    assert result.success
+
+                # Should have at most 2 contexts (limit)
+                ctx_count = len(bm.contexts_by_config)
+                assert ctx_count <= 2, (
+                    f"Expected <= 2 contexts (limit), got {ctx_count}"
+                )
+
+                # Refcounts should all be 0
+                for sig, count in bm._context_refcounts.items():
+                    assert count == 0, f"refcount {sig[:8]} = {count}"
+        run(_test())
+
+    def test_close_clears_everything(self):
+        """close() should clear all tracking dicts."""
+        async def _test():
+            crawler = AsyncWebCrawler(config=BrowserConfig(headless=True))
+            await crawler.start()
+            bm = crawler.crawler_strategy.browser_manager
+
+            html = "
      Test
      "
+            result = await crawler.arun(f"raw:{html}", config=CrawlerRunConfig())
+            assert result.success
+
+            await crawler.close()
+
+            assert len(bm.contexts_by_config) == 0
+            assert len(bm._context_refcounts) == 0
+            assert len(bm._context_last_used) == 0
+            assert len(bm._page_to_sig) == 0
+        run(_test())
diff --git a/tests/browser/test_init_script_dedup.py b/tests/browser/test_init_script_dedup.py
new file mode 100644
index 0000000..b871f13
--- /dev/null
+++ b/tests/browser/test_init_script_dedup.py
@@ -0,0 +1,399 @@
+"""
+Regression tests for init-script deduplication fix (PR #1768).
+
+Problem: context.add_init_script() was called in BOTH setup_context() and
+_crawl_web(), causing unbounded script accumulation on shared contexts
+under concurrent load — ultimately crashing the context.
+
+Fix: Flag-based guard (_crawl4ai_nav_overrider_injected,
+_crawl4ai_shadow_dom_injected) ensures each script type is injected
+at most once per context.
+
+Tests:
+1. setup_context sets flags when crawlerRunConfig has anti-bot options
+2. setup_context without crawlerRunConfig does NOT set flags
+3. _crawl_web skips injection when flags already set (no duplication)
+4. _crawl_web injects and sets flags when they're missing (fallback path)
+5. Concurrent crawls on shared context don't accumulate scripts
+6. Navigator overrides actually work after dedup (functional check)
+"""
+
+import asyncio
+import sys
+import os
+import time
+from unittest.mock import AsyncMock, MagicMock, patch
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.browser_manager import BrowserManager
+
+PASS = 0
+FAIL = 0
+
+
+def check(name, condition):
+    global PASS, FAIL
+    if condition:
+        PASS += 1
+        print(f"  PASS: {name}")
+    else:
+        FAIL += 1
+        print(f"  FAIL: {name}")
+
+
+async def test_setup_context_sets_flags():
+    """setup_context() with crawlerRunConfig should set injection flags."""
+    print("\n" + "=" * 70)
+    print("TEST: setup_context sets flags when crawlerRunConfig has anti-bot opts")
+    print("=" * 70)
+
+    bm = BrowserManager(BrowserConfig(headless=True, extra_args=['--no-sandbox']))
+    await bm.start()
+
+    try:
+        # Create context with navigator overrides
+        config_nav = CrawlerRunConfig(override_navigator=True)
+        ctx_nav = await bm.create_browser_context(config_nav)
+        await bm.setup_context(ctx_nav, config_nav)
+
+        check("nav_overrider flag set after setup_context(override_navigator=True)",
+              getattr(ctx_nav, '_crawl4ai_nav_overrider_injected', False) is True)
+        check("shadow_dom flag NOT set (not requested)",
+              getattr(ctx_nav, '_crawl4ai_shadow_dom_injected', False) is False)
+
+        await ctx_nav.close()
+
+        # Create context with magic mode
+        config_magic = CrawlerRunConfig(magic=True)
+        ctx_magic = await bm.create_browser_context(config_magic)
+        await bm.setup_context(ctx_magic, config_magic)
+
+        check("nav_overrider flag set after setup_context(magic=True)",
+              getattr(ctx_magic, '_crawl4ai_nav_overrider_injected', False) is True)
+
+        await ctx_magic.close()
+
+        # Create context with simulate_user
+        config_sim = CrawlerRunConfig(simulate_user=True)
+        ctx_sim = await bm.create_browser_context(config_sim)
+        await bm.setup_context(ctx_sim, config_sim)
+
+        check("nav_overrider flag set after setup_context(simulate_user=True)",
+              getattr(ctx_sim, '_crawl4ai_nav_overrider_injected', False) is True)
+
+        await ctx_sim.close()
+
+        # Create context with flatten_shadow_dom
+        config_shadow = CrawlerRunConfig(flatten_shadow_dom=True)
+        ctx_shadow = await bm.create_browser_context(config_shadow)
+        await bm.setup_context(ctx_shadow, config_shadow)
+
+        check("shadow_dom flag set after setup_context(flatten_shadow_dom=True)",
+              getattr(ctx_shadow, '_crawl4ai_shadow_dom_injected', False) is True)
+        check("nav_overrider flag NOT set (not requested)",
+              getattr(ctx_shadow, '_crawl4ai_nav_overrider_injected', False) is False)
+
+        await ctx_shadow.close()
+
+        # Create context with both
+        config_both = CrawlerRunConfig(magic=True, flatten_shadow_dom=True)
+        ctx_both = await bm.create_browser_context(config_both)
+        await bm.setup_context(ctx_both, config_both)
+
+        check("both flags set when both features requested",
+              getattr(ctx_both, '_crawl4ai_nav_overrider_injected', False) is True
+              and getattr(ctx_both, '_crawl4ai_shadow_dom_injected', False) is True)
+
+        await ctx_both.close()
+
+    finally:
+        await bm.close()
+
+
+async def test_setup_context_no_config_no_flags():
+    """setup_context() without crawlerRunConfig should NOT set flags."""
+    print("\n" + "=" * 70)
+    print("TEST: setup_context without crawlerRunConfig does NOT set flags")
+    print("=" * 70)
+
+    bm = BrowserManager(BrowserConfig(headless=True, extra_args=['--no-sandbox']))
+    await bm.start()
+
+    try:
+        ctx = await bm.create_browser_context()
+        await bm.setup_context(ctx)  # No crawlerRunConfig
+
+        check("nav_overrider flag NOT set (no crawlerRunConfig)",
+              getattr(ctx, '_crawl4ai_nav_overrider_injected', False) is False)
+        check("shadow_dom flag NOT set (no crawlerRunConfig)",
+              getattr(ctx, '_crawl4ai_shadow_dom_injected', False) is False)
+
+        await ctx.close()
+
+    finally:
+        await bm.close()
+
+
+async def test_no_duplication_standard_path():
+    """
+    Standard path: setup_context() injects scripts + sets flags,
+    then _crawl_web() should skip re-injection.
+
+    We verify by counting add_init_script calls on the context.
+    """
+    print("\n" + "=" * 70)
+    print("TEST: No duplication on standard path (setup_context + _crawl_web)")
+    print("=" * 70)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
+        config = CrawlerRunConfig(magic=True, flatten_shadow_dom=True)
+        html = "
      Test content for dedup check
      "
+
+        result = await crawler.arun(f"raw:{html}", config=config)
+        check("crawl succeeded", result.success)
+
+        # Get the context through the browser manager
+        bm = crawler.crawler_strategy.browser_manager
+        for sig, ctx in bm.contexts_by_config.items():
+            check("nav_overrider flag is set on context",
+                  getattr(ctx, '_crawl4ai_nav_overrider_injected', False) is True)
+            check("shadow_dom flag is set on context",
+                  getattr(ctx, '_crawl4ai_shadow_dom_injected', False) is True)
+
+
+async def test_fallback_path_injects_once():
+    """
+    Fallback path: manually create a context without crawlerRunConfig
+    (simulating managed/persistent/CDP path), then verify _crawl_web()
+    injects scripts exactly once and sets the flags.
+    """
+    print("\n" + "=" * 70)
+    print("TEST: Fallback path injects once and sets flags")
+    print("=" * 70)
+
+    bm = BrowserManager(BrowserConfig(headless=True, extra_args=['--no-sandbox']))
+    await bm.start()
+
+    try:
+        # Create context WITHOUT crawlerRunConfig (simulates persistent/CDP path)
+        ctx = await bm.create_browser_context()
+        await bm.setup_context(ctx)  # No crawlerRunConfig — no flags set
+
+        check("flags NOT set before _crawl_web",
+              not getattr(ctx, '_crawl4ai_nav_overrider_injected', False)
+              and not getattr(ctx, '_crawl4ai_shadow_dom_injected', False))
+
+        # Track add_init_script calls
+        original_add_init_script = ctx.add_init_script
+        call_count = 0
+
+        async def counting_add_init_script(*args, **kwargs):
+            nonlocal call_count
+            call_count += 1
+            return await original_add_init_script(*args, **kwargs)
+
+        ctx.add_init_script = counting_add_init_script
+
+        # Create a page and simulate what _crawl_web does
+        page = await ctx.new_page()
+
+        config = CrawlerRunConfig(magic=True, flatten_shadow_dom=True)
+
+        # First "crawl" — should inject both scripts
+        from crawl4ai.js_snippet import load_js_script
+
+        if config.override_navigator or config.simulate_user or config.magic:
+            if not getattr(ctx, '_crawl4ai_nav_overrider_injected', False):
+                await ctx.add_init_script(load_js_script("navigator_overrider"))
+                ctx._crawl4ai_nav_overrider_injected = True
+
+        if config.flatten_shadow_dom:
+            if not getattr(ctx, '_crawl4ai_shadow_dom_injected', False):
+                await ctx.add_init_script("""
+                    const _origAttachShadow = Element.prototype.attachShadow;
+                    Element.prototype.attachShadow = function(init) {
+                        return _origAttachShadow.call(this, {...init, mode: 'open'});
+                    };
+                """)
+                ctx._crawl4ai_shadow_dom_injected = True
+
+        check("first pass: 2 add_init_script calls (nav + shadow)", call_count == 2)
+
+        # Second "crawl" — should skip both
+        call_count = 0
+
+        if config.override_navigator or config.simulate_user or config.magic:
+            if not getattr(ctx, '_crawl4ai_nav_overrider_injected', False):
+                await ctx.add_init_script(load_js_script("navigator_overrider"))
+                ctx._crawl4ai_nav_overrider_injected = True
+
+        if config.flatten_shadow_dom:
+            if not getattr(ctx, '_crawl4ai_shadow_dom_injected', False):
+                await ctx.add_init_script("""...""")
+                ctx._crawl4ai_shadow_dom_injected = True
+
+        check("second pass: 0 add_init_script calls (flags set)", call_count == 0)
+
+        await page.close()
+        await ctx.close()
+
+    finally:
+        await bm.close()
+
+
+async def test_concurrent_crawls_no_accumulation():
+    """
+    The core bug: N concurrent crawls on a shared context should NOT
+    cause N * add_init_script() calls. With the fix, only 1 call
+    should happen (the first crawl), and the rest should skip.
+    """
+    print("\n" + "=" * 70)
+    print("TEST: Concurrent crawls don't accumulate init scripts")
+    print("=" * 70)
+
+    CONCURRENCY = 10
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
+        config = CrawlerRunConfig(magic=True, flatten_shadow_dom=True)
+
+        # Run N concurrent crawls with identical config (same context)
+        htmls = [
+            f"raw:
      Concurrent page {i} with enough words
      "
+            for i in range(CONCURRENCY)
+        ]
+
+        tasks = [crawler.arun(h, config=config) for h in htmls]
+        results = await asyncio.gather(*tasks)
+
+        success_count = sum(1 for r in results if r.success)
+        check(f"all {CONCURRENCY} concurrent crawls succeeded", success_count == CONCURRENCY)
+
+        # Check that the shared context has the flags set (proving injection happened)
+        bm = crawler.crawler_strategy.browser_manager
+        for sig, ctx in bm.contexts_by_config.items():
+            check("shared context has nav_overrider flag",
+                  getattr(ctx, '_crawl4ai_nav_overrider_injected', False) is True)
+            check("shared context has shadow_dom flag",
+                  getattr(ctx, '_crawl4ai_shadow_dom_injected', False) is True)
+
+        # Verify no context crash — all refcounts should be 0
+        for sig, count in bm._context_refcounts.items():
+            check(f"refcount for {sig[:12]}... is 0 after all crawls", count == 0)
+
+
+async def test_navigator_overrides_functional():
+    """
+    Functional check: after dedup fix, navigator overrides still work.
+    The webdriver property should be undefined (not true).
+    """
+    print("\n" + "=" * 70)
+    print("TEST: Navigator overrides still functional after dedup")
+    print("=" * 70)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
+        config = CrawlerRunConfig(override_navigator=True)
+        html = "
      Navigator test
      "
+
+        result = await crawler.arun(f"raw:{html}", config=config)
+        check("crawl succeeded", result.success)
+
+        # Run a second crawl (same context) to verify scripts still work
+        result2 = await crawler.arun(f"raw:{html}", config=config)
+        check("second crawl on same context succeeded", result2.success)
+
+
+async def test_concurrent_different_configs():
+    """
+    Concurrent crawls with DIFFERENT configs: one with magic, one without.
+    Each config gets its own context. Verify no cross-contamination and
+    no crashes.
+    """
+    print("\n" + "=" * 70)
+    print("TEST: Concurrent crawls with different configs")
+    print("=" * 70)
+
+    CRAWLS_PER_CONFIG = 5
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
+        configs = [
+            CrawlerRunConfig(magic=True),
+            CrawlerRunConfig(magic=False),
+            CrawlerRunConfig(magic=True, flatten_shadow_dom=True),
+        ]
+
+        tasks = []
+        for i in range(CRAWLS_PER_CONFIG):
+            for j, config in enumerate(configs):
+                html = f"raw:
      Config {j} crawl {i}
      "
+                tasks.append(crawler.arun(html, config=config))
+
+        results = await asyncio.gather(*tasks)
+        total = CRAWLS_PER_CONFIG * len(configs)
+        success_count = sum(1 for r in results if r.success)
+        check(f"all {total} mixed-config crawls succeeded", success_count == total)
+
+        bm = crawler.crawler_strategy.browser_manager
+
+        # All refcounts should be 0
+        for sig, count in bm._context_refcounts.items():
+            check(f"refcount 0 for sig {sig[:12]}...", count == 0)
+
+
+async def test_shadow_dom_flattening_functional():
+    """
+    Functional check: shadow DOM flattening works after dedup.
+    The attachShadow override should force shadow roots to open mode.
+    """
+    print("\n" + "=" * 70)
+    print("TEST: Shadow DOM flattening still functional after dedup")
+    print("=" * 70)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
+        config = CrawlerRunConfig(flatten_shadow_dom=True)
+
+        # HTML with a shadow DOM component
+        html = """
+            
      
+            
+        """
+
+        result = await crawler.arun(f"raw:{html}", config=config)
+        check("shadow DOM crawl succeeded", result.success)
+
+        # Second crawl on same context
+        result2 = await crawler.arun(f"raw:{html}", config=config)
+        check("second shadow DOM crawl succeeded", result2.success)
+
+
+async def main():
+    print("=" * 70)
+    print("Init Script Deduplication Tests (PR #1768 fix)")
+    print("=" * 70)
+
+    await test_setup_context_sets_flags()
+    await test_setup_context_no_config_no_flags()
+    await test_no_duplication_standard_path()
+    await test_fallback_path_injects_once()
+    await test_concurrent_crawls_no_accumulation()
+    await test_navigator_overrides_functional()
+    await test_concurrent_different_configs()
+    await test_shadow_dom_flattening_functional()
+
+    print("\n" + "=" * 70)
+    if FAIL == 0:
+        print(f"ALL {PASS} CHECKS PASSED")
+    else:
+        print(f"FAILED: {FAIL} checks failed, {PASS} passed")
+    print("=" * 70)
+
+    sys.exit(1 if FAIL > 0 else 0)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/browser/test_launch_standalone.py b/tests/browser/test_launch_standalone.py
new file mode 100644
index 0000000..d60b12f
--- /dev/null
+++ b/tests/browser/test_launch_standalone.py
@@ -0,0 +1,17 @@
+from crawl4ai.browser_profiler import BrowserProfiler
+import asyncio
+
+
+if __name__ == "__main__":
+    # Test launching a standalone browser
+    async def test_standalone_browser():
+        profiler = BrowserProfiler()
+        cdp_url = await profiler.launch_standalone_browser(
+            browser_type="chromium",
+            user_data_dir="~/.crawl4ai/browser_profile/test-browser-data",
+            debugging_port=9222,
+            headless=False
+        )
+        print(f"CDP URL: {cdp_url}")
+
+    asyncio.run(test_standalone_browser())
\ No newline at end of file
diff --git a/tests/browser/test_page_reuse_race_condition.py b/tests/browser/test_page_reuse_race_condition.py
new file mode 100644
index 0000000..10b14f6
--- /dev/null
+++ b/tests/browser/test_page_reuse_race_condition.py
@@ -0,0 +1,606 @@
+"""
+Real integration tests for page reuse race condition fix.
+
+Tests that when create_isolated_context=False:
+1. Single crawls still work correctly
+2. Concurrent crawls don't cause race conditions
+3. Pages are properly tracked and released
+4. Page reuse works when pages become available
+
+These are REAL tests - no mocking, actual browser operations.
+"""
+
+import asyncio
+import os
+import sys
+import time
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+
+async def test_single_crawl_still_works():
+    """
+    Test 1: Basic single crawl functionality still works with create_isolated_context=False.
+    This ensures we haven't broken existing functionality.
+    """
+    print("\n" + "="*70)
+    print("TEST 1: Single crawl with create_isolated_context=False")
+    print("="*70)
+
+    browser_config = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        create_isolated_context=False,
+    )
+
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun("https://example.com")
+
+            assert result.success, f"Crawl failed: {result.error_message}"
+            assert result.html, "No HTML content returned"
+            assert "Example Domain" in result.html, "Expected content not found"
+
+            print(f"  Status: {result.status_code}")
+            print(f"  HTML length: {len(result.html)} chars")
+            print("  PASSED: Single crawl works correctly")
+            return True
+
+    except Exception as e:
+        print(f"  FAILED: {str(e)}")
+        return False
+
+
+async def test_sequential_crawls_work():
+    """
+    Test 2: Sequential crawls reuse the same page (when released).
+    This tests that page tracking and release works correctly.
+    """
+    print("\n" + "="*70)
+    print("TEST 2: Sequential crawls with page reuse")
+    print("="*70)
+
+    browser_config = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        create_isolated_context=False,
+    )
+
+    urls = [
+        "https://example.com",
+        "https://httpbin.org/html",
+        "https://example.org",
+    ]
+
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            results = []
+            for url in urls:
+                result = await crawler.arun(url)
+                results.append(result)
+                print(f"  Crawled {url}: success={result.success}, status={result.status_code}")
+
+            # All should succeed
+            for i, result in enumerate(results):
+                assert result.success, f"Crawl {i+1} failed: {result.error_message}"
+
+            print("  PASSED: Sequential crawls work correctly")
+            return True
+
+    except Exception as e:
+        print(f"  FAILED: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+async def test_concurrent_crawls_no_race_condition():
+    """
+    Test 3: Multiple concurrent crawls don't cause race conditions.
+    This is the main bug we're fixing - concurrent crawls should each get their own page.
+    """
+    print("\n" + "="*70)
+    print("TEST 3: Concurrent crawls with create_isolated_context=False")
+    print("="*70)
+
+    browser_config = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        create_isolated_context=False,
+    )
+
+    # Use different URLs to ensure they can't accidentally succeed by being on the same page
+    urls = [
+        "https://example.com",
+        "https://httpbin.org/html",
+        "https://example.org",
+        "https://httpbin.org/get",
+        "https://www.iana.org/domains/reserved",
+    ]
+
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            print(f"  Launching {len(urls)} concurrent crawls...")
+            start_time = time.time()
+
+            # Launch all crawls concurrently
+            tasks = [crawler.arun(url) for url in urls]
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+
+            elapsed = time.time() - start_time
+            print(f"  Completed in {elapsed:.2f}s")
+
+            # Check results
+            success_count = 0
+            for i, (url, result) in enumerate(zip(urls, results)):
+                if isinstance(result, Exception):
+                    print(f"  [{i+1}] {url}: EXCEPTION - {result}")
+                elif result.success:
+                    success_count += 1
+                    print(f"  [{i+1}] {url}: OK (status={result.status_code})")
+                else:
+                    print(f"  [{i+1}] {url}: FAILED - {result.error_message}")
+
+            # All should succeed
+            assert success_count == len(urls), f"Only {success_count}/{len(urls)} succeeded"
+
+            print(f"  PASSED: All {len(urls)} concurrent crawls succeeded without race conditions")
+            return True
+
+    except Exception as e:
+        print(f"  FAILED: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+async def test_high_concurrency_stress():
+    """
+    Test 4: High concurrency stress test - many concurrent crawls.
+    This stresses the page tracking system to ensure it handles many concurrent operations.
+    """
+    print("\n" + "="*70)
+    print("TEST 4: High concurrency stress test (10 concurrent crawls)")
+    print("="*70)
+
+    browser_config = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        create_isolated_context=False,
+    )
+
+    # Generate multiple unique URLs
+    base_urls = [
+        "https://example.com",
+        "https://httpbin.org/html",
+        "https://example.org",
+        "https://httpbin.org/get",
+        "https://www.iana.org/domains/reserved",
+    ]
+
+    # Create 10 URLs by adding query params
+    urls = []
+    for i in range(10):
+        url = f"{base_urls[i % len(base_urls)]}?test={i}&t={int(time.time())}"
+        urls.append(url)
+
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            print(f"  Launching {len(urls)} concurrent crawls...")
+            start_time = time.time()
+
+            # Launch all crawls concurrently
+            tasks = [crawler.arun(url) for url in urls]
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+
+            elapsed = time.time() - start_time
+            print(f"  Completed in {elapsed:.2f}s")
+
+            # Count results
+            success_count = 0
+            error_count = 0
+            exception_count = 0
+
+            for url, result in zip(urls, results):
+                if isinstance(result, Exception):
+                    exception_count += 1
+                elif result.success:
+                    success_count += 1
+                else:
+                    error_count += 1
+
+            print(f"  Results: {success_count} success, {error_count} errors, {exception_count} exceptions")
+
+            # At least 80% should succeed (allowing for some network issues)
+            min_success = int(len(urls) * 0.8)
+            assert success_count >= min_success, f"Only {success_count}/{len(urls)} succeeded (min: {min_success})"
+
+            print(f"  PASSED: High concurrency test ({success_count}/{len(urls)} succeeded)")
+            return True
+
+    except Exception as e:
+        print(f"  FAILED: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+async def test_page_tracking_internal_state():
+    """
+    Test 5: Verify internal page tracking state is correct.
+    This directly tests the global page tracking mechanism.
+    """
+    print("\n" + "="*70)
+    print("TEST 5: Internal page tracking state verification")
+    print("="*70)
+
+    browser_config = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        create_isolated_context=False,
+    )
+
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            browser_manager = crawler.crawler_strategy.browser_manager
+
+            # Check endpoint key is set
+            endpoint_key = browser_manager._browser_endpoint_key
+            print(f"  Browser endpoint key: {endpoint_key}")
+            assert endpoint_key, "Endpoint key should be set"
+
+            # Initially, no pages should be in use
+            initial_in_use = len(browser_manager._get_pages_in_use())
+            print(f"  Initial pages in use: {initial_in_use}")
+
+            # Do a crawl
+            result = await crawler.arun("https://example.com")
+            assert result.success, f"Crawl failed: {result.error_message}"
+
+            # After crawl completes, page should be released
+            after_crawl_in_use = len(browser_manager._get_pages_in_use())
+            print(f"  Pages in use after crawl: {after_crawl_in_use}")
+
+            # The page should have been released (or kept as the last page)
+            # Either way, tracking should be consistent
+
+            # Do another crawl - should work fine
+            result2 = await crawler.arun("https://example.org")
+            assert result2.success, f"Second crawl failed: {result2.error_message}"
+
+            final_in_use = len(browser_manager._get_pages_in_use())
+            print(f"  Pages in use after second crawl: {final_in_use}")
+
+            print("  PASSED: Page tracking state is consistent")
+            return True
+
+    except Exception as e:
+        print(f"  FAILED: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+async def test_mixed_sequential_and_concurrent():
+    """
+    Test 6: Mixed sequential and concurrent crawls.
+    Tests realistic usage pattern where some crawls are sequential and some concurrent.
+    """
+    print("\n" + "="*70)
+    print("TEST 6: Mixed sequential and concurrent crawls")
+    print("="*70)
+
+    browser_config = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        create_isolated_context=False,
+    )
+
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            # Sequential crawl 1
+            print("  Phase 1: Sequential crawl")
+            result1 = await crawler.arun("https://example.com")
+            assert result1.success, f"Sequential crawl 1 failed"
+            print(f"    Crawl 1: OK")
+
+            # Concurrent crawls
+            print("  Phase 2: Concurrent crawls (3 URLs)")
+            concurrent_urls = [
+                "https://httpbin.org/html",
+                "https://example.org",
+                "https://httpbin.org/get",
+            ]
+            tasks = [crawler.arun(url) for url in concurrent_urls]
+            concurrent_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+            for i, result in enumerate(concurrent_results):
+                if isinstance(result, Exception):
+                    print(f"    Concurrent {i+1}: EXCEPTION - {result}")
+                else:
+                    assert result.success, f"Concurrent crawl {i+1} failed"
+                    print(f"    Concurrent {i+1}: OK")
+
+            # Sequential crawl 2
+            print("  Phase 3: Sequential crawl")
+            result2 = await crawler.arun("https://www.iana.org/domains/reserved")
+            assert result2.success, f"Sequential crawl 2 failed"
+            print(f"    Crawl 2: OK")
+
+            # Another batch of concurrent
+            print("  Phase 4: More concurrent crawls (2 URLs)")
+            tasks2 = [
+                crawler.arun("https://example.com?test=1"),
+                crawler.arun("https://example.org?test=2"),
+            ]
+            results2 = await asyncio.gather(*tasks2, return_exceptions=True)
+            for i, result in enumerate(results2):
+                if isinstance(result, Exception):
+                    print(f"    Concurrent {i+1}: EXCEPTION - {result}")
+                else:
+                    assert result.success, f"Batch 2 crawl {i+1} failed"
+                    print(f"    Concurrent {i+1}: OK")
+
+            print("  PASSED: Mixed sequential and concurrent crawls work correctly")
+            return True
+
+    except Exception as e:
+        print(f"  FAILED: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+async def test_compare_isolated_vs_shared_context():
+    """
+    Test 7: Compare behavior between isolated and shared context modes.
+    Both should work for concurrent crawls now.
+    """
+    print("\n" + "="*70)
+    print("TEST 7: Compare isolated vs shared context modes")
+    print("="*70)
+
+    urls = [
+        "https://example.com",
+        "https://httpbin.org/html",
+        "https://example.org",
+    ]
+
+    # Test with create_isolated_context=True
+    print("  Testing with create_isolated_context=True:")
+    browser_config_isolated = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+
+    try:
+        async with AsyncWebCrawler(config=browser_config_isolated) as crawler:
+            tasks = [crawler.arun(url) for url in urls]
+            results_isolated = await asyncio.gather(*tasks, return_exceptions=True)
+
+            isolated_success = sum(1 for r in results_isolated if not isinstance(r, Exception) and r.success)
+            print(f"    Isolated context: {isolated_success}/{len(urls)} succeeded")
+    except Exception as e:
+        print(f"    Isolated context: FAILED - {e}")
+        isolated_success = 0
+
+    # Test with create_isolated_context=False
+    print("  Testing with create_isolated_context=False:")
+    browser_config_shared = BrowserConfig(
+        headless=True,
+        use_managed_browser=True,
+        create_isolated_context=False,
+    )
+
+    try:
+        async with AsyncWebCrawler(config=browser_config_shared) as crawler:
+            tasks = [crawler.arun(url) for url in urls]
+            results_shared = await asyncio.gather(*tasks, return_exceptions=True)
+
+            shared_success = sum(1 for r in results_shared if not isinstance(r, Exception) and r.success)
+            print(f"    Shared context: {shared_success}/{len(urls)} succeeded")
+    except Exception as e:
+        print(f"    Shared context: FAILED - {e}")
+        shared_success = 0
+
+    # Both modes should work
+    assert isolated_success == len(urls), f"Isolated context: only {isolated_success}/{len(urls)} succeeded"
+    assert shared_success == len(urls), f"Shared context: only {shared_success}/{len(urls)} succeeded"
+
+    print("  PASSED: Both context modes work correctly for concurrent crawls")
+    return True
+
+
+async def test_multiple_crawlers_same_cdp():
+    """
+    Test 8: Multiple AsyncWebCrawler instances connecting to the same CDP endpoint.
+
+    This tests the realistic scenario where:
+    1. A browser is started externally (or by a managed browser)
+    2. Multiple crawler instances connect to it via CDP URL
+    3. All use create_isolated_context=False to share cookies/session
+    4. Each should get its own page to avoid race conditions
+    """
+    print("\n" + "="*70)
+    print("TEST 8: Multiple crawlers connecting to same CDP endpoint")
+    print("="*70)
+
+    import subprocess
+    import tempfile
+
+    # Start a browser manually using subprocess
+    port = 9444
+    temp_dir = tempfile.mkdtemp(prefix="browser-test-")
+
+    browser_process = None
+    try:
+        # Start chromium with remote debugging - use Playwright's bundled chromium
+        import os
+        playwright_path = os.path.expanduser("~/.cache/ms-playwright/chromium-1200/chrome-linux64/chrome")
+        if not os.path.exists(playwright_path):
+            # Fallback - try to find it
+            for path in [
+                "/usr/bin/chromium",
+                "/usr/bin/chromium-browser",
+                "/usr/bin/google-chrome",
+            ]:
+                if os.path.exists(path):
+                    playwright_path = path
+                    break
+        chrome_path = playwright_path
+
+        cmd = [
+            chrome_path,
+            f"--remote-debugging-port={port}",
+            f"--user-data-dir={temp_dir}",
+            "--headless=new",
+            "--no-sandbox",
+            "--disable-gpu",
+            "--disable-dev-shm-usage",
+        ]
+
+        browser_process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        await asyncio.sleep(2)  # Wait for browser to start
+
+        cdp_url = f"http://localhost:{port}"
+        print(f"  Started browser at {cdp_url}")
+
+        # Both crawlers connect via CDP URL
+        browser_config1 = BrowserConfig(
+            headless=True,
+            cdp_url=cdp_url,
+            create_isolated_context=False,
+        )
+        browser_config2 = BrowserConfig(
+            headless=True,
+            cdp_url=cdp_url,
+            create_isolated_context=False,
+        )
+
+        urls_crawler1 = [
+            "https://example.com?crawler=1",
+            "https://example.org?crawler=1",
+        ]
+        urls_crawler2 = [
+            "https://httpbin.org/html?crawler=2",
+            "https://httpbin.org/get?crawler=2",
+        ]
+
+        async with AsyncWebCrawler(config=browser_config1) as crawler1:
+            async with AsyncWebCrawler(config=browser_config2) as crawler2:
+                bm1 = crawler1.crawler_strategy.browser_manager
+                bm2 = crawler2.crawler_strategy.browser_manager
+
+                print(f"  Crawler 1 endpoint key: {bm1._browser_endpoint_key}")
+                print(f"  Crawler 2 endpoint key: {bm2._browser_endpoint_key}")
+                print(f"  Keys match: {bm1._browser_endpoint_key == bm2._browser_endpoint_key}")
+
+                # Launch concurrent crawls from BOTH crawlers simultaneously
+                print(f"  Launching {len(urls_crawler1) + len(urls_crawler2)} concurrent crawls...")
+
+                tasks1 = [crawler1.arun(url) for url in urls_crawler1]
+                tasks2 = [crawler2.arun(url) for url in urls_crawler2]
+
+                all_results = await asyncio.gather(
+                    *tasks1, *tasks2,
+                    return_exceptions=True
+                )
+
+                # Check results
+                success_count = 0
+                for i, result in enumerate(all_results):
+                    crawler_id = 1 if i < len(urls_crawler1) else 2
+                    url_idx = i if i < len(urls_crawler1) else i - len(urls_crawler1)
+
+                    if isinstance(result, Exception):
+                        print(f"    Crawler {crawler_id}, URL {url_idx+1}: EXCEPTION - {result}")
+                    elif result.success:
+                        success_count += 1
+                        print(f"    Crawler {crawler_id}, URL {url_idx+1}: OK")
+                    else:
+                        print(f"    Crawler {crawler_id}, URL {url_idx+1}: FAILED - {result.error_message}")
+
+                total = len(urls_crawler1) + len(urls_crawler2)
+                assert success_count == total, f"Only {success_count}/{total} succeeded"
+
+                print(f"  PASSED: All {total} concurrent crawls from 2 crawlers succeeded")
+                return True
+
+    except Exception as e:
+        print(f"  FAILED: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+    finally:
+        # Clean up browser process
+        if browser_process:
+            browser_process.terminate()
+            try:
+                browser_process.wait(timeout=5)
+            except:
+                browser_process.kill()
+        # Clean up temp dir
+        import shutil
+        try:
+            shutil.rmtree(temp_dir)
+        except:
+            pass
+
+
+async def run_all_tests():
+    """Run all tests and report results."""
+    print("\n" + "#"*70)
+    print("# PAGE REUSE RACE CONDITION FIX - INTEGRATION TESTS")
+    print("#"*70)
+
+    tests = [
+        ("Single crawl works", test_single_crawl_still_works),
+        ("Sequential crawls work", test_sequential_crawls_work),
+        ("Concurrent crawls no race", test_concurrent_crawls_no_race_condition),
+        ("High concurrency stress", test_high_concurrency_stress),
+        ("Page tracking state", test_page_tracking_internal_state),
+        ("Mixed sequential/concurrent", test_mixed_sequential_and_concurrent),
+        ("Isolated vs shared context", test_compare_isolated_vs_shared_context),
+    ]
+
+    results = []
+    for name, test_func in tests:
+        try:
+            passed = await test_func()
+            results.append((name, passed))
+        except Exception as e:
+            print(f"  EXCEPTION in {name}: {e}")
+            results.append((name, False))
+
+    # Summary
+    print("\n" + "="*70)
+    print("TEST SUMMARY")
+    print("="*70)
+
+    passed = sum(1 for _, p in results if p)
+    total = len(results)
+
+    for name, p in results:
+        status = "PASS" if p else "FAIL"
+        print(f"  [{status}] {name}")
+
+    print("-"*70)
+    print(f"  Total: {passed}/{total} tests passed")
+
+    if passed == total:
+        print("\n  ALL TESTS PASSED!")
+        return 0
+    else:
+        print(f"\n  {total - passed} TESTS FAILED!")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(run_all_tests())
+    sys.exit(exit_code)
diff --git a/tests/browser/test_parallel_crawling.py b/tests/browser/test_parallel_crawling.py
new file mode 100644
index 0000000..9e72f06
--- /dev/null
+++ b/tests/browser/test_parallel_crawling.py
@@ -0,0 +1,902 @@
+"""
+Test examples for parallel crawling with the browser module.
+
+These examples demonstrate the functionality of parallel page creation
+and serve as functional tests for multi-page crawling performance.
+"""
+
+import asyncio
+import os
+import sys
+import time
+from typing import List
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai.browser import BrowserManager
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+async def test_get_pages_basic():
+    """Test basic functionality of get_pages method."""
+    logger.info("Testing basic get_pages functionality", tag="TEST")
+    
+    browser_config = BrowserConfig(headless=True)
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        await manager.start()
+        
+        # Request 3 pages
+        crawler_config = CrawlerRunConfig()
+        pages = await manager.get_pages(crawler_config, count=3)
+        
+        # Verify we got the correct number of pages
+        assert len(pages) == 3, f"Expected 3 pages, got {len(pages)}"
+        
+        # Verify each page is valid
+        for i, (page, context) in enumerate(pages):
+            await page.goto("https://example.com")
+            title = await page.title()
+            logger.info(f"Page {i+1} title: {title}", tag="TEST")
+            assert title, f"Page {i+1} has no title"
+        
+        await manager.close()
+        logger.success("Basic get_pages test completed successfully", tag="TEST")
+        return True
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_parallel_approaches_comparison():
+    """Compare two parallel crawling approaches:
+    1. Create a page for each URL on-demand (get_page + gather)
+    2. Get all pages upfront with get_pages, then use them (get_pages + gather)
+    """
+    logger.info("Comparing different parallel crawling approaches", tag="TEST")
+    
+    urls = [
+        "https://example.com/page1",
+        "https://crawl4ai.com",
+        "https://kidocode.com",
+        "https://bbc.com",
+        # "https://example.com/page1",
+        # "https://example.com/page2",
+        # "https://example.com/page3",
+        # "https://example.com/page4",
+    ]
+    
+    browser_config = BrowserConfig(headless=False)
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        await manager.start()
+        
+        # Approach 1: Create a page for each URL on-demand and run in parallel
+        logger.info("Testing approach 1: get_page for each URL + gather", tag="TEST")
+        start_time = time.time()
+        
+        async def fetch_title_approach1(url):
+            """Create a new page for each URL, go to the URL, and get title"""
+            crawler_config = CrawlerRunConfig(url=url)
+            page, context = await manager.get_page(crawler_config)
+            try:
+                await page.goto(url)
+                title = await page.title()
+                return title
+            finally:
+                await page.close()
+        
+        # Run fetch_title_approach1 for each URL in parallel
+        tasks = [fetch_title_approach1(url) for url in urls]
+        approach1_results = await asyncio.gather(*tasks)
+        
+        approach1_time = time.time() - start_time
+        logger.info(f"Approach 1 time (get_page + gather): {approach1_time:.2f}s", tag="TEST")
+        
+        # Approach 2: Get all pages upfront with get_pages, then use them in parallel
+        logger.info("Testing approach 2: get_pages upfront + gather", tag="TEST")
+        start_time = time.time()
+        
+        # Get all pages upfront
+        crawler_config = CrawlerRunConfig()
+        pages = await manager.get_pages(crawler_config, count=len(urls))
+        
+        async def fetch_title_approach2(page_ctx, url):
+            """Use a pre-created page to go to URL and get title"""
+            page, _ = page_ctx
+            try:
+                await page.goto(url)
+                title = await page.title()
+                return title
+            finally:
+                await page.close()
+        
+        # Use the pre-created pages to fetch titles in parallel
+        tasks = [fetch_title_approach2(page_ctx, url) for page_ctx, url in zip(pages, urls)]
+        approach2_results = await asyncio.gather(*tasks)
+        
+        approach2_time = time.time() - start_time
+        logger.info(f"Approach 2 time (get_pages + gather): {approach2_time:.2f}s", tag="TEST")
+        
+        # Compare results and performance
+        speedup = approach1_time / approach2_time if approach2_time > 0 else 0
+        if speedup > 1:
+            logger.success(f"Approach 2 (get_pages upfront) was {speedup:.2f}x faster", tag="TEST")
+        else:
+            logger.info(f"Approach 1 (get_page + gather) was {1/speedup:.2f}x faster", tag="TEST")
+        
+        # Verify same content was retrieved in both approaches
+        assert len(approach1_results) == len(approach2_results), "Result count mismatch"
+        
+        # Sort results for comparison since parallel execution might complete in different order
+        assert sorted(approach1_results) == sorted(approach2_results), "Results content mismatch"
+        
+        await manager.close()
+        return True
+        
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_multi_browser_scaling(num_browsers=3, pages_per_browser=5):
+    """Test performance with multiple browsers and pages per browser.
+    Compares two approaches:
+    1. On-demand page creation (get_page + gather)
+    2. Pre-created pages (get_pages + gather)
+    """
+    logger.info(f"Testing multi-browser scaling with {num_browsers} browsers × {pages_per_browser} pages", tag="TEST")
+    
+    # Generate test URLs
+    total_pages = num_browsers * pages_per_browser
+    urls = [f"https://example.com/page_{i}" for i in range(total_pages)]
+    
+    # Create browser managers
+    managers = []
+    base_port = 9222
+    
+    try:
+        # Start all browsers in parallel
+        start_tasks = []
+        for i in range(num_browsers):
+            browser_config = BrowserConfig(
+                headless=True  # Using default browser mode like in test_parallel_approaches_comparison
+            )
+            manager = BrowserManager(browser_config=browser_config, logger=logger)
+            start_tasks.append(manager.start())
+            managers.append(manager)
+        
+        await asyncio.gather(*start_tasks)
+        
+        # Distribute URLs among managers
+        urls_per_manager = {}
+        for i, manager in enumerate(managers):
+            start_idx = i * pages_per_browser
+            end_idx = min(start_idx + pages_per_browser, len(urls))
+            urls_per_manager[manager] = urls[start_idx:end_idx]
+        
+        # Approach 1: Create a page for each URL on-demand and run in parallel
+        logger.info("Testing approach 1: get_page for each URL + gather", tag="TEST")
+        start_time = time.time()
+        
+        async def fetch_title_approach1(manager, url):
+            """Create a new page for the URL, go to the URL, and get title"""
+            crawler_config = CrawlerRunConfig(url=url)
+            page, context = await manager.get_page(crawler_config)
+            try:
+                await page.goto(url)
+                title = await page.title()
+                return title
+            finally:
+                await page.close()
+        
+        # Run fetch_title_approach1 for each URL in parallel
+        tasks = []
+        for manager, manager_urls in urls_per_manager.items():
+            for url in manager_urls:
+                tasks.append(fetch_title_approach1(manager, url))
+        
+        approach1_results = await asyncio.gather(*tasks)
+        
+        approach1_time = time.time() - start_time
+        logger.info(f"Approach 1 time (get_page + gather): {approach1_time:.2f}s", tag="TEST")
+        
+        # Approach 2: Get all pages upfront with get_pages, then use them in parallel
+        logger.info("Testing approach 2: get_pages upfront + gather", tag="TEST")
+        start_time = time.time()
+        
+        # Get all pages upfront for each manager
+        all_pages = []
+        for manager, manager_urls in urls_per_manager.items():
+            crawler_config = CrawlerRunConfig()
+            pages = await manager.get_pages(crawler_config, count=len(manager_urls))
+            all_pages.extend(zip(pages, manager_urls))
+        
+        async def fetch_title_approach2(page_ctx, url):
+            """Use a pre-created page to go to URL and get title"""
+            page, _ = page_ctx
+            try:
+                await page.goto(url)
+                title = await page.title()
+                return title
+            finally:
+                await page.close()
+        
+        # Use the pre-created pages to fetch titles in parallel
+        tasks = [fetch_title_approach2(page_ctx, url) for page_ctx, url in all_pages]
+        approach2_results = await asyncio.gather(*tasks)
+        
+        approach2_time = time.time() - start_time
+        logger.info(f"Approach 2 time (get_pages + gather): {approach2_time:.2f}s", tag="TEST")
+        
+        # Compare results and performance
+        speedup = approach1_time / approach2_time if approach2_time > 0 else 0
+        pages_per_second = total_pages / approach2_time
+        
+        # Show a simple summary
+        logger.info(f"📊 Summary: {num_browsers} browsers × {pages_per_browser} pages = {total_pages} total crawls", tag="TEST")
+        logger.info(f"⚡ Performance: {pages_per_second:.1f} pages/second ({pages_per_second*60:.0f} pages/minute)", tag="TEST")
+        logger.info(f"🚀 Total crawl time: {approach2_time:.2f} seconds", tag="TEST")
+        
+        if speedup > 1:
+            logger.success(f"✅ Approach 2 (get_pages upfront) was {speedup:.2f}x faster", tag="TEST")
+        else:
+            logger.info(f"✅ Approach 1 (get_page + gather) was {1/speedup:.2f}x faster", tag="TEST")
+        
+        # Close all managers
+        for manager in managers:
+            await manager.close()
+        
+        return True
+    
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Clean up
+        for manager in managers:
+            try:
+                await manager.close()
+            except:
+                pass
+        return False
+
+async def grid_search_optimal_configuration(total_urls=50):
+    """Perform a grid search to find the optimal balance between number of browsers and pages per browser.
+    
+    This function tests different combinations of browser count and pages per browser,
+    while keeping the total number of URLs constant. It measures performance metrics
+    for each configuration to find the "sweet spot" that provides the best speed 
+    with reasonable memory usage.
+    
+    Args:
+        total_urls: Total number of URLs to crawl (default: 50)
+    """
+    logger.info(f"=== GRID SEARCH FOR OPTIMAL CRAWLING CONFIGURATION ({total_urls} URLs) ===", tag="TEST")
+    
+    # Generate test URLs once
+    urls = [f"https://example.com/page_{i}" for i in range(total_urls)]
+    
+    # Define grid search configurations
+    # We'll use more flexible approach: test all browser counts from 1 to min(20, total_urls)
+    # and distribute pages evenly (some browsers may have 1 more page than others)
+    configurations = []
+    
+    # Maximum number of browsers to test
+    max_browsers_to_test = min(20, total_urls)
+    
+    # Try configurations with 1 to max_browsers_to_test browsers
+    for num_browsers in range(1, max_browsers_to_test + 1):
+        base_pages_per_browser = total_urls // num_browsers
+        remainder = total_urls % num_browsers
+        
+        # Generate exact page distribution array
+        if remainder > 0:
+            # First 'remainder' browsers get one more page
+            page_distribution = [base_pages_per_browser + 1] * remainder + [base_pages_per_browser] * (num_browsers - remainder)
+            pages_distribution = f"{base_pages_per_browser+1} pages × {remainder} browsers, {base_pages_per_browser} pages × {num_browsers - remainder} browsers"
+        else:
+            # All browsers get the same number of pages
+            page_distribution = [base_pages_per_browser] * num_browsers
+            pages_distribution = f"{base_pages_per_browser} pages × {num_browsers} browsers"
+        
+        # Format the distribution as a tuple string like (4, 4, 3, 3)
+        distribution_str = str(tuple(page_distribution))
+            
+        configurations.append((num_browsers, base_pages_per_browser, pages_distribution, page_distribution, distribution_str))
+    
+    # Track results
+    results = []
+    
+    # Test each configuration
+    for num_browsers, pages_per_browser, pages_distribution, page_distribution, distribution_str in configurations:
+        logger.info("-" * 80, tag="TEST")
+        logger.info(f"Testing configuration: {num_browsers} browsers with distribution: {distribution_str}", tag="TEST")
+        logger.info(f"Details: {pages_distribution}", tag="TEST")
+        # Sleep a bit for randomness
+        await asyncio.sleep(0.5)
+        
+        try:
+            # Import psutil for memory tracking
+            try:
+                import psutil
+                process = psutil.Process()
+                initial_memory = process.memory_info().rss / (1024 * 1024)  # MB
+            except ImportError:
+                logger.warning("psutil not available, memory metrics will not be tracked", tag="TEST")
+                initial_memory = 0
+            
+            # Create and start browser managers
+            managers = []
+            start_time = time.time()
+            
+            # Start all browsers in parallel
+            start_tasks = []
+            for i in range(num_browsers):
+                browser_config = BrowserConfig(
+                    headless=True
+                )
+                manager = BrowserManager(browser_config=browser_config, logger=logger)
+                start_tasks.append(manager.start())
+                managers.append(manager)
+            
+            await asyncio.gather(*start_tasks)
+            browser_startup_time = time.time() - start_time
+            
+            # Measure memory after browser startup
+            if initial_memory > 0:
+                browser_memory = process.memory_info().rss / (1024 * 1024) - initial_memory
+            else:
+                browser_memory = 0
+            
+            # Distribute URLs among managers using the exact page distribution
+            urls_per_manager = {}
+            total_assigned = 0
+            
+            for i, manager in enumerate(managers):
+                if i < len(page_distribution):
+                    # Get the exact number of pages for this browser from our distribution
+                    manager_pages = page_distribution[i]
+                    
+                    # Get the URL slice for this manager
+                    start_idx = total_assigned
+                    end_idx = start_idx + manager_pages
+                    urls_per_manager[manager] = urls[start_idx:end_idx]
+                    total_assigned += manager_pages
+                else:
+                    # If we have more managers than our distribution (should never happen)
+                    urls_per_manager[manager] = []
+            
+            # Use the more efficient approach (pre-created pages)
+            logger.info("Running page crawling test...", tag="TEST")
+            crawl_start_time = time.time()
+            
+            # Get all pages upfront for each manager
+            all_pages = []
+            for manager, manager_urls in urls_per_manager.items():
+                if not manager_urls:  # Skip managers with no URLs
+                    continue
+                crawler_config = CrawlerRunConfig()
+                pages = await manager.get_pages(crawler_config, count=len(manager_urls))
+                all_pages.extend(zip(pages, manager_urls))
+            
+            # Measure memory after page creation
+            if initial_memory > 0:
+                pages_memory = process.memory_info().rss / (1024 * 1024) - browser_memory - initial_memory
+            else:
+                pages_memory = 0
+                
+            # Function to crawl a URL with a pre-created page
+            async def fetch_title(page_ctx, url):
+                page, _ = page_ctx
+                try:
+                    await page.goto(url)
+                    title = await page.title()
+                    return title
+                finally:
+                    await page.close()
+            
+            # Use the pre-created pages to fetch titles in parallel
+            tasks = [fetch_title(page_ctx, url) for page_ctx, url in all_pages]
+            crawl_results = await asyncio.gather(*tasks)
+            
+            crawl_time = time.time() - crawl_start_time
+            total_time = time.time() - start_time
+            
+            # Final memory measurement
+            if initial_memory > 0:
+                peak_memory = max(browser_memory + pages_memory, process.memory_info().rss / (1024 * 1024) - initial_memory)
+            else:
+                peak_memory = 0
+                
+            # Close all managers
+            for manager in managers:
+                await manager.close()
+                
+            # Calculate metrics
+            pages_per_second = total_urls / crawl_time
+            
+            # Store result metrics
+            result = {
+                "num_browsers": num_browsers,
+                "pages_per_browser": pages_per_browser,
+                "page_distribution": page_distribution,
+                "distribution_str": distribution_str,
+                "total_urls": total_urls,
+                "browser_startup_time": browser_startup_time,
+                "crawl_time": crawl_time,
+                "total_time": total_time,
+                "browser_memory": browser_memory,
+                "pages_memory": pages_memory,
+                "peak_memory": peak_memory,
+                "pages_per_second": pages_per_second,
+                # Calculate efficiency score (higher is better)
+                # This balances speed vs memory usage
+                "efficiency_score": pages_per_second / (peak_memory + 1) if peak_memory > 0 else pages_per_second,
+            }
+            
+            results.append(result)
+            
+            # Log the results
+            logger.info(f"Browser startup: {browser_startup_time:.2f}s", tag="TEST")
+            logger.info(f"Crawl time: {crawl_time:.2f}s", tag="TEST")
+            logger.info(f"Total time: {total_time:.2f}s", tag="TEST")
+            logger.info(f"Performance: {pages_per_second:.1f} pages/second", tag="TEST")
+            
+            if peak_memory > 0:
+                logger.info(f"Browser memory: {browser_memory:.1f}MB", tag="TEST")
+                logger.info(f"Pages memory: {pages_memory:.1f}MB", tag="TEST")
+                logger.info(f"Peak memory: {peak_memory:.1f}MB", tag="TEST")
+                logger.info(f"Efficiency score: {result['efficiency_score']:.6f}", tag="TEST")
+                
+        except Exception as e:
+            logger.error(f"Error testing configuration: {str(e)}", tag="TEST")
+            import traceback
+            traceback.print_exc()
+            
+            # Clean up
+            for manager in managers:
+                try:
+                    await manager.close()
+                except:
+                    pass
+                    
+    # Print summary of all configurations
+    logger.info("=" * 100, tag="TEST")
+    logger.info("GRID SEARCH RESULTS SUMMARY", tag="TEST")
+    logger.info("=" * 100, tag="TEST")
+    
+    # Rank configurations by efficiency score
+    ranked_results = sorted(results, key=lambda x: x["efficiency_score"], reverse=True)
+    
+    # Also determine rankings by different metrics
+    fastest = sorted(results, key=lambda x: x["crawl_time"])[0]
+    lowest_memory = sorted(results, key=lambda x: x["peak_memory"] if x["peak_memory"] > 0 else float('inf'))[0]
+    most_efficient = ranked_results[0]
+    
+    # Print top performers by category
+    logger.info("🏆 TOP PERFORMERS BY CATEGORY:", tag="TEST")
+    logger.info(f"⚡ Fastest: {fastest['num_browsers']} browsers × ~{fastest['pages_per_browser']} pages " + 
+                f"({fastest['crawl_time']:.2f}s, {fastest['pages_per_second']:.1f} pages/s)", tag="TEST")
+    
+    if lowest_memory["peak_memory"] > 0:
+        logger.info(f"💾 Lowest memory: {lowest_memory['num_browsers']} browsers × ~{lowest_memory['pages_per_browser']} pages " +
+                    f"({lowest_memory['peak_memory']:.1f}MB)", tag="TEST")
+    
+    logger.info(f"🌟 Most efficient: {most_efficient['num_browsers']} browsers × ~{most_efficient['pages_per_browser']} pages " +
+                f"(score: {most_efficient['efficiency_score']:.6f})", tag="TEST")
+    
+    # Print result table header
+    logger.info("\n📊 COMPLETE RANKING TABLE (SORTED BY EFFICIENCY SCORE):", tag="TEST")
+    logger.info("-" * 120, tag="TEST")
+    
+    # Define table header
+    header = f"{'Rank':<5} | {'Browsers':<8} | {'Distribution':<55} | {'Total Time(s)':<12} | {'Speed(p/s)':<12} | {'Memory(MB)':<12} | {'Efficiency':<10} | {'Notes'}"
+    logger.info(header, tag="TEST")
+    logger.info("-" * 120, tag="TEST")
+    
+    # Print each configuration in ranked order
+    for rank, result in enumerate(ranked_results, 1):
+        # Add special notes for top performers
+        notes = []
+        if result == fastest:
+            notes.append("⚡ Fastest")
+        if result == lowest_memory:
+            notes.append("💾 Lowest Memory")
+        if result == most_efficient:
+            notes.append("🌟 Most Efficient")
+        
+        notes_str = " | ".join(notes) if notes else ""
+        
+        # Format memory if available
+        memory_str = f"{result['peak_memory']:.1f}" if result['peak_memory'] > 0 else "N/A"
+        
+        # Get the distribution string
+        dist_str = result.get('distribution_str', str(tuple([result['pages_per_browser']] * result['num_browsers'])))
+        
+        # Build the row
+        row = f"{rank:<5} | {result['num_browsers']:<8} | {dist_str:<55} | {result['total_time']:.2f}s{' ':<7} | "
+        row += f"{result['pages_per_second']:.2f}{' ':<6} | {memory_str}{' ':<6} | {result['efficiency_score']:.4f}{' ':<4} | {notes_str}"
+        
+        logger.info(row, tag="TEST")
+    
+    logger.info("-" * 120, tag="TEST")
+    
+    # Generate visualization if matplotlib is available
+    try:
+        import matplotlib.pyplot as plt
+        import numpy as np
+        
+        # Extract data for plotting from ranked results
+        browser_counts = [r["num_browsers"] for r in ranked_results]
+        efficiency_scores = [r["efficiency_score"] for r in ranked_results]
+        crawl_times = [r["crawl_time"] for r in ranked_results]
+        total_times = [r["total_time"] for r in ranked_results]
+        
+        # Filter results with memory data
+        memory_results = [r for r in ranked_results if r["peak_memory"] > 0]
+        memory_browser_counts = [r["num_browsers"] for r in memory_results]
+        peak_memories = [r["peak_memory"] for r in memory_results]
+        
+        # Create figure with clean design
+        plt.figure(figsize=(14, 12), facecolor='white')
+        plt.style.use('ggplot')
+        
+        # Create grid for subplots
+        gs = plt.GridSpec(3, 1, height_ratios=[1, 1, 1], hspace=0.3)
+        
+        # Plot 1: Efficiency Score (higher is better)
+        ax1 = plt.subplot(gs[0])
+        bar_colors = ['#3498db'] * len(browser_counts)
+        
+        # Highlight the most efficient
+        most_efficient_idx = browser_counts.index(most_efficient["num_browsers"])
+        bar_colors[most_efficient_idx] = '#e74c3c'  # Red for most efficient
+        
+        bars = ax1.bar(range(len(browser_counts)), efficiency_scores, color=bar_colors)
+        ax1.set_xticks(range(len(browser_counts)))
+        ax1.set_xticklabels([f"{bc}" for bc in browser_counts], rotation=45)
+        ax1.set_xlabel('Number of Browsers')
+        ax1.set_ylabel('Efficiency Score (higher is better)')
+        ax1.set_title('Browser Configuration Efficiency (higher is better)')
+        
+        # Add value labels on top of bars
+        for bar, score in zip(bars, efficiency_scores):
+            height = bar.get_height()
+            ax1.text(bar.get_x() + bar.get_width()/2., height + 0.02*max(efficiency_scores),
+                    f'{score:.3f}', ha='center', va='bottom', rotation=90, fontsize=8)
+        
+        # Highlight best configuration
+        ax1.text(0.02, 0.90, f"🌟 Most Efficient: {most_efficient['num_browsers']} browsers with ~{most_efficient['pages_per_browser']} pages",
+                transform=ax1.transAxes, fontsize=12, verticalalignment='top',
+                bbox=dict(boxstyle='round,pad=0.5', facecolor='yellow', alpha=0.3))
+        
+        # Plot 2: Time Performance
+        ax2 = plt.subplot(gs[1])
+        
+        # Plot both total time and crawl time
+        ax2.plot(browser_counts, crawl_times, 'bo-', label='Crawl Time (s)', linewidth=2)
+        ax2.plot(browser_counts, total_times, 'go--', label='Total Time (s)', linewidth=2, alpha=0.6)
+        
+        # Mark the fastest configuration
+        fastest_idx = browser_counts.index(fastest["num_browsers"])
+        ax2.plot(browser_counts[fastest_idx], crawl_times[fastest_idx], 'ro', ms=10, 
+                label=f'Fastest: {fastest["num_browsers"]} browsers')
+        
+        ax2.set_xlabel('Number of Browsers')
+        ax2.set_ylabel('Time (seconds)')
+        ax2.set_title(f'Time Performance for {total_urls} URLs by Browser Count')
+        ax2.grid(True, linestyle='--', alpha=0.7)
+        ax2.legend(loc='upper right')
+        
+        # Plot pages per second on second y-axis
+        pages_per_second = [total_urls/t for t in crawl_times]
+        ax2_twin = ax2.twinx()
+        ax2_twin.plot(browser_counts, pages_per_second, 'r^--', label='Pages/second', alpha=0.5)
+        ax2_twin.set_ylabel('Pages per second')
+        
+        # Add note about the fastest configuration
+        ax2.text(0.02, 0.90, f"⚡ Fastest: {fastest['num_browsers']} browsers with ~{fastest['pages_per_browser']} pages" +
+                f"\n   {fastest['crawl_time']:.2f}s ({fastest['pages_per_second']:.1f} pages/s)",
+                transform=ax2.transAxes, fontsize=12, verticalalignment='top',
+                bbox=dict(boxstyle='round,pad=0.5', facecolor='lightblue', alpha=0.3))
+        
+        # Plot 3: Memory Usage (if available)
+        if memory_results:
+            ax3 = plt.subplot(gs[2])
+            
+            # Prepare data for grouped bar chart
+            memory_per_browser = [m/n for m, n in zip(peak_memories, memory_browser_counts)]
+            memory_per_page = [m/(n*p) for m, n, p in zip(
+                [r["peak_memory"] for r in memory_results],
+                [r["num_browsers"] for r in memory_results],
+                [r["pages_per_browser"] for r in memory_results])]
+            
+            x = np.arange(len(memory_browser_counts))
+            width = 0.35
+            
+            # Create grouped bars
+            ax3.bar(x - width/2, peak_memories, width, label='Total Memory (MB)', color='#9b59b6')
+            ax3.bar(x + width/2, memory_per_browser, width, label='Memory per Browser (MB)', color='#3498db')
+            
+            # Configure axis
+            ax3.set_xticks(x)
+            ax3.set_xticklabels([f"{bc}" for bc in memory_browser_counts], rotation=45)
+            ax3.set_xlabel('Number of Browsers')
+            ax3.set_ylabel('Memory (MB)')
+            ax3.set_title('Memory Usage by Browser Configuration')
+            ax3.legend(loc='upper left')
+            ax3.grid(True, linestyle='--', alpha=0.7)
+            
+            # Add second y-axis for memory per page
+            ax3_twin = ax3.twinx()
+            ax3_twin.plot(x, memory_per_page, 'ro-', label='Memory per Page (MB)')
+            ax3_twin.set_ylabel('Memory per Page (MB)')
+            
+            # Get lowest memory configuration
+            lowest_memory_idx = memory_browser_counts.index(lowest_memory["num_browsers"])
+            
+            # Add note about lowest memory configuration
+            ax3.text(0.02, 0.90, f"💾 Lowest Memory: {lowest_memory['num_browsers']} browsers with ~{lowest_memory['pages_per_browser']} pages" +
+                    f"\n   {lowest_memory['peak_memory']:.1f}MB ({lowest_memory['peak_memory']/total_urls:.2f}MB per page)",
+                    transform=ax3.transAxes, fontsize=12, verticalalignment='top',
+                    bbox=dict(boxstyle='round,pad=0.5', facecolor='lightgreen', alpha=0.3))
+        
+        # Add overall title
+        plt.suptitle(f'Browser Scaling Grid Search Results for {total_urls} URLs', fontsize=16, y=0.98)
+        
+        # Add timestamp and info at the bottom
+        plt.figtext(0.5, 0.01, f"Generated by Crawl4AI at {time.strftime('%Y-%m-%d %H:%M:%S')}", 
+                   ha="center", fontsize=10, style='italic')
+        
+        # Get current directory and save the figure there
+        import os
+        __current_file = os.path.abspath(__file__)
+        current_dir = os.path.dirname(__current_file)
+        output_file = os.path.join(current_dir, 'browser_scaling_grid_search.png')
+        
+        # Adjust layout and save figure with high DPI
+        plt.tight_layout(rect=[0, 0.03, 1, 0.97])
+        plt.savefig(output_file, dpi=200, bbox_inches='tight')
+        logger.success(f"Visualization saved to {output_file}", tag="TEST")
+        
+    except ImportError:
+        logger.warning("matplotlib not available, skipping visualization", tag="TEST")
+    
+    return most_efficient["num_browsers"], most_efficient["pages_per_browser"]
+    
+async def find_optimal_browser_config(total_urls=50, verbose=True, rate_limit_delay=0.2):
+    """Find optimal browser configuration for crawling a specific number of URLs.
+    
+    Args:
+        total_urls: Number of URLs to crawl
+        verbose: Whether to print progress
+        rate_limit_delay: Delay between page loads to avoid rate limiting
+        
+    Returns:
+        dict: Contains fastest, lowest_memory, and optimal configurations
+    """
+    if verbose:
+        print(f"\n=== Finding optimal configuration for crawling {total_urls} URLs ===\n")
+    
+    # Generate test URLs with timestamp to avoid caching
+    timestamp = int(time.time())
+    urls = [f"https://example.com/page_{i}?t={timestamp}" for i in range(total_urls)]
+    
+    # Limit browser configurations to test (1 browser to max 10)
+    max_browsers = min(10, total_urls)
+    configs_to_test = []
+    
+    # Generate configurations (browser count, pages distribution)
+    for num_browsers in range(1, max_browsers + 1):
+        base_pages = total_urls // num_browsers
+        remainder = total_urls % num_browsers
+        
+        # Create distribution array like [3, 3, 2, 2] (some browsers get one more page)
+        if remainder > 0:
+            distribution = [base_pages + 1] * remainder + [base_pages] * (num_browsers - remainder)
+        else:
+            distribution = [base_pages] * num_browsers
+            
+        configs_to_test.append((num_browsers, distribution))
+    
+    results = []
+    
+    # Test each configuration
+    for browser_count, page_distribution in configs_to_test:
+        if verbose:
+            print(f"Testing {browser_count} browsers with distribution {tuple(page_distribution)}")
+        
+        try:
+            # Track memory if possible
+            try:
+                import psutil
+                process = psutil.Process()
+                start_memory = process.memory_info().rss / (1024 * 1024)  # MB
+            except ImportError:
+                if verbose: 
+                    print("Memory tracking not available (psutil not installed)")
+                start_memory = 0
+            
+            # Start browsers in parallel
+            managers = []
+            start_tasks = []
+            start_time = time.time()
+            
+            for i in range(browser_count):
+                config = BrowserConfig(headless=True)
+                manager = BrowserManager(browser_config=config, logger=logger)
+                start_tasks.append(manager.start())
+                managers.append(manager)
+            
+            await asyncio.gather(*start_tasks)
+            
+            # Distribute URLs among browsers
+            urls_per_manager = {}
+            url_index = 0
+            
+            for i, manager in enumerate(managers):
+                pages_for_this_browser = page_distribution[i]
+                end_index = url_index + pages_for_this_browser
+                urls_per_manager[manager] = urls[url_index:end_index]
+                url_index = end_index
+            
+            # Create pages for each browser
+            all_pages = []
+            for manager, manager_urls in urls_per_manager.items():
+                if not manager_urls:
+                    continue
+                pages = await manager.get_pages(CrawlerRunConfig(), count=len(manager_urls))
+                all_pages.extend(zip(pages, manager_urls))
+            
+            # Crawl pages with delay to avoid rate limiting
+            async def crawl_page(page_ctx, url):
+                page, _ = page_ctx
+                try:
+                    await page.goto(url)
+                    if rate_limit_delay > 0:
+                        await asyncio.sleep(rate_limit_delay)
+                    title = await page.title()
+                    return title
+                finally:
+                    await page.close()
+            
+            crawl_start = time.time()
+            crawl_tasks = [crawl_page(page_ctx, url) for page_ctx, url in all_pages]
+            await asyncio.gather(*crawl_tasks)
+            crawl_time = time.time() - crawl_start
+            total_time = time.time() - start_time
+            
+            # Measure final memory usage
+            if start_memory > 0:
+                end_memory = process.memory_info().rss / (1024 * 1024)
+                memory_used = end_memory - start_memory
+            else:
+                memory_used = 0
+            
+            # Close all browsers
+            for manager in managers:
+                await manager.close()
+            
+            # Calculate metrics
+            pages_per_second = total_urls / crawl_time
+            
+            # Calculate efficiency score (higher is better)
+            # This balances speed vs memory
+            if memory_used > 0:
+                efficiency = pages_per_second / (memory_used + 1)
+            else:
+                efficiency = pages_per_second
+            
+            # Store result
+            result = {
+                "browser_count": browser_count,
+                "distribution": tuple(page_distribution),
+                "crawl_time": crawl_time,
+                "total_time": total_time,
+                "memory_used": memory_used,
+                "pages_per_second": pages_per_second, 
+                "efficiency": efficiency
+            }
+            
+            results.append(result)
+            
+            if verbose:
+                print(f"  ✓ Crawled {total_urls} pages in {crawl_time:.2f}s ({pages_per_second:.1f} pages/sec)")
+                if memory_used > 0:
+                    print(f"  ✓ Memory used: {memory_used:.1f}MB ({memory_used/total_urls:.1f}MB per page)")
+                print(f"  ✓ Efficiency score: {efficiency:.4f}")
+            
+        except Exception as e:
+            if verbose:
+                print(f"  ✗ Error: {str(e)}")
+            
+            # Clean up
+            for manager in managers:
+                try:
+                    await manager.close()
+                except:
+                    pass
+    
+    # If no successful results, return None
+    if not results:
+        return None
+    
+    # Find best configurations
+    fastest = sorted(results, key=lambda x: x["crawl_time"])[0]
+    
+    # Only consider memory if available
+    memory_results = [r for r in results if r["memory_used"] > 0]
+    if memory_results:
+        lowest_memory = sorted(memory_results, key=lambda x: x["memory_used"])[0]
+    else:
+        lowest_memory = fastest
+    
+    # Find most efficient (balanced speed vs memory)
+    optimal = sorted(results, key=lambda x: x["efficiency"], reverse=True)[0]
+    
+    # Print summary
+    if verbose:
+        print("\n=== OPTIMAL CONFIGURATIONS ===")
+        print(f"⚡ Fastest: {fastest['browser_count']} browsers {fastest['distribution']}")
+        print(f"   {fastest['crawl_time']:.2f}s, {fastest['pages_per_second']:.1f} pages/sec")
+        
+        print(f"💾 Memory-efficient: {lowest_memory['browser_count']} browsers {lowest_memory['distribution']}")
+        if lowest_memory["memory_used"] > 0:
+            print(f"   {lowest_memory['memory_used']:.1f}MB, {lowest_memory['memory_used']/total_urls:.2f}MB per page")
+        
+        print(f"🌟 Balanced optimal: {optimal['browser_count']} browsers {optimal['distribution']}")
+        print(f"   {optimal['crawl_time']:.2f}s, {optimal['pages_per_second']:.1f} pages/sec, score: {optimal['efficiency']:.4f}")
+    
+    return {
+        "fastest": fastest,
+        "lowest_memory": lowest_memory,
+        "optimal": optimal,
+        "all_configs": results
+    }
+
+async def run_tests():
+    """Run all tests sequentially."""
+    results = []
+    
+    # Find optimal configuration using our utility function
+    configs = await find_optimal_browser_config(
+        total_urls=20,  # Use a small number for faster testing
+        verbose=True,
+        rate_limit_delay=0.2  # 200ms delay between page loads to avoid rate limiting
+    )
+    
+    if configs:
+        # Show the optimal configuration
+        optimal = configs["optimal"]
+        print(f"\n🎯 Recommended configuration for production use:")
+        print(f"   {optimal['browser_count']} browsers with distribution {optimal['distribution']}")
+        print(f"   Estimated performance: {optimal['pages_per_second']:.1f} pages/second")
+        results.append(True)
+    else:
+        print("\n❌ Failed to find optimal configuration")
+        results.append(False)
+    
+    # Print summary
+    total = len(results)
+    passed = sum(results)
+    print(f"\nTests complete: {passed}/{total} passed")
+    
+    if passed == total:
+        print("All tests passed!")
+    else:
+        print(f"{total - passed} tests failed")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
\ No newline at end of file
diff --git a/tests/browser/test_playwright_strategy.py b/tests/browser/test_playwright_strategy.py
new file mode 100644
index 0000000..94003b5
--- /dev/null
+++ b/tests/browser/test_playwright_strategy.py
@@ -0,0 +1,316 @@
+"""Test examples for PlaywrightBrowserStrategy.
+
+These examples demonstrate the functionality of PlaywrightBrowserStrategy
+and serve as functional tests.
+"""
+
+import asyncio
+import os
+import re
+import sys
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai.browser import BrowserManager
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+
+
+async def test_start_close():
+    # Create browser config for standard Playwright
+    browser_config = BrowserConfig(
+        headless=True,
+        viewport_width=1280,
+        viewport_height=800
+    )
+    
+    # Create browser manager with the config
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        for _ in range(4):
+            # Start the browser
+            await manager.start()
+            logger.info("Browser started successfully", tag="TEST")
+
+            # Get a page
+            page, context = await manager.get_page(CrawlerRunConfig())
+            logger.info("Got page successfully", tag="TEST")
+            
+            # Navigate to a website
+            await page.goto("https://example.com")
+            logger.info("Navigated to example.com", tag="TEST")
+            
+            # Get page title
+            title = await page.title()
+            logger.info(f"Page title: {title}", tag="TEST")
+            
+            # Clean up
+            await manager.close()
+            logger.info("Browser closed successfully", tag="TEST")
+   
+            await asyncio.sleep(1)  # Wait for a moment before restarting
+
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+    return True
+
+async def test_playwright_basic():
+    """Test basic Playwright browser functionality."""
+    logger.info("Testing standard Playwright browser", tag="TEST")
+    
+    # Create browser config for standard Playwright
+    browser_config = BrowserConfig(
+        headless=True,
+        viewport_width=1280,
+        viewport_height=800
+    )
+    
+    # Create browser manager with the config
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Create crawler config
+        crawler_config = CrawlerRunConfig(url="https://example.com")
+        
+        # Get a page
+        page, context = await manager.get_page(crawler_config)
+        logger.info("Got page successfully", tag="TEST")
+        
+        # Navigate to a website
+        await page.goto("https://example.com")
+        logger.info("Navigated to example.com", tag="TEST")
+        
+        # Get page title
+        title = await page.title()
+        logger.info(f"Page title: {title}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        return True
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_playwright_text_mode():
+    """Test Playwright browser in text-only mode."""
+    logger.info("Testing Playwright text mode", tag="TEST")
+    
+    # Create browser config with text mode enabled
+    browser_config = BrowserConfig(
+        headless=True,
+        text_mode=True  # Enable text-only mode
+    )
+    
+    # Create browser manager with the config
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully in text mode", tag="TEST")
+        
+        # Get a page
+        crawler_config = CrawlerRunConfig(url="https://example.com")
+        page, context = await manager.get_page(crawler_config)
+        
+        # Navigate to a website
+        await page.goto("https://example.com")
+        logger.info("Navigated to example.com", tag="TEST")
+        
+        # Get page title
+        title = await page.title()
+        logger.info(f"Page title: {title}", tag="TEST")
+        
+        # Check if images are blocked in text mode
+        # We'll check if any image requests were made
+        has_images = False
+        async with page.expect_request("**/*.{png,jpg,jpeg,gif,webp,svg}", timeout=1000) as request_info:
+            try:
+                # Try to load a page with images
+                await page.goto("https://picsum.photos/", wait_until="domcontentloaded")
+                request = await request_info.value
+                has_images = True
+            except:
+                # Timeout without image requests means text mode is working
+                has_images = False
+        
+        logger.info(f"Text mode image blocking working: {not has_images}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        return True
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_playwright_context_reuse():
+    """Test context caching and reuse with identical configurations."""
+    logger.info("Testing context reuse with identical configurations", tag="TEST")
+    
+    # Create browser config
+    browser_config = BrowserConfig(headless=True)
+    
+    # Create browser manager
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        # Start the browser
+        await manager.start()
+        logger.info("Browser started successfully", tag="TEST")
+        
+        # Create identical crawler configs
+        crawler_config1 = CrawlerRunConfig(
+            css_selector="body",
+        )
+        
+        crawler_config2 = CrawlerRunConfig(
+            css_selector="body",
+        )
+        
+        # Get pages with these configs
+        page1, context1 = await manager.get_page(crawler_config1)
+        page2, context2 = await manager.get_page(crawler_config2)
+        
+        # Check if contexts are reused
+        is_same_context = context1 == context2
+        logger.info(f"Contexts reused: {is_same_context}", tag="TEST")
+        
+        # Now try with a different config
+        crawler_config3 = CrawlerRunConfig()
+        
+        page3, context3 = await manager.get_page(crawler_config3)
+        
+        # This should be a different context
+        is_different_context = context1 != context3
+        logger.info(f"Different contexts for different configs: {is_different_context}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        # Both tests should pass for success
+        return is_same_context and is_different_context
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Ensure cleanup
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def test_playwright_session_management():
+    """Test session management with Playwright browser."""
+    logger.info("Testing session management with Playwright browser", tag="TEST")
+    
+    browser_config = BrowserConfig(
+        headless=True
+    )
+    
+    manager = BrowserManager(browser_config=browser_config, logger=logger)
+    
+    try:
+        await manager.start()
+        logger.info("Browser launched successfully", tag="TEST")
+        
+        # Create two sessions
+        session1_id = "playwright_session_1"
+        session2_id = "playwright_session_2"
+        
+        # Set up first session
+        crawler_config1 = CrawlerRunConfig(session_id=session1_id, url="https://example.com")
+        page1, context1 = await manager.get_page(crawler_config1)
+        await page1.goto("https://example.com")
+        await page1.evaluate("localStorage.setItem('playwright_session1_data', 'test_value1')")
+        logger.info(f"Set up session 1 with ID: {session1_id}", tag="TEST")
+        
+        # Set up second session
+        crawler_config2 = CrawlerRunConfig(session_id=session2_id, url="https://example.org")
+        page2, context2 = await manager.get_page(crawler_config2)
+        await page2.goto("https://example.org")
+        await page2.evaluate("localStorage.setItem('playwright_session2_data', 'test_value2')")
+        logger.info(f"Set up session 2 with ID: {session2_id}", tag="TEST")
+        
+        # Get first session again
+        page1_again, context1_again = await manager.get_page(crawler_config1)
+        
+        # Verify it's the same page and data persists
+        is_same_page = page1 == page1_again
+        is_same_context = context1 == context1_again
+        data1 = await page1_again.evaluate("localStorage.getItem('playwright_session1_data')")
+        logger.info(f"Session 1 reuse successful: {is_same_page}, data: {data1}", tag="TEST")
+        
+        # Kill first session
+        await manager.kill_session(session1_id)
+        logger.info(f"Killed session 1", tag="TEST")
+        
+        # Verify second session still works
+        data2 = await page2.evaluate("localStorage.getItem('playwright_session2_data')")
+        logger.info(f"Session 2 still functional after killing session 1, data: {data2}", tag="TEST")
+        
+        # Clean up
+        await manager.close()
+        logger.info("Browser closed successfully", tag="TEST")
+        
+        return is_same_page and is_same_context and data1 == "test_value1" and data2 == "test_value2"
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        try:
+            await manager.close()
+        except:
+            pass
+        return False
+
+async def run_tests():
+    """Run all tests sequentially."""
+    results = []
+    
+    # results.append(await test_start_close())
+    # results.append(await test_playwright_basic())
+    # results.append(await test_playwright_text_mode())
+    # results.append(await test_playwright_context_reuse())
+    results.append(await test_playwright_session_management())
+    
+    # Print summary
+    total = len(results)
+    passed = sum(results)
+    logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
+    
+    if passed == total:
+        logger.success("All tests passed!", tag="SUMMARY")
+    else:
+        logger.error(f"{total - passed} tests failed", tag="SUMMARY")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
diff --git a/tests/browser/test_profile_shrink.py b/tests/browser/test_profile_shrink.py
new file mode 100644
index 0000000..6fac2e7
--- /dev/null
+++ b/tests/browser/test_profile_shrink.py
@@ -0,0 +1,1053 @@
+"""
+Tests for profile shrinking functionality.
+
+Test approach:
+1. Unit tests for core shrink logic with mock file structures
+2. Integration tests with real Playwright browser to verify auth preservation
+3. Edge case handling (empty profiles, missing profiles, permission errors)
+"""
+
+import pytest
+import shutil
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+
+from crawl4ai.browser_profiler import (
+    ShrinkLevel,
+    KEEP_PATTERNS,
+    shrink_profile,
+    _get_size,
+    _format_size,
+    BrowserProfiler,
+)
+
+
+class TestShrinkLevel:
+    """Test ShrinkLevel enum."""
+
+    def test_enum_values(self):
+        assert ShrinkLevel.NONE.value == "none"
+        assert ShrinkLevel.LIGHT.value == "light"
+        assert ShrinkLevel.MEDIUM.value == "medium"
+        assert ShrinkLevel.AGGRESSIVE.value == "aggressive"
+        assert ShrinkLevel.MINIMAL.value == "minimal"
+
+    def test_enum_from_string(self):
+        assert ShrinkLevel("aggressive") == ShrinkLevel.AGGRESSIVE
+        assert ShrinkLevel("minimal") == ShrinkLevel.MINIMAL
+
+    def test_keep_patterns_defined_for_all_levels(self):
+        for level in ShrinkLevel:
+            assert level in KEEP_PATTERNS
+
+
+class TestHelperFunctions:
+    """Test helper functions."""
+
+    def test_format_size_bytes(self):
+        assert _format_size(500) == "500.0 B"
+
+    def test_format_size_kilobytes(self):
+        assert _format_size(2048) == "2.0 KB"
+
+    def test_format_size_megabytes(self):
+        assert _format_size(5 * 1024 * 1024) == "5.0 MB"
+
+    def test_format_size_gigabytes(self):
+        assert _format_size(3 * 1024 * 1024 * 1024) == "3.0 GB"
+
+    def test_get_size_file(self, tmp_path):
+        test_file = tmp_path / "test.txt"
+        test_file.write_text("x" * 100)
+        assert _get_size(test_file) == 100
+
+    def test_get_size_directory(self, tmp_path):
+        (tmp_path / "file1.txt").write_text("a" * 50)
+        (tmp_path / "file2.txt").write_text("b" * 50)
+        subdir = tmp_path / "subdir"
+        subdir.mkdir()
+        (subdir / "file3.txt").write_text("c" * 100)
+        assert _get_size(tmp_path) == 200
+
+    def test_get_size_empty_directory(self, tmp_path):
+        assert _get_size(tmp_path) == 0
+
+
+class TestShrinkProfile:
+    """Test shrink_profile function."""
+
+    @pytest.fixture
+    def mock_profile(self, tmp_path):
+        """Create a mock Chrome profile structure."""
+        profile = tmp_path / "test_profile"
+        profile.mkdir()
+
+        # Essential auth directories (should be kept)
+        (profile / "Network").mkdir()
+        (profile / "Network" / "Cookies").write_bytes(b"x" * 1000)
+        (profile / "Local Storage").mkdir()
+        (profile / "Local Storage" / "leveldb").mkdir()
+        (profile / "Local Storage" / "leveldb" / "data").write_bytes(b"y" * 500)
+        (profile / "IndexedDB").mkdir()
+        (profile / "IndexedDB" / "db").write_bytes(b"z" * 300)
+        (profile / "Preferences").write_text('{"profile": {}}')
+
+        # Cache directories (should be removed)
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data_0").write_bytes(b"0" * 10000)
+        (profile / "Cache" / "data_1").write_bytes(b"1" * 10000)
+        (profile / "Code Cache").mkdir()
+        (profile / "Code Cache" / "js").mkdir()
+        (profile / "Code Cache" / "js" / "bytecode").write_bytes(b"c" * 5000)
+        (profile / "GPUCache").mkdir()
+        (profile / "GPUCache" / "data").write_bytes(b"g" * 2000)
+        (profile / "Service Worker").mkdir()
+        (profile / "Service Worker" / "CacheStorage").mkdir()
+        (profile / "Service Worker" / "CacheStorage" / "cache").write_bytes(b"s" * 50000)
+
+        # History and other files (removed at MEDIUM+)
+        (profile / "History").write_bytes(b"h" * 1000)
+        (profile / "Favicons").write_bytes(b"f" * 500)
+        (profile / "Visited Links").write_bytes(b"v" * 200)
+
+        return str(profile)
+
+    def test_shrink_none_keeps_everything(self, mock_profile):
+        result = shrink_profile(mock_profile, ShrinkLevel.NONE)
+        assert result["removed"] == []
+        assert result["kept"] == []
+        assert result["bytes_freed"] == 0
+
+    def test_shrink_aggressive_removes_caches(self, mock_profile):
+        result = shrink_profile(mock_profile, ShrinkLevel.AGGRESSIVE)
+
+        # Auth data kept
+        assert "Network" in result["kept"]
+        assert "Local Storage" in result["kept"]
+        assert "IndexedDB" in result["kept"]
+        assert "Preferences" in result["kept"]
+
+        # Caches removed
+        assert "Cache" in result["removed"]
+        assert "Code Cache" in result["removed"]
+        assert "GPUCache" in result["removed"]
+        assert "Service Worker" in result["removed"]
+
+        # Verify bytes freed > 0
+        assert result["bytes_freed"] > 0
+        assert result["size_after"] < result["size_before"]
+
+    def test_shrink_minimal_keeps_only_essential(self, mock_profile):
+        result = shrink_profile(mock_profile, ShrinkLevel.MINIMAL)
+
+        # Only Network and Local Storage kept
+        assert set(result["kept"]) == {"Network", "Local Storage"}
+
+        # IndexedDB and Preferences removed at MINIMAL
+        assert "IndexedDB" in result["removed"]
+        assert "Preferences" in result["removed"]
+
+    def test_shrink_light_keeps_history(self, mock_profile):
+        result = shrink_profile(mock_profile, ShrinkLevel.LIGHT)
+
+        # History kept at LIGHT level
+        assert "History" in result["kept"]
+
+        # Caches still removed
+        assert "Cache" in result["removed"]
+
+    def test_shrink_medium_removes_history(self, mock_profile):
+        result = shrink_profile(mock_profile, ShrinkLevel.MEDIUM)
+
+        # History removed at MEDIUM
+        assert "History" in result["removed"]
+        assert "Favicons" in result["removed"]
+
+        # Auth still kept
+        assert "Network" in result["kept"]
+
+    def test_shrink_dry_run_no_changes(self, mock_profile):
+        size_before = _get_size(Path(mock_profile))
+
+        result = shrink_profile(mock_profile, ShrinkLevel.AGGRESSIVE, dry_run=True)
+
+        size_after = _get_size(Path(mock_profile))
+        assert size_before == size_after
+        assert result["size_after"] is None
+        assert len(result["removed"]) > 0  # Still reports what would be removed
+
+    def test_shrink_nonexistent_profile_raises(self):
+        with pytest.raises(ValueError, match="Profile not found"):
+            shrink_profile("/nonexistent/path", ShrinkLevel.AGGRESSIVE)
+
+    def test_shrink_empty_profile(self, tmp_path):
+        empty_profile = tmp_path / "empty"
+        empty_profile.mkdir()
+
+        result = shrink_profile(str(empty_profile), ShrinkLevel.AGGRESSIVE)
+        assert result["removed"] == []
+        assert result["kept"] == []
+        assert result["errors"] == []
+
+
+class TestBrowserProfilerShrink:
+    """Test BrowserProfiler.shrink() method."""
+
+    @pytest.fixture
+    def profiler(self):
+        return BrowserProfiler()
+
+    @pytest.fixture
+    def mock_profile_in_profiles_dir(self, profiler, tmp_path):
+        """Create a mock profile in the profiler's profiles directory."""
+        # Temporarily override profiles_dir
+        original_dir = profiler.profiles_dir
+        profiler.profiles_dir = str(tmp_path)
+
+        profile = tmp_path / "test_profile"
+        profile.mkdir()
+        (profile / "Network").mkdir()
+        (profile / "Network" / "Cookies").write_text("cookies")
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data").write_bytes(b"x" * 1000)
+        (profile / "Preferences").write_text("{}")
+
+        yield "test_profile", str(profile)
+
+        # Cleanup
+        profiler.profiles_dir = original_dir
+
+    def test_shrink_by_name(self, profiler, mock_profile_in_profiles_dir):
+        name, path = mock_profile_in_profiles_dir
+
+        result = profiler.shrink(name, ShrinkLevel.AGGRESSIVE)
+
+        assert "Cache" in result["removed"]
+        assert "Network" in result["kept"]
+        assert "Preferences" in result["kept"]
+
+    def test_shrink_by_path(self, profiler, mock_profile_in_profiles_dir):
+        _, path = mock_profile_in_profiles_dir
+
+        result = profiler.shrink(path, ShrinkLevel.AGGRESSIVE)
+
+        assert "Cache" in result["removed"]
+
+    def test_shrink_nonexistent_raises(self, profiler):
+        with pytest.raises(ValueError, match="Profile not found"):
+            profiler.shrink("nonexistent_profile")
+
+
+class TestKeepPatterns:
+    """Test that KEEP_PATTERNS are correctly defined."""
+
+    def test_aggressive_keeps_auth_essentials(self):
+        keep = KEEP_PATTERNS[ShrinkLevel.AGGRESSIVE]
+        assert "Network" in keep  # Cookies (Chrome 96+)
+        assert "Cookies" in keep  # Cookies (older Chrome)
+        assert "Local Storage" in keep  # JWT/tokens
+        assert "IndexedDB" in keep  # Some sites use this
+        assert "Preferences" in keep  # Profile identity
+
+    def test_minimal_is_subset_of_aggressive(self):
+        minimal = KEEP_PATTERNS[ShrinkLevel.MINIMAL]
+        aggressive = KEEP_PATTERNS[ShrinkLevel.AGGRESSIVE]
+        assert minimal.issubset(aggressive)
+
+    def test_aggressive_is_subset_of_medium(self):
+        aggressive = KEEP_PATTERNS[ShrinkLevel.AGGRESSIVE]
+        medium = KEEP_PATTERNS[ShrinkLevel.MEDIUM]
+        assert aggressive.issubset(medium)
+
+    def test_medium_is_subset_of_light(self):
+        medium = KEEP_PATTERNS[ShrinkLevel.MEDIUM]
+        light = KEEP_PATTERNS[ShrinkLevel.LIGHT]
+        assert medium.issubset(light)
+
+
+class TestIntegrationWithPlaywright:
+    """Integration tests using real Playwright browser.
+
+    These tests verify that auth data survives shrinking and the browser
+    can still launch successfully after shrinking.
+    """
+
+    @staticmethod
+    async def _create_seeded_profile(profile_path: str) -> str:
+        """Create a real profile with seeded auth data using Playwright."""
+        from playwright.async_api import async_playwright
+
+        async with async_playwright() as p:
+            browser = await p.chromium.launch_persistent_context(
+                profile_path,
+                headless=True,
+            )
+            page = await browser.new_page()
+
+            # Navigate to a real site to enable localStorage/cookies
+            try:
+                await page.goto("https://example.com", timeout=15000)
+            except Exception:
+                # Fallback to about:blank which still allows localStorage
+                await page.goto("about:blank")
+
+            # Seed test data (localStorage works on any origin)
+            await page.evaluate("""
+                () => {
+                    localStorage.setItem('jwt', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9');
+                    localStorage.setItem('refresh', 'refresh_token_abc');
+                }
+            """)
+
+            await browser.close()
+
+        return profile_path
+
+    @pytest.mark.asyncio
+    async def test_browser_launches_after_aggressive_shrink(self, tmp_path):
+        """Verify browser can launch after aggressive shrinking."""
+        pytest.importorskip("playwright")
+        from playwright.async_api import async_playwright
+
+        profile_path = str(tmp_path / "playwright_profile")
+        await self._create_seeded_profile(profile_path)
+
+        # Shrink the profile
+        result = shrink_profile(profile_path, ShrinkLevel.AGGRESSIVE)
+        assert result["bytes_freed"] >= 0
+
+        # Verify browser launches and localStorage survives
+        async with async_playwright() as p:
+            browser = await p.chromium.launch_persistent_context(
+                profile_path,
+                headless=True,
+            )
+            page = await browser.new_page()
+
+            # Navigate to same origin to access localStorage
+            try:
+                await page.goto("https://example.com", timeout=15000)
+            except Exception:
+                await page.goto("about:blank")
+
+            # Verify localStorage survived
+            jwt = await page.evaluate("localStorage.getItem('jwt')")
+            assert jwt == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"
+
+            refresh = await page.evaluate("localStorage.getItem('refresh')")
+            assert refresh == "refresh_token_abc"
+
+            await browser.close()
+
+    @pytest.mark.asyncio
+    async def test_browser_launches_after_minimal_shrink(self, tmp_path):
+        """Verify browser launches after minimal shrinking (most aggressive)."""
+        pytest.importorskip("playwright")
+        from playwright.async_api import async_playwright
+
+        profile_path = str(tmp_path / "playwright_profile")
+        await self._create_seeded_profile(profile_path)
+
+        # Shrink to minimal
+        result = shrink_profile(profile_path, ShrinkLevel.MINIMAL)
+        assert result["bytes_freed"] >= 0
+
+        # Verify browser still launches
+        async with async_playwright() as p:
+            browser = await p.chromium.launch_persistent_context(
+                profile_path,
+                headless=True,
+            )
+            page = await browser.new_page()
+
+            # Navigate to same origin to access localStorage
+            try:
+                await page.goto("https://example.com", timeout=15000)
+            except Exception:
+                await page.goto("about:blank")
+
+            # localStorage should still work
+            jwt = await page.evaluate("localStorage.getItem('jwt')")
+            assert jwt == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"
+
+            await browser.close()
+
+    @pytest.mark.asyncio
+    async def test_shrink_actually_reduces_size(self, tmp_path):
+        """Verify shrinking actually reduces profile size."""
+        pytest.importorskip("playwright")
+
+        profile_path = str(tmp_path / "playwright_profile")
+        await self._create_seeded_profile(profile_path)
+
+        size_before = _get_size(Path(profile_path))
+
+        result = shrink_profile(profile_path, ShrinkLevel.AGGRESSIVE)
+
+        size_after = _get_size(Path(profile_path))
+
+        # Profile should be smaller (or same if no cache was generated)
+        assert size_after <= size_before
+        assert result["size_before"] == size_before
+        assert result["size_after"] == size_after
+
+
+class TestCLIIntegration:
+    """Test CLI command integration."""
+
+    def test_cli_import(self):
+        """Verify CLI imports work."""
+        from crawl4ai.cli import shrink_cmd
+        assert callable(shrink_cmd)
+
+    def test_shrink_level_import(self):
+        """Verify ShrinkLevel can be imported from cli."""
+        from crawl4ai.browser_profiler import ShrinkLevel
+        assert ShrinkLevel.AGGRESSIVE.value == "aggressive"
+
+
+class TestEdgeCases:
+    """Edge case tests to ensure robustness."""
+
+    def test_shrink_profile_with_symlinks(self, tmp_path):
+        """Test shrinking profile with symlinks doesn't follow them."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+        (profile / "Local Storage").mkdir()
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data").write_bytes(b"x" * 1000)
+
+        # Create symlink pointing outside profile
+        external_dir = tmp_path / "external"
+        external_dir.mkdir()
+        important_file = external_dir / "important.txt"
+        important_file.write_text("DO NOT DELETE")
+
+        # Symlink inside Cache pointing to external
+        symlink = profile / "Cache" / "external_link"
+        symlink.symlink_to(external_dir)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        # External file should NOT be deleted
+        assert important_file.exists(), "Symlink target was deleted!"
+        assert "Cache" in result["removed"]
+
+    def test_shrink_with_special_characters_in_names(self, tmp_path):
+        """Test shrinking handles special chars in filenames."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        # Create dirs/files with special characters
+        (profile / "Local Storage").mkdir()
+        (profile / "Cache (old)").mkdir()
+        (profile / "Cache (old)" / "data").write_bytes(b"x" * 100)
+        (profile / "Test[1]").mkdir()
+        (profile / "Test[1]" / "file").write_bytes(b"y" * 100)
+        (profile / "Spaced Name").mkdir()
+        (profile / "file with spaces.txt").write_bytes(b"z" * 50)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert "Cache (old)" in result["removed"]
+        assert "Test[1]" in result["removed"]
+        assert "Spaced Name" in result["removed"]
+        assert "file with spaces.txt" in result["removed"]
+        assert "Local Storage" in result["kept"]
+
+    def test_shrink_with_unicode_filenames(self, tmp_path):
+        """Test shrinking handles unicode filenames."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Кэш").mkdir()  # Russian "Cache"
+        (profile / "Кэш" / "данные").write_bytes(b"x" * 100)
+        (profile / "缓存").mkdir()  # Chinese "Cache"
+        (profile / "キャッシュ").mkdir()  # Japanese "Cache"
+        (profile / "émojis_🎉").mkdir()
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert "Local Storage" in result["kept"]
+        assert len(result["removed"]) >= 4
+
+    def test_shrink_with_hidden_files(self, tmp_path):
+        """Test shrinking handles hidden (dot) files."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / ".hidden_cache").mkdir()
+        (profile / ".hidden_cache" / "data").write_bytes(b"x" * 1000)
+        (profile / ".DS_Store").write_bytes(b"y" * 100)
+        (profile / ".git").mkdir()
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        # Hidden files should be removed (not in keep list)
+        assert ".hidden_cache" in result["removed"]
+        assert ".DS_Store" in result["removed"]
+        assert ".git" in result["removed"]
+
+    def test_shrink_with_empty_directories(self, tmp_path):
+        """Test shrinking handles empty directories."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Empty Cache").mkdir()
+        (profile / "Another Empty").mkdir()
+        (profile / "Nested").mkdir()
+        (profile / "Nested" / "Also Empty").mkdir()
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert "Empty Cache" in result["removed"]
+        assert "Another Empty" in result["removed"]
+        assert "Nested" in result["removed"]
+        assert not (profile / "Empty Cache").exists()
+
+    def test_shrink_twice_same_profile(self, tmp_path):
+        """Test shrinking same profile twice is idempotent."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Local Storage" / "data").write_bytes(b"x" * 100)
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data").write_bytes(b"y" * 1000)
+
+        # First shrink
+        result1 = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+        assert "Cache" in result1["removed"]
+        assert result1["bytes_freed"] > 0
+
+        # Second shrink - should be no-op
+        result2 = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+        assert result2["removed"] == []
+        assert result2["bytes_freed"] == 0
+        assert "Local Storage" in result2["kept"]
+
+    def test_shrink_preserves_storage_state_json(self, tmp_path):
+        """Test that storage_state.json is preserved."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        # storage_state.json should be kept (starts with no pattern but is important)
+        (profile / "storage_state.json").write_text('{"cookies": []}')
+        (profile / "Local Storage").mkdir()
+        (profile / "Cache").mkdir()
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        # storage_state.json doesn't match keep patterns, so it gets removed
+        # This is expected - the shrink function preserves Chrome's auth files,
+        # not Crawl4AI's exported state file
+        # If we want to keep it, we need to add it to KEEP_PATTERNS
+
+    def test_shrink_with_very_deep_nesting(self, tmp_path):
+        """Test shrinking deeply nested directories."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+
+        # Create deeply nested cache
+        deep = profile / "Cache"
+        for i in range(20):
+            deep = deep / f"level_{i}"
+        deep.mkdir(parents=True)
+        (deep / "deep_file.txt").write_bytes(b"x" * 100)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert "Cache" in result["removed"]
+        assert not (profile / "Cache").exists()
+
+    def test_shrink_with_large_files(self, tmp_path):
+        """Test shrinking handles large files efficiently."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Cache").mkdir()
+
+        # Create a 10MB file
+        large_file = profile / "Cache" / "large_file.bin"
+        large_file.write_bytes(b"x" * (10 * 1024 * 1024))
+
+        size_before = _get_size(profile)
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+        size_after = _get_size(profile)
+
+        assert result["bytes_freed"] >= 10 * 1024 * 1024
+        assert size_after < size_before
+
+    def test_shrink_with_read_only_files(self, tmp_path):
+        """Test shrinking handles read-only files gracefully."""
+        import stat
+
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        cache = profile / "Cache"
+        cache.mkdir()
+        readonly_file = cache / "readonly.txt"
+        readonly_file.write_bytes(b"x" * 100)
+
+        # Make file read-only
+        readonly_file.chmod(stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+
+        try:
+            result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+            # On some systems this will succeed, on others it will error
+            # Either way, it shouldn't crash
+            if result["errors"]:
+                assert "Cache" in str(result["errors"][0]) or len(result["errors"]) > 0
+        finally:
+            # Restore permissions for cleanup
+            try:
+                readonly_file.chmod(stat.S_IRWXU)
+            except:
+                pass
+
+    def test_shrink_with_many_small_files(self, tmp_path):
+        """Test shrinking handles many small files efficiently."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        cache = profile / "Cache"
+        cache.mkdir()
+
+        # Create 1000 small files
+        for i in range(1000):
+            (cache / f"file_{i:04d}.txt").write_bytes(b"x" * 100)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert "Cache" in result["removed"]
+        assert result["bytes_freed"] >= 100 * 1000
+        assert not cache.exists()
+
+    def test_shrink_default_subdirectory_structure(self, tmp_path):
+        """Test shrinking when profile has Default/ subdirectory."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        # Chrome-style structure with Default/
+        default = profile / "Default"
+        default.mkdir()
+        (default / "Local Storage").mkdir()
+        (default / "Local Storage" / "leveldb").mkdir()
+        (default / "Cookies").write_bytes(b"cookies" * 100)
+        (default / "Cache").mkdir()
+        (default / "Cache" / "data").write_bytes(b"x" * 10000)
+        (default / "GPUCache").mkdir()
+        (default / "GPUCache" / "data").write_bytes(b"y" * 5000)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        # Should shrink inside Default/
+        assert "Cache" in result["removed"]
+        assert "GPUCache" in result["removed"]
+        assert "Local Storage" in result["kept"]
+        assert "Cookies" in result["kept"]
+        assert (default / "Local Storage").exists()
+        assert (default / "Cookies").exists()
+        assert not (default / "Cache").exists()
+
+    def test_shrink_mixed_files_and_directories(self, tmp_path):
+        """Test shrinking mix of files and directories."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Preferences").write_text("{}")
+        (profile / "Cookies").write_bytes(b"x" * 500)
+        (profile / "Cookies-journal").write_bytes(b"y" * 100)
+        (profile / "History").write_bytes(b"z" * 1000)
+        (profile / "Cache").mkdir()
+        (profile / "random_file.txt").write_bytes(b"a" * 200)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        # Files and dirs properly categorized
+        assert "Local Storage" in result["kept"]
+        assert "Preferences" in result["kept"]
+        assert "Cookies" in result["kept"]
+        assert "Cookies-journal" in result["kept"]
+        assert "History" in result["removed"]
+        assert "Cache" in result["removed"]
+        assert "random_file.txt" in result["removed"]
+
+    def test_shrink_level_none_is_noop(self, tmp_path):
+        """Test ShrinkLevel.NONE does absolutely nothing."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data").write_bytes(b"x" * 1000)
+
+        size_before = _get_size(profile)
+        result = shrink_profile(str(profile), ShrinkLevel.NONE)
+        size_after = _get_size(profile)
+
+        assert result["removed"] == []
+        assert result["kept"] == []
+        assert result["bytes_freed"] == 0
+        assert size_before == size_after
+
+    def test_shrink_result_sizes_are_accurate(self, tmp_path):
+        """Test that reported sizes match actual sizes."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Local Storage" / "data").write_bytes(b"k" * 500)
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data").write_bytes(b"x" * 2000)
+
+        actual_before = _get_size(profile)
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+        actual_after = _get_size(profile)
+
+        assert result["size_before"] == actual_before
+        assert result["size_after"] == actual_after
+        assert result["size_before"] - result["size_after"] == result["bytes_freed"]
+
+    def test_shrink_all_levels_progressively_smaller(self, tmp_path):
+        """Test that stricter levels remove more data."""
+        def create_full_profile(path):
+            path.mkdir(exist_ok=True)
+            (path / "Network").mkdir(exist_ok=True)
+            (path / "Cookies").write_bytes(b"c" * 100)
+            (path / "Local Storage").mkdir(exist_ok=True)
+            (path / "IndexedDB").mkdir(exist_ok=True)
+            (path / "Preferences").write_text("{}")
+            (path / "History").write_bytes(b"h" * 500)
+            (path / "Bookmarks").write_text("[]")
+            (path / "Cache").mkdir(exist_ok=True)
+            (path / "Cache" / "data").write_bytes(b"x" * 2000)
+
+        results = {}
+        for level in [ShrinkLevel.LIGHT, ShrinkLevel.MEDIUM,
+                      ShrinkLevel.AGGRESSIVE, ShrinkLevel.MINIMAL]:
+            profile = tmp_path / f"profile_{level.value}"
+            create_full_profile(profile)
+            results[level] = shrink_profile(str(profile), level)
+
+        # Stricter levels should remove more
+        assert len(results[ShrinkLevel.LIGHT]["kept"]) >= len(results[ShrinkLevel.MEDIUM]["kept"])
+        assert len(results[ShrinkLevel.MEDIUM]["kept"]) >= len(results[ShrinkLevel.AGGRESSIVE]["kept"])
+        assert len(results[ShrinkLevel.AGGRESSIVE]["kept"]) >= len(results[ShrinkLevel.MINIMAL]["kept"])
+
+    def test_shrink_with_broken_symlinks(self, tmp_path):
+        """Test shrinking handles broken symlinks."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Cache").mkdir()
+
+        # Create broken symlink
+        broken_link = profile / "Cache" / "broken_link"
+        broken_link.symlink_to("/nonexistent/path/that/does/not/exist")
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert "Cache" in result["removed"]
+        assert not (profile / "Cache").exists()
+
+    def test_shrink_dry_run_reports_would_free(self, tmp_path):
+        """Test dry run accurately reports what would be freed."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data").write_bytes(b"x" * 5000)
+
+        dry_result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE, dry_run=True)
+
+        # Nothing should be removed yet
+        assert (profile / "Cache").exists()
+        assert dry_result["size_after"] is None
+
+        # Actually shrink
+        real_result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        # Dry run should have predicted the freed bytes
+        assert dry_result["bytes_freed"] == real_result["bytes_freed"]
+        assert dry_result["removed"] == real_result["removed"]
+
+
+class TestBrowserProfilerEdgeCases:
+    """Edge cases for BrowserProfiler.shrink() method."""
+
+    def test_profiler_shrink_relative_path(self, tmp_path):
+        """Test profiler.shrink with profile name resolution."""
+        profiler = BrowserProfiler()
+        original_dir = profiler.profiles_dir
+        profiler.profiles_dir = str(tmp_path)
+
+        try:
+            profile = tmp_path / "test_profile"
+            profile.mkdir()
+            (profile / "Preferences").write_text("{}")
+            (profile / "Cache").mkdir()
+            (profile / "Cache" / "data").write_bytes(b"x" * 100)
+
+            result = profiler.shrink("test_profile", ShrinkLevel.AGGRESSIVE)
+            assert "Cache" in result["removed"]
+        finally:
+            profiler.profiles_dir = original_dir
+
+    def test_profiler_shrink_absolute_path(self, tmp_path):
+        """Test profiler.shrink with absolute path."""
+        profiler = BrowserProfiler()
+
+        profile = tmp_path / "absolute_profile"
+        profile.mkdir()
+        (profile / "Preferences").write_text("{}")
+        (profile / "Cache").mkdir()
+
+        result = profiler.shrink(str(profile), ShrinkLevel.AGGRESSIVE)
+        assert "Cache" in result["removed"]
+
+    def test_profiler_shrink_invalid_name(self):
+        """Test profiler.shrink with invalid profile name."""
+        profiler = BrowserProfiler()
+
+        with pytest.raises(ValueError, match="Profile not found"):
+            profiler.shrink("definitely_nonexistent_profile_12345")
+
+
+class TestStressAndCornerCases:
+    """Stress tests and extreme corner cases."""
+
+    def test_shrink_file_instead_of_directory(self, tmp_path):
+        """Test shrinking a file (not directory) raises error."""
+        file_path = tmp_path / "not_a_profile.txt"
+        file_path.write_text("I am a file")
+
+        with pytest.raises(ValueError, match="Profile not found"):
+            shrink_profile(str(file_path), ShrinkLevel.AGGRESSIVE)
+
+    def test_shrink_with_circular_symlinks(self, tmp_path):
+        """Test shrinking handles circular symlinks gracefully."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        cache = profile / "Cache"
+        cache.mkdir()
+
+        # Create circular symlink: Cache/link -> Cache
+        circular = cache / "circular"
+        circular.symlink_to(cache)
+
+        # Should not hang or crash
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+        assert "Cache" in result["removed"]
+
+    def test_shrink_with_very_long_filenames(self, tmp_path):
+        """Test shrinking handles very long filenames."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+
+        # Create file with very long name (near filesystem limit)
+        long_name = "a" * 200  # Most filesystems support 255 chars
+        (profile / long_name).write_bytes(b"x" * 100)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+        assert long_name in result["removed"]
+
+    def test_shrink_profile_only_has_kept_items(self, tmp_path):
+        """Test shrinking profile that only has items to keep."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Local Storage" / "leveldb").mkdir()
+        (profile / "Cookies").write_bytes(b"c" * 100)
+        (profile / "Preferences").write_text("{}")
+        (profile / "IndexedDB").mkdir()
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert result["removed"] == []
+        assert result["bytes_freed"] == 0
+        assert len(result["kept"]) == 4
+
+    def test_shrink_with_files_matching_keep_prefix(self, tmp_path):
+        """Test that files starting with keep patterns are kept."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        # These should be kept (match patterns)
+        (profile / "Local Storage").mkdir()
+        (profile / "Local Storage Extra").mkdir()  # Starts with "Local Storage"
+        (profile / "Cookies").write_bytes(b"c" * 100)
+        (profile / "Cookies-journal").write_bytes(b"j" * 50)
+        (profile / "CookiesBackup").write_bytes(b"b" * 50)  # Starts with "Cookies"
+
+        # This should be removed
+        (profile / "Cache").mkdir()
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert "Local Storage" in result["kept"]
+        assert "Local Storage Extra" in result["kept"]
+        assert "Cookies" in result["kept"]
+        assert "Cookies-journal" in result["kept"]
+        assert "CookiesBackup" in result["kept"]
+        assert "Cache" in result["removed"]
+
+    def test_shrink_calculates_size_correctly_with_nested_dirs(self, tmp_path):
+        """Test size calculation is accurate for nested structures."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+
+        # Create nested cache with known sizes
+        cache = profile / "Cache"
+        cache.mkdir()
+        (cache / "level1").mkdir()
+        (cache / "level1" / "level2").mkdir()
+        (cache / "level1" / "file1.bin").write_bytes(b"x" * 1000)
+        (cache / "level1" / "level2" / "file2.bin").write_bytes(b"y" * 2000)
+        (cache / "file0.bin").write_bytes(b"z" * 500)
+
+        expected_freed = 1000 + 2000 + 500  # Total bytes in Cache
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert result["bytes_freed"] == expected_freed
+
+    def test_shrink_empty_default_subdirectory(self, tmp_path):
+        """Test shrinking when Default/ exists but is empty."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+        (profile / "Default").mkdir()
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert result["removed"] == []
+        assert result["kept"] == []
+        assert result["bytes_freed"] == 0
+
+    def test_shrink_with_both_root_and_default_structure(self, tmp_path):
+        """Test when profile has items at root AND in Default/."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        # Items at root level
+        (profile / "SomeRootFile.txt").write_bytes(b"r" * 100)
+
+        # Items in Default/
+        default = profile / "Default"
+        default.mkdir()
+        (default / "Local Storage").mkdir()
+        (default / "Cache").mkdir()
+        (default / "Cache" / "data").write_bytes(b"x" * 1000)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        # Should shrink inside Default/, ignoring root level
+        assert "Cache" in result["removed"]
+        assert "Local Storage" in result["kept"]
+        # Root file should be untouched
+        assert (profile / "SomeRootFile.txt").exists()
+
+    def test_shrink_minimal_vs_aggressive_indexeddb(self, tmp_path):
+        """Test that MINIMAL removes IndexedDB but AGGRESSIVE keeps it."""
+        def create_profile(path):
+            path.mkdir()
+            (path / "Local Storage").mkdir()
+            (path / "IndexedDB").mkdir()
+            (path / "IndexedDB" / "data").write_bytes(b"i" * 500)
+
+        # Test AGGRESSIVE
+        profile_agg = tmp_path / "aggressive"
+        create_profile(profile_agg)
+        result_agg = shrink_profile(str(profile_agg), ShrinkLevel.AGGRESSIVE)
+        assert "IndexedDB" in result_agg["kept"]
+
+        # Test MINIMAL
+        profile_min = tmp_path / "minimal"
+        create_profile(profile_min)
+        result_min = shrink_profile(str(profile_min), ShrinkLevel.MINIMAL)
+        assert "IndexedDB" in result_min["removed"]
+
+    def test_shrink_handles_oserror_gracefully(self, tmp_path):
+        """Test that OSErrors during iteration don't crash the function."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data").write_bytes(b"x" * 100)
+
+        # This should work without issues
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+        assert result["errors"] == []
+
+    def test_format_size_edge_values(self):
+        """Test _format_size with edge values."""
+        assert _format_size(0) == "0.0 B"
+        assert _format_size(1) == "1.0 B"
+        assert _format_size(1023) == "1023.0 B"
+        assert _format_size(1024) == "1.0 KB"
+        assert _format_size(1024 * 1024 - 1) == "1024.0 KB"
+        assert _format_size(1024 * 1024) == "1.0 MB"
+        assert _format_size(1024 * 1024 * 1024) == "1.0 GB"
+        assert _format_size(1024 * 1024 * 1024 * 1024) == "1.0 TB"
+
+    def test_get_size_with_permission_error(self, tmp_path):
+        """Test _get_size handles permission errors gracefully."""
+        import stat
+
+        profile = tmp_path / "profile"
+        profile.mkdir()
+        restricted = profile / "restricted"
+        restricted.mkdir()
+        (restricted / "file.txt").write_bytes(b"x" * 100)
+
+        # Remove read permission on directory
+        restricted.chmod(stat.S_IWUSR)
+
+        try:
+            # Should not raise, should return partial size
+            size = _get_size(profile)
+            assert size >= 0
+        finally:
+            # Restore permissions
+            restricted.chmod(stat.S_IRWXU)
+
+    def test_shrink_with_cookies_in_network_subdirectory(self, tmp_path):
+        """Test modern Chrome structure with Cookies in Network/."""
+        profile = tmp_path / "profile"
+        profile.mkdir()
+
+        # Chrome 96+ structure
+        network = profile / "Network"
+        network.mkdir()
+        (network / "Cookies").write_bytes(b"c" * 500)
+        (network / "TransportSecurity").write_bytes(b"t" * 100)
+
+        (profile / "Local Storage").mkdir()
+        (profile / "Cache").mkdir()
+        (profile / "Cache" / "data").write_bytes(b"x" * 1000)
+
+        result = shrink_profile(str(profile), ShrinkLevel.AGGRESSIVE)
+
+        assert "Network" in result["kept"]
+        assert "Local Storage" in result["kept"]
+        assert "Cache" in result["removed"]
+        assert (network / "Cookies").exists()
diff --git a/tests/browser/test_profiles.py b/tests/browser/test_profiles.py
new file mode 100644
index 0000000..e49a250
--- /dev/null
+++ b/tests/browser/test_profiles.py
@@ -0,0 +1,180 @@
+"""Test examples for BrowserProfileManager.
+
+These examples demonstrate the functionality of BrowserProfileManager
+and serve as functional tests.
+"""
+
+import asyncio
+import os
+import sys
+import uuid
+import shutil
+
+from crawl4ai import BrowserProfiler
+from crawl4ai.browser_manager import BrowserManager
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+async def test_profile_creation():
+    """Test creating and managing browser profiles."""
+    logger.info("Testing profile creation and management", tag="TEST")
+    
+    profile_manager = BrowserProfiler(logger=logger)
+    
+    try:
+        # List existing profiles
+        profiles = profile_manager.list_profiles()
+        logger.info(f"Found {len(profiles)} existing profiles", tag="TEST")
+        
+        # Generate a unique profile name for testing
+        test_profile_name = f"test-profile-{uuid.uuid4().hex[:8]}"
+        
+        # Create a test profile directory
+        profile_path = os.path.join(profile_manager.profiles_dir, test_profile_name)
+        os.makedirs(os.path.join(profile_path, "Default"), exist_ok=True)
+        
+        # Create a dummy Preferences file to simulate a Chrome profile
+        with open(os.path.join(profile_path, "Default", "Preferences"), "w") as f:
+            f.write("{\"test\": true}")
+        
+        logger.info(f"Created test profile at: {profile_path}", tag="TEST")
+        
+        # Verify the profile is now in the list
+        profiles = profile_manager.list_profiles()
+        profile_found = any(p["name"] == test_profile_name for p in profiles)
+        logger.info(f"Profile found in list: {profile_found}", tag="TEST")
+        
+        # Try to get the profile path
+        retrieved_path = profile_manager.get_profile_path(test_profile_name)
+        path_match = retrieved_path == profile_path
+        logger.info(f"Retrieved correct profile path: {path_match}", tag="TEST")
+        
+        # Delete the profile
+        success = profile_manager.delete_profile(test_profile_name)
+        logger.info(f"Profile deletion successful: {success}", tag="TEST")
+        
+        # Verify it's gone
+        profiles_after = profile_manager.list_profiles()
+        profile_removed = not any(p["name"] == test_profile_name for p in profiles_after)
+        logger.info(f"Profile removed from list: {profile_removed}", tag="TEST")
+        
+        # Clean up just in case
+        if os.path.exists(profile_path):
+            shutil.rmtree(profile_path, ignore_errors=True)
+        
+        return profile_found and path_match and success and profile_removed
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Clean up test directory
+        try:
+            if os.path.exists(profile_path):
+                shutil.rmtree(profile_path, ignore_errors=True)
+        except:
+            pass
+        return False
+
+async def test_profile_with_browser():
+    """Test using a profile with a browser."""
+    logger.info("Testing using a profile with a browser", tag="TEST")
+    
+    profile_manager = BrowserProfiler(logger=logger)
+    test_profile_name = f"test-browser-profile-{uuid.uuid4().hex[:8]}"
+    profile_path = None
+    
+    try:
+        # Create a test profile directory
+        profile_path = os.path.join(profile_manager.profiles_dir, test_profile_name)
+        os.makedirs(os.path.join(profile_path, "Default"), exist_ok=True)
+        
+        # Create a dummy Preferences file to simulate a Chrome profile
+        with open(os.path.join(profile_path, "Default", "Preferences"), "w") as f:
+            f.write("{\"test\": true}")
+        
+        logger.info(f"Created test profile at: {profile_path}", tag="TEST")
+        
+        # Now use this profile with a browser
+        browser_config = BrowserConfig(
+            user_data_dir=profile_path,
+            use_managed_browser=True,
+            use_persistent_context=True,
+            headless=True
+        )
+        
+        manager = BrowserManager(browser_config=browser_config, logger=logger)
+        
+        # Start the browser with the profile
+        await manager.start()
+        logger.info("Browser started with profile", tag="TEST")
+        
+        # Create a page
+        crawler_config = CrawlerRunConfig()
+        page, context = await manager.get_page(crawler_config)
+        
+        # Navigate and set some data to verify profile works
+        await page.goto("https://example.com")
+        await page.evaluate("localStorage.setItem('test_data', 'profile_value')")
+        
+        # Close browser
+        await manager.close()
+        logger.info("First browser session closed", tag="TEST")
+        
+        # Create a new browser with the same profile
+        manager2 = BrowserManager(browser_config=browser_config, logger=logger)
+        await manager2.start()
+        logger.info("Second browser session started with same profile", tag="TEST")
+        
+        # Get a page and check if the data persists
+        page2, context2 = await manager2.get_page(crawler_config)
+        await page2.goto("https://example.com")
+        data = await page2.evaluate("localStorage.getItem('test_data')")
+        
+        # Verify data persisted
+        data_persisted = data == "profile_value"
+        logger.info(f"Data persisted across sessions: {data_persisted}", tag="TEST")
+        
+        # Clean up
+        await manager2.close()
+        logger.info("Second browser session closed", tag="TEST")
+        
+        # Delete the test profile
+        success = profile_manager.delete_profile(test_profile_name)
+        logger.info(f"Test profile deleted: {success}", tag="TEST")
+        
+        return data_persisted and success
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}", tag="TEST")
+        # Clean up
+        try:
+            if profile_path and os.path.exists(profile_path):
+                shutil.rmtree(profile_path, ignore_errors=True)
+        except:
+            pass
+        return False
+
+async def run_tests():
+    """Run all tests sequentially."""
+    results = []
+    
+    results.append(await test_profile_creation())
+    results.append(await test_profile_with_browser())
+    
+    # Print summary
+    total = len(results)
+    passed = sum(results)
+    logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
+    
+    if passed == total:
+        logger.success("All tests passed!", tag="SUMMARY")
+    else:
+        logger.error(f"{total - passed} tests failed", tag="SUMMARY")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
diff --git a/tests/browser/test_repro_1640.py b/tests/browser/test_repro_1640.py
new file mode 100644
index 0000000..ee33381
--- /dev/null
+++ b/tests/browser/test_repro_1640.py
@@ -0,0 +1,424 @@
+"""
+Regression tests for PR #1640 — memory leak / hang under high concurrency
+with max_pages_before_recycle enabled.
+
+Tests three bugs that were fixed:
+
+Bug 1: Race condition — release_page_with_context() runs BEFORE
+       _maybe_bump_browser_version() adds the sig to _pending_cleanup.
+       FIX: Don't add refcount-0 sigs to pending; clean them up immediately.
+
+Bug 2: The finally block in _crawl_web can fail before calling
+       release_page_with_context(), leaking the refcount permanently.
+       FIX: Call release_page_with_context() FIRST in the finally block.
+
+Bug 3: Accumulated pending_cleanup entries hit _max_pending_browsers cap,
+       blocking ALL get_page() calls → system-wide deadlock.
+       FIX: 30s timeout on safety cap wait + force-clean stuck entries.
+
+Exit code 0 = all tests pass. Exit code 1 = regression found.
+"""
+
+import asyncio
+import sys
+import os
+import time
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.browser_manager import BrowserManager
+
+PASS = 0
+FAIL = 0
+
+
+def check(name, condition):
+    global PASS, FAIL
+    if condition:
+        PASS += 1
+        print(f"  PASS: {name}")
+    else:
+        FAIL += 1
+        print(f"  FAIL: {name}")
+
+
+async def test_bug1_multi_config_race():
+    """
+    Bug 1 fix: idle sigs (refcount=0) must NOT be added to _pending_cleanup.
+    They should be cleaned up immediately during the version bump.
+    """
+    print("\n" + "="*70)
+    print("TEST: Bug 1 — idle sig must not get stuck in _pending_cleanup")
+    print("="*70)
+
+    config = BrowserConfig(
+        headless=True,
+        extra_args=['--no-sandbox', '--disable-gpu'],
+        max_pages_before_recycle=3,
+    )
+    bm = BrowserManager(config)
+    await bm.start()
+
+    try:
+        config_a = CrawlerRunConfig(magic=True, cache_mode="bypass")
+        config_b = CrawlerRunConfig(magic=False, cache_mode="bypass")
+
+        # Use config A, then release → refcount 0
+        page_a, _ = await bm.get_page(config_a)
+        sig_a = bm._page_to_sig.get(page_a)
+        await bm.release_page_with_context(page_a)
+        await page_a.close()
+
+        print(f"  sig_a refcount after release: {bm._context_refcounts.get(sig_a)}")
+
+        # Use config B twice → pages_served hits threshold → version bump
+        page_b1, _ = await bm.get_page(config_b)
+        page_b2, _ = await bm.get_page(config_b)
+        sig_b = bm._page_to_sig.get(page_b1)
+
+        # At this point the version should have bumped (3 pages served >= threshold 3)
+        print(f"  _browser_version: {bm._browser_version}")
+        print(f"  _pending_cleanup sigs: {list(bm._pending_cleanup.keys())}")
+
+        # sig_a (refcount=0) must NOT be in _pending_cleanup
+        check("sig_a NOT in _pending_cleanup",
+              sig_a not in bm._pending_cleanup)
+
+        # sig_a should have been cleaned up from _context_refcounts
+        check("sig_a cleaned from _context_refcounts",
+              sig_a not in bm._context_refcounts)
+
+        # sig_b (refcount>0) SHOULD be in _pending_cleanup (it will drain naturally)
+        check("sig_b IS in _pending_cleanup (active, will drain)",
+              sig_b in bm._pending_cleanup)
+
+        # Release B pages → sig_b drains → cleaned up
+        await bm.release_page_with_context(page_b1)
+        await page_b1.close()
+        await bm.release_page_with_context(page_b2)
+        await page_b2.close()
+
+        check("sig_b cleaned after release",
+              sig_b not in bm._pending_cleanup)
+
+        check("_pending_cleanup is empty",
+              len(bm._pending_cleanup) == 0)
+
+    finally:
+        await bm.close()
+
+
+async def test_bug2_release_always_called():
+    """
+    Bug 2 fix: release_page_with_context() must be called even when
+    the browser is in a bad state.
+
+    The fix moves release_page_with_context() to the FIRST line of
+    the finally block in _crawl_web, wrapped in try/except.
+    Here we verify that release_page_with_context itself works even
+    after browser crash, and that the fixed finally block pattern
+    always decrements the refcount.
+    """
+    print("\n" + "="*70)
+    print("TEST: Bug 2 — release_page_with_context must work after browser crash")
+    print("="*70)
+
+    config = BrowserConfig(
+        headless=True,
+        extra_args=['--no-sandbox', '--disable-gpu'],
+        max_pages_before_recycle=5,
+    )
+    bm = BrowserManager(config)
+    await bm.start()
+
+    try:
+        crawl_config = CrawlerRunConfig(magic=True, cache_mode="bypass")
+
+        page, ctx = await bm.get_page(crawl_config)
+        sig = bm._page_to_sig.get(page)
+        print(f"  sig refcount before crash: {bm._context_refcounts.get(sig)}")
+
+        check("refcount is 1 before crash",
+              bm._context_refcounts.get(sig) == 1)
+
+        # Simulate browser crash
+        if bm.browser:
+            await bm.browser.close()
+            bm.browser = None
+
+        # The FIX: call release_page_with_context even after crash
+        # (simulating what the fixed finally block does)
+        try:
+            await bm.release_page_with_context(page)
+        except Exception:
+            pass
+
+        refcount_after = bm._context_refcounts.get(sig, 0)
+        print(f"  sig refcount after crash + release: {refcount_after}")
+
+        check("refcount decremented to 0 after crash + release",
+              refcount_after == 0)
+
+        check("page removed from _page_to_sig",
+              page not in bm._page_to_sig)
+
+    finally:
+        bm.browser = None
+        bm.contexts_by_config.clear()
+        bm._context_refcounts.clear()
+        bm._context_last_used.clear()
+        bm._page_to_sig.clear()
+        if bm.playwright:
+            await bm.playwright.stop()
+
+
+async def test_bug3_safety_cap_timeout():
+    """
+    Bug 3 fix: the safety cap wait must have a timeout.
+    When stuck entries accumulate, the timeout fires and force-cleans
+    entries with refcount 0, preventing permanent deadlock.
+    """
+    print("\n" + "="*70)
+    print("TEST: Bug 3 — safety cap wait must not block forever")
+    print("="*70)
+
+    config = BrowserConfig(
+        headless=True,
+        extra_args=['--no-sandbox', '--disable-gpu'],
+        max_pages_before_recycle=2,
+    )
+    bm = BrowserManager(config)
+    await bm.start()
+
+    try:
+        crawl_config = CrawlerRunConfig(magic=True, cache_mode="bypass")
+
+        # Inject stuck entries WITH refcount 0 (simulating leaked refcounts
+        # that were later force-decremented or never properly tracked)
+        print(f"  Safety cap: {bm._max_pending_browsers}")
+        for i in range(bm._max_pending_browsers):
+            fake_sig = f"stuck_sig_{i}"
+            bm._pending_cleanup[fake_sig] = {"version": i, "done": asyncio.Event()}
+            # refcount 0 = stuck (no future release will clean these up)
+            bm._context_refcounts[fake_sig] = 0
+
+        print(f"  Injected {len(bm._pending_cleanup)} stuck entries (refcount=0)")
+
+        bm._pages_served = bm.config.max_pages_before_recycle
+
+        # The fix: get_page should NOT block forever.
+        # The 30s timeout will fire, force-clean stuck entries, and proceed.
+        # We use a 35s test timeout to allow the 30s internal timeout to fire.
+        print(f"  Calling get_page() — should unblock after ~30s timeout...")
+        start = time.monotonic()
+        try:
+            page, ctx = await asyncio.wait_for(
+                bm.get_page(crawl_config),
+                timeout=35.0
+            )
+            elapsed = time.monotonic() - start
+            print(f"  get_page() returned after {elapsed:.1f}s")
+
+            check("get_page() did NOT deadlock (returned within 35s)", True)
+            check("stuck entries were force-cleaned",
+                  len(bm._pending_cleanup) < bm._max_pending_browsers)
+
+            await bm.release_page_with_context(page)
+            await page.close()
+
+        except asyncio.TimeoutError:
+            elapsed = time.monotonic() - start
+            print(f"  get_page() STILL blocked after {elapsed:.1f}s")
+            check("get_page() did NOT deadlock", False)
+
+    finally:
+        bm._pending_cleanup.clear()
+        bm._context_refcounts.clear()
+        await bm.close()
+
+
+async def test_real_concurrent_crawl():
+    """
+    Integration test: run many concurrent crawls with recycling
+    and verify no stuck entries or deadlocks.
+    """
+    print("\n" + "="*70)
+    print("TEST: Real concurrent crawls with recycling")
+    print("="*70)
+
+    config = BrowserConfig(
+        headless=True,
+        extra_args=['--no-sandbox', '--disable-gpu'],
+        max_pages_before_recycle=10,
+    )
+    bm = BrowserManager(config)
+    await bm.start()
+
+    TOTAL = 80
+    CONCURRENT = 8
+    completed = 0
+    errors = 0
+
+    sem = asyncio.Semaphore(CONCURRENT)
+
+    async def do_crawl(i):
+        nonlocal completed, errors
+        async with sem:
+            try:
+                crawl_config = CrawlerRunConfig(magic=True, cache_mode="bypass")
+                page, ctx = await asyncio.wait_for(
+                    bm.get_page(crawl_config),
+                    timeout=30.0
+                )
+
+                try:
+                    await page.goto("https://example.com", timeout=15000)
+                except Exception:
+                    pass
+
+                # Use the FIXED finally pattern: release first, then close
+                try:
+                    await bm.release_page_with_context(page)
+                except Exception:
+                    pass
+                try:
+                    await page.close()
+                except Exception:
+                    pass
+
+                completed += 1
+                if completed % 20 == 0:
+                    print(f"  [{completed}/{TOTAL}] version={bm._browser_version} "
+                          f"pending={len(bm._pending_cleanup)} "
+                          f"pages_served={bm._pages_served}")
+
+            except asyncio.TimeoutError:
+                errors += 1
+                print(f"  [{i}] TIMEOUT in get_page()!")
+            except Exception as e:
+                errors += 1
+                if errors <= 3:
+                    print(f"  [{i}] Error: {e}")
+
+    start = time.monotonic()
+    tasks = [asyncio.create_task(do_crawl(i)) for i in range(TOTAL)]
+    await asyncio.gather(*tasks)
+    elapsed = time.monotonic() - start
+
+    print(f"\n  Results: {completed}/{TOTAL} completed, {errors} errors, {elapsed:.1f}s")
+
+    stuck = [s for s in bm._pending_cleanup if bm._context_refcounts.get(s, 0) == 0]
+
+    check(f"all {TOTAL} crawls completed", completed == TOTAL)
+    check("no errors", errors == 0)
+    check("no stuck entries in _pending_cleanup", len(stuck) == 0)
+    check("no timeouts (no deadlock)", errors == 0)
+
+    await bm.close()
+
+
+async def test_multi_config_concurrent():
+    """
+    Integration test: concurrent crawls with DIFFERENT configs to
+    exercise the multi-sig path that triggered Bug 1.
+    """
+    print("\n" + "="*70)
+    print("TEST: Multi-config concurrent crawls")
+    print("="*70)
+
+    config = BrowserConfig(
+        headless=True,
+        extra_args=['--no-sandbox', '--disable-gpu'],
+        max_pages_before_recycle=5,
+    )
+    bm = BrowserManager(config)
+    await bm.start()
+
+    TOTAL = 40
+    CONCURRENT = 6
+    completed = 0
+    errors = 0
+
+    sem = asyncio.Semaphore(CONCURRENT)
+    configs = [
+        CrawlerRunConfig(magic=True, cache_mode="bypass"),
+        CrawlerRunConfig(magic=False, cache_mode="bypass"),
+        CrawlerRunConfig(magic=True, simulate_user=True, cache_mode="bypass"),
+    ]
+
+    async def do_crawl(i):
+        nonlocal completed, errors
+        async with sem:
+            try:
+                crawl_config = configs[i % len(configs)]
+                page, ctx = await asyncio.wait_for(
+                    bm.get_page(crawl_config),
+                    timeout=30.0
+                )
+
+                try:
+                    await page.goto("https://example.com", timeout=15000)
+                except Exception:
+                    pass
+
+                try:
+                    await bm.release_page_with_context(page)
+                except Exception:
+                    pass
+                try:
+                    await page.close()
+                except Exception:
+                    pass
+
+                completed += 1
+
+            except asyncio.TimeoutError:
+                errors += 1
+                print(f"  [{i}] TIMEOUT!")
+                print(f"    pending={len(bm._pending_cleanup)}")
+            except Exception as e:
+                errors += 1
+                if errors <= 3:
+                    print(f"  [{i}] Error: {e}")
+
+    start = time.monotonic()
+    tasks = [asyncio.create_task(do_crawl(i)) for i in range(TOTAL)]
+    await asyncio.gather(*tasks)
+    elapsed = time.monotonic() - start
+
+    stuck = [s for s in bm._pending_cleanup if bm._context_refcounts.get(s, 0) == 0]
+
+    print(f"\n  Results: {completed}/{TOTAL}, {errors} errors, {elapsed:.1f}s")
+    print(f"  Final: version={bm._browser_version} pending={len(bm._pending_cleanup)} stuck={len(stuck)}")
+
+    check(f"all {TOTAL} multi-config crawls completed", completed == TOTAL)
+    check("no stuck entries", len(stuck) == 0)
+    check("no timeouts", errors == 0)
+
+    await bm.close()
+
+
+async def main():
+    print("="*70)
+    print("PR #1640 Regression Tests")
+    print("="*70)
+
+    await test_bug2_release_always_called()
+    await test_bug1_multi_config_race()
+    await test_bug3_safety_cap_timeout()
+    await test_real_concurrent_crawl()
+    await test_multi_config_concurrent()
+
+    print("\n" + "="*70)
+    if FAIL == 0:
+        print(f"ALL {PASS} CHECKS PASSED")
+    else:
+        print(f"FAILED: {FAIL} checks failed, {PASS} passed")
+    print("="*70)
+
+    sys.exit(1 if FAIL > 0 else 0)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/browser/test_resource_filtering.py b/tests/browser/test_resource_filtering.py
new file mode 100644
index 0000000..552aadd
--- /dev/null
+++ b/tests/browser/test_resource_filtering.py
@@ -0,0 +1,178 @@
+"""E2E tests for avoid_ads / avoid_css resource filtering.
+
+These tests launch real browsers and crawl real websites to verify
+that route-based resource blocking actually works.
+
+Domains used:
+  - books.toscrape.com  (CSS-heavy practice site, designed for scraping)
+  - quotes.toscrape.com (simple practice site)
+  - httpbin.org/html    (static HTML, no trackers)
+  - en.wikipedia.org    (real site with analytics)
+"""
+
+import pytest
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+
+# ---------------------------------------------------------------------------
+# Basic success tests — flags should not break crawling
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_crawl_with_avoid_css_succeeds():
+    """Crawl books.toscrape.com with avoid_css=True — page should load fine."""
+    browser_config = BrowserConfig(headless=True, avoid_css=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://books.toscrape.com",
+            config=CrawlerRunConfig(cache_mode="bypass"),
+        )
+        assert result.success, f"Crawl failed: {result.error_message}"
+        assert len(result.html) > 500, "Page HTML is suspiciously short"
+
+
+@pytest.mark.asyncio
+async def test_crawl_with_avoid_ads_succeeds():
+    """Crawl Wikipedia with avoid_ads=True — content should be intact."""
+    browser_config = BrowserConfig(headless=True, avoid_ads=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://en.wikipedia.org/wiki/Web_scraping",
+            config=CrawlerRunConfig(cache_mode="bypass"),
+        )
+        assert result.success, f"Crawl failed: {result.error_message}"
+        # Wikipedia article content must be present
+        html_lower = result.html.lower()
+        assert "web scraping" in html_lower, "Wikipedia content missing"
+
+
+@pytest.mark.asyncio
+async def test_crawl_with_both_flags_succeeds():
+    """Both avoid_css and avoid_ads enabled simultaneously."""
+    browser_config = BrowserConfig(headless=True, avoid_css=True, avoid_ads=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://quotes.toscrape.com",
+            config=CrawlerRunConfig(cache_mode="bypass"),
+        )
+        assert result.success, f"Crawl failed: {result.error_message}"
+        html_lower = result.html.lower()
+        assert "quote" in html_lower or "toscrape" in html_lower
+
+
+@pytest.mark.asyncio
+async def test_avoid_ads_does_not_block_page_content():
+    """avoid_ads must not interfere with first-party page content."""
+    browser_config = BrowserConfig(headless=True, avoid_ads=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://httpbin.org/html",
+            config=CrawlerRunConfig(cache_mode="bypass"),
+        )
+        assert result.success, f"Crawl failed: {result.error_message}"
+        # httpbin.org/html serves a Moby Dick excerpt
+        assert "Herman Melville" in result.html, "First-party content missing"
+
+
+# ---------------------------------------------------------------------------
+# Network-level verification — prove routes actually block requests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_without_flags_css_loads_normally():
+    """Baseline: without avoid_css, CSS responses should appear in network log."""
+    browser_config = BrowserConfig(headless=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://books.toscrape.com",
+            config=CrawlerRunConfig(
+                cache_mode="bypass",
+                capture_network_requests=True,
+            ),
+        )
+        assert result.success
+        assert result.network_requests is not None, "Network requests not captured"
+
+        # There should be successful CSS responses
+        css_responses = [
+            r
+            for r in result.network_requests
+            if r.get("event_type") == "response" and ".css" in r.get("url", "")
+        ]
+        assert (
+            len(css_responses) > 0
+        ), "CSS should load normally without avoid_css flag"
+
+
+@pytest.mark.asyncio
+async def test_avoid_css_blocks_css_requests():
+    """With avoid_css=True, CSS requests must be aborted (no successful responses)."""
+    browser_config = BrowserConfig(headless=True, avoid_css=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://books.toscrape.com",
+            config=CrawlerRunConfig(
+                cache_mode="bypass",
+                capture_network_requests=True,
+            ),
+        )
+        assert result.success
+        assert result.network_requests is not None, "Network requests not captured"
+
+        # No CSS should have gotten a successful response
+        css_responses = [
+            r
+            for r in result.network_requests
+            if r.get("event_type") == "response" and ".css" in r.get("url", "")
+        ]
+        assert (
+            len(css_responses) == 0
+        ), f"CSS responses should be blocked, but found: {[r['url'] for r in css_responses]}"
+
+        # There SHOULD be request_failed events for CSS (proves blocking happened)
+        css_failures = [
+            r
+            for r in result.network_requests
+            if r.get("event_type") == "request_failed"
+            and ".css" in r.get("url", "")
+        ]
+        assert (
+            len(css_failures) > 0
+        ), "Expected request_failed events for blocked CSS files"
+
+
+@pytest.mark.asyncio
+async def test_avoid_css_with_text_mode_combines():
+    """Both avoid_css and text_mode should combine their blocking rules."""
+    browser_config = BrowserConfig(
+        headless=True, avoid_css=True, text_mode=True
+    )
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://books.toscrape.com",
+            config=CrawlerRunConfig(
+                cache_mode="bypass",
+                capture_network_requests=True,
+            ),
+        )
+        assert result.success
+        assert result.network_requests is not None
+
+        successful = [
+            r for r in result.network_requests if r.get("event_type") == "response"
+        ]
+
+        # CSS should be blocked (via avoid_css)
+        css_hits = [r for r in successful if ".css" in r.get("url", "")]
+        assert len(css_hits) == 0, "CSS should be blocked by avoid_css"
+
+        # Images should be blocked (via text_mode)
+        img_exts = (".jpg", ".jpeg", ".png", ".gif", ".webp")
+        img_hits = [
+            r
+            for r in successful
+            if any(r.get("url", "").lower().endswith(ext) for ext in img_exts)
+        ]
+        assert len(img_hits) == 0, "Images should be blocked by text_mode"
diff --git a/tests/cache_validation/__init__.py b/tests/cache_validation/__init__.py
new file mode 100644
index 0000000..c65c085
--- /dev/null
+++ b/tests/cache_validation/__init__.py
@@ -0,0 +1 @@
+# Cache validation test suite
diff --git a/tests/cache_validation/conftest.py b/tests/cache_validation/conftest.py
new file mode 100644
index 0000000..0fd6875
--- /dev/null
+++ b/tests/cache_validation/conftest.py
@@ -0,0 +1,40 @@
+"""Pytest fixtures for cache validation tests."""
+
+import pytest
+
+
+def pytest_configure(config):
+    """Register custom markers."""
+    config.addinivalue_line(
+        "markers", "integration: marks tests as integration tests (may require network)"
+    )
+
+
+@pytest.fixture
+def sample_head_html():
+    """Sample HTML head section for testing."""
+    return '''
+    
+        
+        Test Page Title
+        
+        
+        
+        
+        
+        
+        
+    
+    '''
+
+
+@pytest.fixture
+def minimal_head_html():
+    """Minimal head with just a title."""
+    return 'Minimal'
+
+
+@pytest.fixture
+def empty_head_html():
+    """Empty head section."""
+    return ''
diff --git a/tests/cache_validation/test_end_to_end.py b/tests/cache_validation/test_end_to_end.py
new file mode 100644
index 0000000..0238393
--- /dev/null
+++ b/tests/cache_validation/test_end_to_end.py
@@ -0,0 +1,449 @@
+"""
+End-to-end tests for Smart Cache validation.
+
+Tests the full flow:
+1. Fresh crawl (browser launch) - SLOW
+2. Cached crawl without validation (check_cache_freshness=False) - FAST
+3. Cached crawl with validation (check_cache_freshness=True) - FAST (304/fingerprint)
+
+Verifies all layers:
+- Database storage of etag, last_modified, head_fingerprint, cached_at
+- Cache validation logic
+- HTTP conditional requests (304 Not Modified)
+- Performance improvements
+"""
+
+import pytest
+import time
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.async_database import async_db_manager
+
+
+class TestEndToEndCacheValidation:
+    """End-to-end tests for the complete cache validation flow."""
+
+    @pytest.mark.asyncio
+    async def test_full_cache_flow_docs_python(self):
+        """
+        Test complete cache flow with docs.python.org:
+        1. Fresh crawl (slow - browser) - using BYPASS to force fresh
+        2. Cache hit without validation (fast)
+        3. Cache hit with validation (fast - 304)
+        """
+        url = "https://docs.python.org/3/"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+
+        # ========== CRAWL 1: Fresh crawl (force with WRITE_ONLY to skip cache read) ==========
+        config1 = CrawlerRunConfig(
+            cache_mode=CacheMode.WRITE_ONLY,  # Skip reading, write new data
+            check_cache_freshness=False,
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            start1 = time.perf_counter()
+            result1 = await crawler.arun(url, config=config1)
+            time1 = time.perf_counter() - start1
+
+        assert result1.success, f"First crawl failed: {result1.error_message}"
+        # WRITE_ONLY means we did a fresh crawl and wrote to cache
+        assert result1.cache_status == "miss", f"Expected 'miss', got '{result1.cache_status}'"
+
+        print(f"\n[CRAWL 1] Fresh crawl: {time1:.2f}s (cache_status: {result1.cache_status})")
+
+        # Verify data is stored in database
+        metadata = await async_db_manager.aget_cache_metadata(url)
+        assert metadata is not None, "Metadata should be stored in database"
+        assert metadata.get("etag") or metadata.get("last_modified"), "Should have ETag or Last-Modified"
+        print(f"  - Stored ETag: {metadata.get('etag', 'N/A')[:30]}...")
+        print(f"  - Stored Last-Modified: {metadata.get('last_modified', 'N/A')}")
+        print(f"  - Stored head_fingerprint: {metadata.get('head_fingerprint', 'N/A')}")
+        print(f"  - Stored cached_at: {metadata.get('cached_at', 'N/A')}")
+
+        # ========== CRAWL 2: Cache hit WITHOUT validation ==========
+        config2 = CrawlerRunConfig(
+            cache_mode=CacheMode.ENABLED,
+            check_cache_freshness=False,  # Skip validation - pure cache hit
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            start2 = time.perf_counter()
+            result2 = await crawler.arun(url, config=config2)
+            time2 = time.perf_counter() - start2
+
+        assert result2.success, f"Second crawl failed: {result2.error_message}"
+        assert result2.cache_status == "hit", f"Expected 'hit', got '{result2.cache_status}'"
+
+        print(f"\n[CRAWL 2] Cache hit (no validation): {time2:.2f}s (cache_status: {result2.cache_status})")
+        print(f"  - Speedup: {time1/time2:.1f}x faster than fresh crawl")
+
+        # Should be MUCH faster - no browser, no HTTP request
+        assert time2 < time1 / 2, f"Cache hit should be at least 2x faster (was {time1/time2:.1f}x)"
+
+        # ========== CRAWL 3: Cache hit WITH validation (304) ==========
+        config3 = CrawlerRunConfig(
+            cache_mode=CacheMode.ENABLED,
+            check_cache_freshness=True,  # Validate cache freshness
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            start3 = time.perf_counter()
+            result3 = await crawler.arun(url, config=config3)
+            time3 = time.perf_counter() - start3
+
+        assert result3.success, f"Third crawl failed: {result3.error_message}"
+        # Should be "hit_validated" (304) or "hit_fallback" (error during validation)
+        assert result3.cache_status in ["hit_validated", "hit_fallback"], \
+            f"Expected validated cache hit, got '{result3.cache_status}'"
+
+        print(f"\n[CRAWL 3] Cache hit (with validation): {time3:.2f}s (cache_status: {result3.cache_status})")
+        print(f"  - Speedup: {time1/time3:.1f}x faster than fresh crawl")
+
+        # Should still be fast - just a HEAD request, no browser
+        assert time3 < time1 / 2, f"Validated cache hit should be faster than fresh crawl"
+
+        # ========== SUMMARY ==========
+        print(f"\n{'='*60}")
+        print(f"PERFORMANCE SUMMARY for {url}")
+        print(f"{'='*60}")
+        print(f"  Fresh crawl (browser):        {time1:.2f}s")
+        print(f"  Cache hit (no validation):    {time2:.2f}s ({time1/time2:.1f}x faster)")
+        print(f"  Cache hit (with validation):  {time3:.2f}s ({time1/time3:.1f}x faster)")
+        print(f"{'='*60}")
+
+    @pytest.mark.asyncio
+    async def test_full_cache_flow_crawl4ai_docs(self):
+        """Test with docs.crawl4ai.com."""
+        url = "https://docs.crawl4ai.com/"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+
+        # Fresh crawl - use WRITE_ONLY to ensure we get fresh data
+        config1 = CrawlerRunConfig(cache_mode=CacheMode.WRITE_ONLY, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            start1 = time.perf_counter()
+            result1 = await crawler.arun(url, config=config1)
+            time1 = time.perf_counter() - start1
+
+        assert result1.success
+        assert result1.cache_status == "miss"
+        print(f"\n[docs.crawl4ai.com] Fresh: {time1:.2f}s")
+
+        # Cache hit with validation
+        config2 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            start2 = time.perf_counter()
+            result2 = await crawler.arun(url, config=config2)
+            time2 = time.perf_counter() - start2
+
+        assert result2.success
+        assert result2.cache_status in ["hit_validated", "hit_fallback"]
+        print(f"[docs.crawl4ai.com] Validated: {time2:.2f}s ({time1/time2:.1f}x faster)")
+
+    @pytest.mark.asyncio
+    async def test_verify_database_storage(self):
+        """Verify all validation metadata is properly stored in database."""
+        url = "https://docs.python.org/3/library/asyncio.html"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+        config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=False)
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url, config=config)
+
+        assert result.success
+
+        # Verify all fields in database
+        metadata = await async_db_manager.aget_cache_metadata(url)
+
+        assert metadata is not None, "Metadata must be stored"
+        assert "url" in metadata
+        assert "etag" in metadata
+        assert "last_modified" in metadata
+        assert "head_fingerprint" in metadata
+        assert "cached_at" in metadata
+        assert "response_headers" in metadata
+
+        print(f"\nDatabase storage verification for {url}:")
+        print(f"  - etag: {metadata['etag'][:40] if metadata['etag'] else 'None'}...")
+        print(f"  - last_modified: {metadata['last_modified']}")
+        print(f"  - head_fingerprint: {metadata['head_fingerprint']}")
+        print(f"  - cached_at: {metadata['cached_at']}")
+        print(f"  - response_headers keys: {list(metadata['response_headers'].keys())[:5]}...")
+
+        # At least one validation field should be populated
+        has_validation_data = (
+            metadata["etag"] or
+            metadata["last_modified"] or
+            metadata["head_fingerprint"]
+        )
+        assert has_validation_data, "Should have at least one validation field"
+
+    @pytest.mark.asyncio
+    async def test_head_fingerprint_stored_and_used(self):
+        """Verify head fingerprint is computed, stored, and used for validation."""
+        url = "https://example.com/"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+
+        # Fresh crawl
+        config1 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result1 = await crawler.arun(url, config=config1)
+
+        assert result1.success
+        assert result1.head_fingerprint, "head_fingerprint should be set on CrawlResult"
+
+        # Verify in database
+        metadata = await async_db_manager.aget_cache_metadata(url)
+        assert metadata["head_fingerprint"], "head_fingerprint should be stored in database"
+        assert metadata["head_fingerprint"] == result1.head_fingerprint
+
+        print(f"\nHead fingerprint for {url}:")
+        print(f"  - CrawlResult.head_fingerprint: {result1.head_fingerprint}")
+        print(f"  - Database head_fingerprint: {metadata['head_fingerprint']}")
+
+        # Validate using fingerprint
+        config2 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result2 = await crawler.arun(url, config=config2)
+
+        assert result2.success
+        assert result2.cache_status in ["hit_validated", "hit_fallback"]
+        print(f"  - Validation result: {result2.cache_status}")
+
+
+class TestCacheValidationPerformance:
+    """Performance benchmarks for cache validation."""
+
+    @pytest.mark.asyncio
+    async def test_multiple_urls_performance(self):
+        """Test cache performance across multiple URLs."""
+        urls = [
+            "https://docs.python.org/3/",
+            "https://docs.python.org/3/library/asyncio.html",
+            "https://en.wikipedia.org/wiki/Python_(programming_language)",
+        ]
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+        fresh_times = []
+        cached_times = []
+
+        print(f"\n{'='*70}")
+        print("MULTI-URL PERFORMANCE TEST")
+        print(f"{'='*70}")
+
+        # Fresh crawls - use WRITE_ONLY to force fresh crawl
+        for url in urls:
+            config = CrawlerRunConfig(cache_mode=CacheMode.WRITE_ONLY, check_cache_freshness=False)
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                start = time.perf_counter()
+                result = await crawler.arun(url, config=config)
+                elapsed = time.perf_counter() - start
+                fresh_times.append(elapsed)
+                print(f"Fresh:  {url[:50]:50} {elapsed:.2f}s ({result.cache_status})")
+
+        # Cached crawls with validation
+        for url in urls:
+            config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=True)
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                start = time.perf_counter()
+                result = await crawler.arun(url, config=config)
+                elapsed = time.perf_counter() - start
+                cached_times.append(elapsed)
+                print(f"Cached: {url[:50]:50} {elapsed:.2f}s ({result.cache_status})")
+
+        avg_fresh = sum(fresh_times) / len(fresh_times)
+        avg_cached = sum(cached_times) / len(cached_times)
+        total_fresh = sum(fresh_times)
+        total_cached = sum(cached_times)
+
+        print(f"\n{'='*70}")
+        print(f"RESULTS:")
+        print(f"  Total fresh crawl time:  {total_fresh:.2f}s")
+        print(f"  Total cached time:       {total_cached:.2f}s")
+        print(f"  Average speedup:         {avg_fresh/avg_cached:.1f}x")
+        print(f"  Time saved:              {total_fresh - total_cached:.2f}s")
+        print(f"{'='*70}")
+
+        # Cached should be significantly faster
+        assert avg_cached < avg_fresh / 2, "Cached crawls should be at least 2x faster"
+
+    @pytest.mark.asyncio
+    async def test_repeated_access_same_url(self):
+        """Test repeated access to the same URL shows consistent cache hits."""
+        url = "https://docs.python.org/3/"
+        num_accesses = 5
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+
+        print(f"\n{'='*60}")
+        print(f"REPEATED ACCESS TEST: {url}")
+        print(f"{'='*60}")
+
+        # First access - fresh crawl
+        config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            start = time.perf_counter()
+            result = await crawler.arun(url, config=config)
+            fresh_time = time.perf_counter() - start
+        print(f"Access 1 (fresh):     {fresh_time:.2f}s - {result.cache_status}")
+
+        # Repeated accesses - should all be cache hits
+        cached_times = []
+        for i in range(2, num_accesses + 1):
+            config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=True)
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                start = time.perf_counter()
+                result = await crawler.arun(url, config=config)
+                elapsed = time.perf_counter() - start
+                cached_times.append(elapsed)
+            print(f"Access {i} (cached):    {elapsed:.2f}s - {result.cache_status}")
+            assert result.cache_status in ["hit", "hit_validated", "hit_fallback"]
+
+        avg_cached = sum(cached_times) / len(cached_times)
+        print(f"\nAverage cached time: {avg_cached:.2f}s")
+        print(f"Speedup over fresh:  {fresh_time/avg_cached:.1f}x")
+
+
+class TestCacheValidationModes:
+    """Test different cache modes and their behavior."""
+
+    @pytest.mark.asyncio
+    async def test_cache_bypass_always_fresh(self):
+        """CacheMode.BYPASS should always do fresh crawl."""
+        # Use a unique URL path to avoid cache from other tests
+        url = "https://example.com/test-bypass"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+
+        # First crawl with WRITE_ONLY to populate cache (always fresh)
+        config1 = CrawlerRunConfig(cache_mode=CacheMode.WRITE_ONLY, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result1 = await crawler.arun(url, config=config1)
+        assert result1.cache_status == "miss"
+
+        # Second crawl with BYPASS - should NOT use cache
+        config2 = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result2 = await crawler.arun(url, config=config2)
+
+        # BYPASS mode means no cache interaction
+        assert result2.cache_status is None or result2.cache_status == "miss"
+        print(f"\nCacheMode.BYPASS result: {result2.cache_status}")
+
+    @pytest.mark.asyncio
+    async def test_validation_disabled_uses_cache_directly(self):
+        """With check_cache_freshness=False, should use cache without HTTP validation."""
+        url = "https://docs.python.org/3/tutorial/"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+
+        # Fresh crawl - use WRITE_ONLY to force fresh
+        config1 = CrawlerRunConfig(cache_mode=CacheMode.WRITE_ONLY, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result1 = await crawler.arun(url, config=config1)
+        assert result1.cache_status == "miss"
+
+        # Cached with validation DISABLED - should be "hit" (not "hit_validated")
+        config2 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            start = time.perf_counter()
+            result2 = await crawler.arun(url, config=config2)
+            elapsed = time.perf_counter() - start
+
+        assert result2.cache_status == "hit", f"Expected 'hit', got '{result2.cache_status}'"
+        print(f"\nValidation disabled: {elapsed:.3f}s (cache_status: {result2.cache_status})")
+
+        # Should be very fast - no HTTP request at all
+        assert elapsed < 1.0, "Cache hit without validation should be < 1 second"
+
+    @pytest.mark.asyncio
+    async def test_validation_enabled_checks_freshness(self):
+        """With check_cache_freshness=True, should validate before using cache."""
+        url = "https://docs.python.org/3/reference/"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+
+        # Fresh crawl
+        config1 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result1 = await crawler.arun(url, config=config1)
+
+        # Cached with validation ENABLED - should be "hit_validated"
+        config2 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            start = time.perf_counter()
+            result2 = await crawler.arun(url, config=config2)
+            elapsed = time.perf_counter() - start
+
+        assert result2.cache_status in ["hit_validated", "hit_fallback"]
+        print(f"\nValidation enabled: {elapsed:.3f}s (cache_status: {result2.cache_status})")
+
+
+class TestCacheValidationResponseHeaders:
+    """Test that response headers are properly stored and retrieved."""
+
+    @pytest.mark.asyncio
+    async def test_response_headers_stored(self):
+        """Verify response headers including ETag and Last-Modified are stored."""
+        url = "https://docs.python.org/3/"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+        config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=False)
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url, config=config)
+
+        assert result.success
+        assert result.response_headers is not None
+
+        # Check that cache-relevant headers are captured
+        headers = result.response_headers
+        print(f"\nResponse headers for {url}:")
+
+        # Look for ETag (case-insensitive)
+        etag = headers.get("etag") or headers.get("ETag")
+        print(f"  - ETag: {etag}")
+
+        # Look for Last-Modified
+        last_modified = headers.get("last-modified") or headers.get("Last-Modified")
+        print(f"  - Last-Modified: {last_modified}")
+
+        # Look for Cache-Control
+        cache_control = headers.get("cache-control") or headers.get("Cache-Control")
+        print(f"  - Cache-Control: {cache_control}")
+
+        # At least one should be present for docs.python.org
+        assert etag or last_modified, "Should have ETag or Last-Modified header"
+
+    @pytest.mark.asyncio
+    async def test_headers_used_for_validation(self):
+        """Verify stored headers are used for conditional requests."""
+        url = "https://docs.crawl4ai.com/"
+
+        browser_config = BrowserConfig(headless=True, verbose=False)
+
+        # Fresh crawl to store headers
+        config1 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result1 = await crawler.arun(url, config=config1)
+
+        # Get stored metadata
+        metadata = await async_db_manager.aget_cache_metadata(url)
+        stored_etag = metadata.get("etag")
+        stored_last_modified = metadata.get("last_modified")
+
+        print(f"\nStored validation data for {url}:")
+        print(f"  - etag: {stored_etag}")
+        print(f"  - last_modified: {stored_last_modified}")
+
+        # Validate - should use stored headers
+        config2 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED, check_cache_freshness=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result2 = await crawler.arun(url, config=config2)
+
+        # Should get validated hit (304 response)
+        assert result2.cache_status in ["hit_validated", "hit_fallback"]
+        print(f"  - Validation result: {result2.cache_status}")
diff --git a/tests/cache_validation/test_head_fingerprint.py b/tests/cache_validation/test_head_fingerprint.py
new file mode 100644
index 0000000..287f255
--- /dev/null
+++ b/tests/cache_validation/test_head_fingerprint.py
@@ -0,0 +1,97 @@
+"""Unit tests for head fingerprinting."""
+
+import pytest
+from crawl4ai.utils import compute_head_fingerprint
+
+
+class TestHeadFingerprint:
+    """Tests for the compute_head_fingerprint function."""
+
+    def test_same_content_same_fingerprint(self):
+        """Identical  content produces same fingerprint."""
+        head = "Test Page"
+        fp1 = compute_head_fingerprint(head)
+        fp2 = compute_head_fingerprint(head)
+        assert fp1 == fp2
+        assert fp1 != ""
+
+    def test_different_title_different_fingerprint(self):
+        """Different title produces different fingerprint."""
+        head1 = "Title A"
+        head2 = "Title B"
+        assert compute_head_fingerprint(head1) != compute_head_fingerprint(head2)
+
+    def test_empty_head_returns_empty_string(self):
+        """Empty or None head should return empty fingerprint."""
+        assert compute_head_fingerprint("") == ""
+        assert compute_head_fingerprint(None) == ""
+
+    def test_head_without_signals_returns_empty(self):
+        """Head without title or key meta tags returns empty."""
+        head = ""
+        assert compute_head_fingerprint(head) == ""
+
+    def test_extracts_title(self):
+        """Title is extracted and included in fingerprint."""
+        head1 = "My Title"
+        head2 = "My Title"
+        # Same title should produce same fingerprint
+        assert compute_head_fingerprint(head1) == compute_head_fingerprint(head2)
+
+    def test_extracts_meta_description(self):
+        """Meta description is extracted."""
+        head1 = ''
+        head2 = ''
+        assert compute_head_fingerprint(head1) != compute_head_fingerprint(head2)
+
+    def test_extracts_og_tags(self):
+        """Open Graph tags are extracted."""
+        head1 = ''
+        head2 = ''
+        assert compute_head_fingerprint(head1) != compute_head_fingerprint(head2)
+
+    def test_extracts_og_image(self):
+        """og:image is extracted and affects fingerprint."""
+        head1 = ''
+        head2 = ''
+        assert compute_head_fingerprint(head1) != compute_head_fingerprint(head2)
+
+    def test_extracts_article_modified_time(self):
+        """article:modified_time is extracted."""
+        head1 = ''
+        head2 = ''
+        assert compute_head_fingerprint(head1) != compute_head_fingerprint(head2)
+
+    def test_case_insensitive(self):
+        """Fingerprinting is case-insensitive for tags."""
+        head1 = "Test"
+        head2 = "test"
+        # Both should extract title (case insensitive)
+        fp1 = compute_head_fingerprint(head1)
+        fp2 = compute_head_fingerprint(head2)
+        assert fp1 != ""
+        assert fp2 != ""
+
+    def test_handles_attribute_order(self):
+        """Handles different attribute orders in meta tags."""
+        head1 = ''
+        head2 = ''
+        assert compute_head_fingerprint(head1) == compute_head_fingerprint(head2)
+
+    def test_real_world_head(self):
+        """Test with a realistic head section."""
+        head = '''
+        
+            
+            Python Documentation
+            
+            
+            
+            
+            
+        
+        '''
+        fp = compute_head_fingerprint(head)
+        assert fp != ""
+        # Should be deterministic
+        assert fp == compute_head_fingerprint(head)
diff --git a/tests/cache_validation/test_real_domains.py b/tests/cache_validation/test_real_domains.py
new file mode 100644
index 0000000..2c1a4f2
--- /dev/null
+++ b/tests/cache_validation/test_real_domains.py
@@ -0,0 +1,354 @@
+"""
+Real-world tests for cache validation using actual HTTP requests.
+No mocks - all tests hit real servers.
+"""
+
+import pytest
+from crawl4ai.cache_validator import CacheValidator, CacheValidationResult
+from crawl4ai.utils import compute_head_fingerprint
+
+
+class TestRealDomainsConditionalSupport:
+    """Test domains that support HTTP conditional requests (ETag/Last-Modified)."""
+
+    @pytest.mark.asyncio
+    async def test_docs_python_org_etag(self):
+        """docs.python.org supports ETag - should return 304."""
+        url = "https://docs.python.org/3/"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            # First fetch to get ETag
+            head_html, etag, last_modified = await validator._fetch_head(url)
+
+            assert head_html is not None, "Should fetch head content"
+            assert etag is not None, "docs.python.org should return ETag"
+
+            # Validate with the ETag we just got
+            result = await validator.validate(url=url, stored_etag=etag)
+
+            assert result.status == CacheValidationResult.FRESH, f"Expected FRESH, got {result.status}: {result.reason}"
+            assert "304" in result.reason
+
+    @pytest.mark.asyncio
+    async def test_docs_crawl4ai_etag(self):
+        """docs.crawl4ai.com supports ETag - should return 304."""
+        url = "https://docs.crawl4ai.com/"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            head_html, etag, last_modified = await validator._fetch_head(url)
+
+            assert etag is not None, "docs.crawl4ai.com should return ETag"
+
+            result = await validator.validate(url=url, stored_etag=etag)
+
+            assert result.status == CacheValidationResult.FRESH, f"Expected FRESH, got {result.status}: {result.reason}"
+
+    @pytest.mark.asyncio
+    async def test_wikipedia_last_modified(self):
+        """Wikipedia supports Last-Modified - should return 304."""
+        url = "https://en.wikipedia.org/wiki/Web_crawler"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            head_html, etag, last_modified = await validator._fetch_head(url)
+
+            assert last_modified is not None, "Wikipedia should return Last-Modified"
+
+            result = await validator.validate(url=url, stored_last_modified=last_modified)
+
+            assert result.status == CacheValidationResult.FRESH, f"Expected FRESH, got {result.status}: {result.reason}"
+
+    @pytest.mark.asyncio
+    async def test_github_pages(self):
+        """GitHub Pages supports conditional requests."""
+        url = "https://pages.github.com/"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            head_html, etag, last_modified = await validator._fetch_head(url)
+
+            # GitHub Pages typically has at least one
+            has_conditional = etag is not None or last_modified is not None
+            assert has_conditional, "GitHub Pages should support conditional requests"
+
+            result = await validator.validate(
+                url=url,
+                stored_etag=etag,
+                stored_last_modified=last_modified,
+            )
+
+            assert result.status == CacheValidationResult.FRESH
+
+    @pytest.mark.asyncio
+    async def test_httpbin_etag(self):
+        """httpbin.org/etag endpoint for testing ETag."""
+        url = "https://httpbin.org/etag/test-etag-value"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            result = await validator.validate(url=url, stored_etag='"test-etag-value"')
+
+            # httpbin should return 304 for matching ETag
+            assert result.status == CacheValidationResult.FRESH, f"Expected FRESH, got {result.status}: {result.reason}"
+
+
+class TestRealDomainsNoConditionalSupport:
+    """Test domains that may NOT support HTTP conditional requests."""
+
+    @pytest.mark.asyncio
+    async def test_dynamic_site_fingerprint_fallback(self):
+        """Test fingerprint-based validation for sites without conditional support."""
+        # Use a site that changes frequently but has stable head
+        url = "https://example.com/"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            # Get head and compute fingerprint
+            head_html, etag, last_modified = await validator._fetch_head(url)
+
+            assert head_html is not None
+            fingerprint = compute_head_fingerprint(head_html)
+
+            # Validate using fingerprint (not etag/last-modified)
+            result = await validator.validate(
+                url=url,
+                stored_head_fingerprint=fingerprint,
+            )
+
+            # Should be FRESH since fingerprint should match
+            assert result.status == CacheValidationResult.FRESH, f"Expected FRESH, got {result.status}: {result.reason}"
+            assert "fingerprint" in result.reason.lower()
+
+    @pytest.mark.asyncio
+    async def test_news_site_changes_frequently(self):
+        """News sites change frequently - test that we can detect changes."""
+        url = "https://www.bbc.com/news"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            head_html, etag, last_modified = await validator._fetch_head(url)
+
+            # BBC News has ETag but it changes with content
+            assert head_html is not None
+
+            # Using a fake old ETag should return STALE (200 with different content)
+            result = await validator.validate(
+                url=url,
+                stored_etag='"fake-old-etag-12345"',
+            )
+
+            # Should be STALE because the ETag doesn't match
+            assert result.status == CacheValidationResult.STALE, f"Expected STALE, got {result.status}: {result.reason}"
+
+
+class TestRealDomainsEdgeCases:
+    """Edge cases with real domains."""
+
+    @pytest.mark.asyncio
+    async def test_nonexistent_domain(self):
+        """Non-existent domain should return ERROR."""
+        url = "https://this-domain-definitely-does-not-exist-xyz123.com/"
+
+        async with CacheValidator(timeout=5.0) as validator:
+            result = await validator.validate(url=url, stored_etag='"test"')
+
+            assert result.status == CacheValidationResult.ERROR
+
+    @pytest.mark.asyncio
+    async def test_timeout_slow_server(self):
+        """Test timeout handling with a slow endpoint."""
+        # httpbin delay endpoint
+        url = "https://httpbin.org/delay/10"
+
+        async with CacheValidator(timeout=2.0) as validator:  # 2 second timeout
+            result = await validator.validate(url=url, stored_etag='"test"')
+
+            # Should timeout and return ERROR
+            assert result.status == CacheValidationResult.ERROR
+            assert "timeout" in result.reason.lower() or "timed out" in result.reason.lower()
+
+    @pytest.mark.asyncio
+    async def test_redirect_handling(self):
+        """Test that redirects are followed."""
+        # httpbin redirect
+        url = "https://httpbin.org/redirect/1"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            head_html, etag, last_modified = await validator._fetch_head(url)
+
+            # Should follow redirect and get content
+            # The final page might not have useful head content, but shouldn't error
+            # This tests that redirects are handled
+
+    @pytest.mark.asyncio
+    async def test_https_only(self):
+        """Test HTTPS connection."""
+        url = "https://www.google.com/"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            head_html, etag, last_modified = await validator._fetch_head(url)
+
+            assert head_html is not None
+            assert "."""
+        url = "https://docs.python.org/3/"
+
+        async with CacheValidator(timeout=15.0) as validator:
+            head_html, _, _ = await validator._fetch_head(url)
+
+            assert head_html is not None
+            assert "" in head_html.lower()
+            # Should NOT contain body content
+            assert "") < head_html.lower().find(" package name)
+PACKAGE_MAPPINGS = {
+    'bs4': 'beautifulsoup4',
+    'PIL': 'pillow',
+    'cv2': 'opencv-python',
+    'sklearn': 'scikit-learn',
+    'yaml': 'PyYAML',
+    'OpenSSL': 'pyOpenSSL',
+    'sqlalchemy': 'SQLAlchemy',
+    'playwright': 'playwright',
+    'patchright': 'patchright',
+    'dotenv': 'python-dotenv',
+    'fake_useragent': 'fake-useragent',
+    'playwright_stealth': 'playwright-stealth',
+    'sentence_transformers': 'sentence-transformers',
+    'rank_bm25': 'rank-bm25',
+    'snowballstemmer': 'snowballstemmer',
+    'pypdf': 'pypdf',
+    'pdf2image': 'pdf2image',
+}
+
+
+class ImportVisitor(ast.NodeVisitor):
+    """AST visitor to extract imports from Python files"""
+    
+    def __init__(self):
+        self.imports = {}  # Changed to dict to store line numbers
+        self.from_imports = {}
+    
+    def visit_Import(self, node):
+        for alias in node.names:
+            module_name = alias.name.split('.')[0]
+            if module_name not in self.imports:
+                self.imports[module_name] = []
+            self.imports[module_name].append(node.lineno)
+    
+    def visit_ImportFrom(self, node):
+        if node.module and node.level == 0:  # absolute imports only
+            module_name = node.module.split('.')[0]
+            if module_name not in self.from_imports:
+                self.from_imports[module_name] = []
+            self.from_imports[module_name].append(node.lineno)
+
+
+def extract_imports_from_file(filepath: Path) -> Dict[str, List[int]]:
+    """Extract all imports from a Python file with line numbers"""
+    all_imports = {}
+    
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            content = f.read()
+        
+        tree = ast.parse(content)
+        visitor = ImportVisitor()
+        visitor.visit(tree)
+        
+        # Merge imports and from_imports
+        for module, lines in visitor.imports.items():
+            if module not in all_imports:
+                all_imports[module] = []
+            all_imports[module].extend(lines)
+            
+        for module, lines in visitor.from_imports.items():
+            if module not in all_imports:
+                all_imports[module] = []
+            all_imports[module].extend(lines)
+        
+    except Exception as e:
+        # Silently skip files that can't be parsed
+        pass
+    
+    return all_imports
+
+
+def get_codebase_imports_with_files(root_dir: Path) -> Dict[str, List[Tuple[str, List[int]]]]:
+    """Get all imports from the crawl4ai library and docs folders with file locations and line numbers"""
+    import_to_files = defaultdict(list)
+    
+    # Only scan crawl4ai library folder and docs folder
+    target_dirs = [
+        root_dir / 'crawl4ai',
+        root_dir / 'docs'
+    ]
+    
+    for target_dir in target_dirs:
+        if not target_dir.exists():
+            continue
+            
+        for py_file in target_dir.rglob('*.py'):
+            # Skip __pycache__ directories
+            if '__pycache__' in py_file.parts:
+                continue
+            
+            # Skip setup.py and similar files
+            if py_file.name in ['setup.py', 'setup.cfg', 'conf.py']:
+                continue
+                
+            imports = extract_imports_from_file(py_file)
+            
+            # Map each import to the file and line numbers
+            for imp, line_numbers in imports.items():
+                relative_path = py_file.relative_to(root_dir)
+                import_to_files[imp].append((str(relative_path), sorted(line_numbers)))
+    
+    return dict(import_to_files)
+
+
+def get_declared_dependencies() -> Set[str]:
+    """Get declared dependencies from pyproject.toml and requirements.txt"""
+    declared = set()
+    
+    # Read from pyproject.toml
+    if Path('pyproject.toml').exists():
+        with open('pyproject.toml', 'r') as f:
+            data = toml.load(f)
+        
+        # Get main dependencies
+        deps = data.get('project', {}).get('dependencies', [])
+        for dep in deps:
+            # Parse dependency string (e.g., "numpy>=1.26.0,<3")
+            match = re.match(r'^([a-zA-Z0-9_-]+)', dep)
+            if match:
+                pkg_name = match.group(1).lower()
+                declared.add(pkg_name)
+        
+        # Get optional dependencies
+        optional = data.get('project', {}).get('optional-dependencies', {})
+        for group, deps in optional.items():
+            for dep in deps:
+                match = re.match(r'^([a-zA-Z0-9_-]+)', dep)
+                if match:
+                    pkg_name = match.group(1).lower()
+                    declared.add(pkg_name)
+    
+    # Also check requirements.txt as backup
+    if Path('requirements.txt').exists():
+        with open('requirements.txt', 'r') as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    match = re.match(r'^([a-zA-Z0-9_-]+)', line)
+                    if match:
+                        pkg_name = match.group(1).lower()
+                        declared.add(pkg_name)
+    
+    return declared
+
+
+def normalize_package_name(name: str) -> str:
+    """Normalize package name for comparison"""
+    # Handle known mappings first
+    if name in PACKAGE_MAPPINGS:
+        return PACKAGE_MAPPINGS[name].lower()
+    
+    # Basic normalization
+    return name.lower().replace('_', '-')
+
+
+def check_missing_dependencies():
+    """Main function to check for missing dependencies"""
+    print("🔍 Analyzing crawl4ai library and docs folders...\n")
+    
+    # Get all imports with their file locations
+    root_dir = Path('.')
+    import_to_files = get_codebase_imports_with_files(root_dir)
+    
+    # Get declared dependencies
+    declared_deps = get_declared_dependencies()
+    
+    # Normalize declared dependencies
+    normalized_declared = {normalize_package_name(dep) for dep in declared_deps}
+    
+    # Categorize imports
+    external_imports = {}
+    local_imports = {}
+    
+    # Known local packages
+    local_packages = {'crawl4ai'}
+    
+    for imp, file_info in import_to_files.items():
+        # Skip standard library
+        if imp in STDLIB_MODULES:
+            continue
+            
+        # Check if it's a local import
+        if any(imp.startswith(local) for local in local_packages):
+            local_imports[imp] = file_info
+        else:
+            external_imports[imp] = file_info
+    
+    # Check which external imports are not declared
+    not_declared = {}
+    declared_imports = {}
+    
+    for imp, file_info in external_imports.items():
+        normalized_imp = normalize_package_name(imp)
+        
+        # Check if import is covered by declared dependencies
+        found = False
+        for declared in normalized_declared:
+            if normalized_imp == declared or normalized_imp.startswith(declared + '.') or declared.startswith(normalized_imp):
+                found = True
+                break
+        
+        if found:
+            declared_imports[imp] = file_info
+        else:
+            not_declared[imp] = file_info
+    
+    # Print results
+    print(f"📊 Summary:")
+    print(f"  - Total unique imports: {len(import_to_files)}")
+    print(f"  - External imports: {len(external_imports)}")
+    print(f"  - Declared dependencies: {len(declared_deps)}")
+    print(f"  - External imports NOT in dependencies: {len(not_declared)}\n")
+    
+    if not_declared:
+        print("❌ External imports NOT declared in pyproject.toml or requirements.txt:\n")
+        
+        # Sort by import name
+        for imp in sorted(not_declared.keys()):
+            file_info = not_declared[imp]
+            print(f"  📦 {imp}")
+            if imp in PACKAGE_MAPPINGS:
+                print(f"     → Package name: {PACKAGE_MAPPINGS[imp]}")
+            
+            # Show up to 3 files that use this import
+            for i, (file_path, line_numbers) in enumerate(file_info[:3]):
+                # Format line numbers for clickable output
+                if len(line_numbers) == 1:
+                    print(f"     - {file_path}:{line_numbers[0]}")
+                else:
+                    # Show first few line numbers
+                    line_str = ','.join(str(ln) for ln in line_numbers[:3])
+                    if len(line_numbers) > 3:
+                        line_str += f"... ({len(line_numbers)} imports)"
+                    print(f"     - {file_path}: lines {line_str}")
+            
+            if len(file_info) > 3:
+                print(f"     ... and {len(file_info) - 3} more files")
+            print()
+    
+    # Check for potentially unused dependencies
+    print("\n🔎 Checking declared dependencies usage...\n")
+    
+    # Get all used external packages
+    used_packages = set()
+    for imp in external_imports.keys():
+        normalized = normalize_package_name(imp)
+        used_packages.add(normalized)
+    
+    # Find unused
+    unused = []
+    for dep in declared_deps:
+        normalized_dep = normalize_package_name(dep)
+        
+        # Check if any import uses this dependency
+        found_usage = False
+        for used in used_packages:
+            if used == normalized_dep or used.startswith(normalized_dep) or normalized_dep.startswith(used):
+                found_usage = True
+                break
+        
+        if not found_usage:
+            # Some packages are commonly unused directly
+            indirect_deps = {'wheel', 'setuptools', 'pip', 'colorama', 'certifi', 'packaging', 'urllib3'}
+            if normalized_dep not in indirect_deps:
+                unused.append(dep)
+    
+    if unused:
+        print("⚠️  Declared dependencies with NO imports found:")
+        for dep in sorted(unused):
+            print(f"  - {dep}")
+        print("\n  Note: These might be used indirectly or by other dependencies")
+    else:
+        print("✅ All declared dependencies have corresponding imports")
+    
+    print("\n" + "="*60)
+    print("💡 How to use this report:")
+    print("  1. Check each ❌ import to see if it's legitimate")
+    print("  2. If legitimate, add the package to pyproject.toml")
+    print("  3. If it's an internal module or typo, fix the import")
+    print("  4. Review unused dependencies - remove if truly not needed")
+    print("="*60)
+
+
+if __name__ == '__main__':
+    check_missing_dependencies()
\ No newline at end of file
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
new file mode 100644
index 0000000..ed8f6d7
--- /dev/null
+++ b/tests/cli/test_cli.py
@@ -0,0 +1,257 @@
+import pytest
+from click.testing import CliRunner
+from pathlib import Path
+from unittest.mock import patch
+import json
+import yaml
+from crawl4ai.cli import cli, load_config_file, parse_key_values
+from crawl4ai.models import CrawlResult, MarkdownGenerationResult
+import tempfile
+import os
+import click
+
+@pytest.fixture
+def runner():
+    return CliRunner()
+
+@pytest.fixture
+def temp_config_dir():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        old_home = os.environ.get('HOME')
+        os.environ['HOME'] = tmpdir
+        yield Path(tmpdir)
+        if old_home:
+            os.environ['HOME'] = old_home
+
+@pytest.fixture
+def sample_configs(temp_config_dir):
+    configs = {
+        'browser.yml': {
+            'headless': True,
+            'viewport_width': 1280,
+            'user_agent_mode': 'random'
+        },
+        'crawler.yml': {
+            'cache_mode': 'bypass',
+            'wait_until': 'networkidle',
+            'scan_full_page': True
+        },
+        'extract_css.yml': {
+            'type': 'json-css',
+            'params': {'verbose': True}
+        },
+        'css_schema.json': {
+            'name': 'ArticleExtractor',
+            'baseSelector': '.article',
+            'fields': [
+                {'name': 'title', 'selector': 'h1.title', 'type': 'text'},
+                {'name': 'link', 'selector': 'a.read-more', 'type': 'attribute', 'attribute': 'href'}
+            ]
+        }
+    }
+    
+    for filename, content in configs.items():
+        path = temp_config_dir / filename
+        with open(path, 'w') as f:
+            if filename.endswith('.yml'):
+                yaml.dump(content, f)
+            else:
+                json.dump(content, f)
+                
+    return {name: str(temp_config_dir / name) for name in configs}
+
+class TestCLIBasics:
+    def test_help(self, runner):
+        result = runner.invoke(cli, ['--help'])
+        assert result.exit_code == 0
+        assert 'Crawl4AI CLI' in result.output
+
+    def test_examples(self, runner):
+        result = runner.invoke(cli, ['--example'])
+        assert result.exit_code == 0
+        assert 'Examples' in result.output
+
+    def test_missing_url(self, runner):
+        result = runner.invoke(cli)
+        assert result.exit_code != 0
+        assert 'URL argument is required' in result.output
+
+class TestConfigParsing:
+    def test_parse_key_values_basic(self):
+        result = parse_key_values(None, None, "key1=value1,key2=true")
+        assert result == {'key1': 'value1', 'key2': True}
+
+    def test_parse_key_values_invalid(self):
+        with pytest.raises(click.BadParameter):
+            parse_key_values(None, None, "invalid_format")
+
+class TestConfigLoading:
+    def test_load_yaml_config(self, sample_configs):
+        config = load_config_file(sample_configs['browser.yml'])
+        assert config['headless'] is True
+        assert config['viewport_width'] == 1280
+
+    def test_load_json_config(self, sample_configs):
+        config = load_config_file(sample_configs['css_schema.json'])
+        assert config['name'] == 'ArticleExtractor'
+        assert len(config['fields']) == 2
+
+    def test_load_nonexistent_config(self):
+        with pytest.raises(click.BadParameter):
+            load_config_file('nonexistent.yml')
+
+class TestLLMConfig:
+    def test_llm_config_creation(self, temp_config_dir, runner):
+        def input_simulation(inputs):
+            return runner.invoke(cli, ['https://example.com', '-q', 'test question'], 
+                               input='\n'.join(inputs))
+            
+class TestCrawlingFeatures:
+    def test_basic_crawl(self, runner):
+        result = runner.invoke(cli, ['https://example.com'])
+        assert result.exit_code == 0
+
+
+class TestErrorHandling:
+    def test_invalid_config_file(self, runner):
+        result = runner.invoke(cli, [
+            'https://example.com',
+            '--browser-config', 'nonexistent.yml'
+        ])
+        assert result.exit_code != 0
+
+    def test_invalid_schema(self, runner, temp_config_dir):
+        invalid_schema = temp_config_dir / 'invalid_schema.json'
+        with open(invalid_schema, 'w') as f:
+            f.write('invalid json')
+            
+        result = runner.invoke(cli, [
+            'https://example.com',
+            '--schema', str(invalid_schema)
+        ])
+        assert result.exit_code != 0
+
+class TestDeepCrawlOutput:
+    """Tests for deep crawl output formatting"""
+
+    @pytest.fixture
+    def mock_crawl_results(self):
+        """Create mock CrawlResult objects simulating deep crawl results"""
+        def make_result(url, content):
+            markdown = MarkdownGenerationResult(
+                raw_markdown=content,
+                markdown_with_citations=content,
+                references_markdown="",
+                fit_markdown=content,
+            )
+            result = CrawlResult(
+                url=url,
+                html=f"{content}",
+                success=True,
+                metadata={"depth": 0},
+            )
+            result._markdown = markdown
+            return result
+
+        return [
+            make_result("https://example.com/", "# Homepage\n\nWelcome to the homepage."),
+            make_result("https://example.com/about", "# About\n\nAbout us page content."),
+            make_result("https://example.com/contact", "# Contact\n\nContact information."),
+        ]
+
+    def test_deep_crawl_markdown_output_includes_all_pages(self, runner, mock_crawl_results):
+        """Test that deep crawl with markdown output includes all pages, not just the first"""
+        with patch('crawl4ai.cli.anyio.run') as mock_anyio_run:
+            # Return list of results (simulating deep crawl)
+            mock_anyio_run.return_value = mock_crawl_results
+
+            result = runner.invoke(cli, [
+                'crawl',
+                'https://example.com',
+                '--deep-crawl', 'bfs',
+                '--max-pages', '3',
+                '-o', 'markdown'
+            ])
+
+            assert result.exit_code == 0, f"CLI failed with: {result.output}"
+            # Should contain content from ALL pages
+            assert 'https://example.com/' in result.output
+            assert 'https://example.com/about' in result.output
+            assert 'https://example.com/contact' in result.output
+            assert 'Homepage' in result.output
+            assert 'About us page content' in result.output
+            assert 'Contact information' in result.output
+
+    def test_deep_crawl_markdown_fit_output_includes_all_pages(self, runner, mock_crawl_results):
+        """Test that deep crawl with markdown-fit output includes all pages"""
+        with patch('crawl4ai.cli.anyio.run') as mock_anyio_run:
+            mock_anyio_run.return_value = mock_crawl_results
+
+            result = runner.invoke(cli, [
+                'crawl',
+                'https://example.com',
+                '--deep-crawl', 'bfs',
+                '--max-pages', '3',
+                '-o', 'markdown-fit'
+            ])
+
+            assert result.exit_code == 0, f"CLI failed with: {result.output}"
+            # Should contain all URLs
+            assert 'https://example.com/' in result.output
+            assert 'https://example.com/about' in result.output
+            assert 'https://example.com/contact' in result.output
+
+    def test_deep_crawl_file_output_includes_all_pages(self, runner, mock_crawl_results, tmp_path):
+        """Test that deep crawl with file output includes all pages"""
+        output_file = tmp_path / "output.md"
+
+        with patch('crawl4ai.cli.anyio.run') as mock_anyio_run:
+            mock_anyio_run.return_value = mock_crawl_results
+
+            result = runner.invoke(cli, [
+                'crawl',
+                'https://example.com',
+                '--deep-crawl', 'bfs',
+                '--max-pages', '3',
+                '-o', 'markdown',
+                '-O', str(output_file)
+            ])
+
+            assert result.exit_code == 0, f"CLI failed with: {result.output}"
+            content = output_file.read_text()
+            # Should contain content from ALL pages
+            assert 'https://example.com/' in content
+            assert 'https://example.com/about' in content
+            assert 'https://example.com/contact' in content
+
+    def test_single_crawl_markdown_output_unchanged(self, runner):
+        """Test that single (non-deep) crawl still works correctly"""
+        markdown = MarkdownGenerationResult(
+            raw_markdown="# Single Page\n\nContent here.",
+            markdown_with_citations="# Single Page\n\nContent here.",
+            references_markdown="",
+        )
+        single_result = CrawlResult(
+            url="https://example.com/",
+            html="test",
+            success=True,
+        )
+        single_result._markdown = markdown
+
+        with patch('crawl4ai.cli.anyio.run') as mock_anyio_run:
+            # Return single result (not a list)
+            mock_anyio_run.return_value = single_result
+
+            result = runner.invoke(cli, [
+                'crawl',
+                'https://example.com',
+                '-o', 'markdown'
+            ])
+
+            assert result.exit_code == 0, f"CLI failed with: {result.output}"
+            assert '# Single Page' in result.output
+            assert 'Content here' in result.output
+
+
+if __name__ == '__main__':
+    pytest.main(['-v', '-s', '--tb=native', __file__])
diff --git a/tests/deep_crawling/__init__.py b/tests/deep_crawling/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/deep_crawling/test_deep_crawl_cancellation.py b/tests/deep_crawling/test_deep_crawl_cancellation.py
new file mode 100644
index 0000000..0ffb8a7
--- /dev/null
+++ b/tests/deep_crawling/test_deep_crawl_cancellation.py
@@ -0,0 +1,597 @@
+"""
+Test Suite: Deep Crawl Cancellation Tests
+
+Tests that verify:
+1. should_cancel callback is called before each URL
+2. cancel() method immediately stops the crawl
+3. cancelled property correctly reflects state
+4. Strategy reuse works after cancellation
+5. Both sync and async should_cancel callbacks work
+6. Callback exceptions don't crash the crawl
+7. State notifications include cancelled flag
+"""
+
+import pytest
+import asyncio
+from typing import Dict, Any, List
+from unittest.mock import MagicMock
+
+from crawl4ai.deep_crawling import (
+    BFSDeepCrawlStrategy,
+    DFSDeepCrawlStrategy,
+    BestFirstCrawlingStrategy,
+)
+
+
+# ============================================================================
+# Helper Functions for Mock Crawler
+# ============================================================================
+
+def create_mock_config(stream=False):
+    """Create a mock CrawlerRunConfig."""
+    config = MagicMock()
+    config.stream = stream
+
+    def clone_config(**kwargs):
+        """Clone returns a new config with overridden values."""
+        new_config = MagicMock()
+        new_config.stream = kwargs.get('stream', stream)
+        new_config.clone = MagicMock(side_effect=clone_config)
+        return new_config
+
+    config.clone = MagicMock(side_effect=clone_config)
+    return config
+
+
+def create_mock_crawler_with_links(num_links: int = 3):
+    """Create mock crawler that returns results with links."""
+    call_count = 0
+
+    async def mock_arun_many(urls, config):
+        nonlocal call_count
+        results = []
+        for url in urls:
+            call_count += 1
+            result = MagicMock()
+            result.url = url
+            result.success = True
+            result.metadata = {}
+
+            # Generate child links
+            links = []
+            for i in range(num_links):
+                link_url = f"{url}/child{call_count}_{i}"
+                links.append({"href": link_url})
+
+            result.links = {"internal": links, "external": []}
+            results.append(result)
+
+        # For streaming mode, return async generator
+        if config.stream:
+            async def gen():
+                for r in results:
+                    yield r
+            return gen()
+        return results
+
+    crawler = MagicMock()
+    crawler.arun_many = mock_arun_many
+    return crawler
+
+
+def create_mock_crawler_tracking(crawl_order: List[str], return_no_links: bool = False):
+    """Create mock crawler that tracks crawl order."""
+
+    async def mock_arun_many(urls, config):
+        results = []
+        for url in urls:
+            crawl_order.append(url)
+            result = MagicMock()
+            result.url = url
+            result.success = True
+            result.metadata = {}
+            result.links = {"internal": [], "external": []} if return_no_links else {"internal": [{"href": f"{url}/child"}], "external": []}
+            results.append(result)
+
+        # For streaming mode, return async generator
+        if config.stream:
+            async def gen():
+                for r in results:
+                    yield r
+            return gen()
+        return results
+
+    crawler = MagicMock()
+    crawler.arun_many = mock_arun_many
+    return crawler
+
+
+# ============================================================================
+# TEST SUITE: Cancellation via should_cancel Callback
+# ============================================================================
+
+class TestBFSCancellation:
+    """BFS strategy cancellation tests."""
+
+    @pytest.mark.asyncio
+    async def test_cancel_via_async_callback(self):
+        """Verify async should_cancel callback stops crawl."""
+        pages_crawled = 0
+        cancel_after = 3
+
+        async def check_cancel():
+            return pages_crawled >= cancel_after
+
+        async def track_pages(state: Dict[str, Any]):
+            nonlocal pages_crawled
+            pages_crawled = state.get("pages_crawled", 0)
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=5,
+            max_pages=100,
+            should_cancel=check_cancel,
+            on_state_change=track_pages,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=5)
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Should have stopped after cancel_after pages
+        assert strategy.cancelled == True
+        assert strategy._pages_crawled >= cancel_after
+        assert strategy._pages_crawled < 100  # Should not have crawled all pages
+
+    @pytest.mark.asyncio
+    async def test_cancel_via_sync_callback(self):
+        """Verify sync should_cancel callback works."""
+        cancel_flag = False
+
+        def check_cancel():
+            return cancel_flag
+
+        async def set_cancel_after_3(state: Dict[str, Any]):
+            nonlocal cancel_flag
+            if state.get("pages_crawled", 0) >= 3:
+                cancel_flag = True
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=5,
+            max_pages=100,
+            should_cancel=check_cancel,
+            on_state_change=set_cancel_after_3,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=5)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert strategy.cancelled == True
+        assert strategy._pages_crawled >= 3
+
+    @pytest.mark.asyncio
+    async def test_cancel_method_stops_crawl(self):
+        """Verify cancel() method immediately stops the crawl."""
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=5,
+            max_pages=100,
+        )
+
+        async def cancel_after_2_pages(state: Dict[str, Any]):
+            if state.get("pages_crawled", 0) >= 2:
+                strategy.cancel()
+
+        strategy._on_state_change = cancel_after_2_pages
+
+        mock_crawler = create_mock_crawler_with_links(num_links=5)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert strategy.cancelled == True
+        assert strategy._pages_crawled >= 2
+        assert strategy._pages_crawled < 100
+
+    @pytest.mark.asyncio
+    async def test_cancelled_property_reflects_state(self):
+        """Verify cancelled property correctly reflects state."""
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+
+        # Before cancel
+        assert strategy.cancelled == False
+
+        # After cancel()
+        strategy.cancel()
+        assert strategy.cancelled == True
+
+    @pytest.mark.asyncio
+    async def test_strategy_reuse_after_cancellation(self):
+        """Verify strategy can be reused after cancellation."""
+        call_count = 0
+
+        async def cancel_first_time():
+            return call_count == 1
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=5,
+            should_cancel=cancel_first_time,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=2)
+        mock_config = create_mock_config()
+
+        # First crawl - should be cancelled
+        call_count = 1
+        results1 = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+        assert strategy.cancelled == True
+
+        # Second crawl - should work normally (cancel_first_time returns False)
+        call_count = 2
+        results2 = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+        assert strategy.cancelled == False
+        assert len(results2) > len(results1)
+
+    @pytest.mark.asyncio
+    async def test_callback_exception_continues_crawl(self):
+        """Verify callback exception doesn't crash crawl (fail-open)."""
+        exception_count = 0
+
+        async def failing_callback():
+            nonlocal exception_count
+            exception_count += 1
+            raise ConnectionError("Redis connection failed")
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=3,
+            should_cancel=failing_callback,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=2)
+        mock_config = create_mock_config()
+
+        # Should not raise, should complete crawl
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert exception_count > 0  # Callback was called
+        assert len(results) > 0  # Crawl completed
+        assert strategy.cancelled == False  # Not cancelled due to exception
+
+    @pytest.mark.asyncio
+    async def test_state_includes_cancelled_flag(self):
+        """Verify state notifications include cancelled flag."""
+        states: List[Dict] = []
+        cancel_at = 3
+
+        async def capture_state(state: Dict[str, Any]):
+            states.append(state)
+
+        async def cancel_after_3():
+            return len(states) >= cancel_at
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=5,
+            max_pages=100,
+            should_cancel=cancel_after_3,
+            on_state_change=capture_state,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=5)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Last state should have cancelled=True
+        assert len(states) > 0
+        assert states[-1].get("cancelled") == True
+
+    @pytest.mark.asyncio
+    async def test_cancel_before_first_url(self):
+        """Verify cancel before first URL returns empty results."""
+        async def always_cancel():
+            return True
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=5,
+            max_pages=100,
+            should_cancel=always_cancel,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=5)
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert strategy.cancelled == True
+        assert len(results) == 0
+
+
+class TestDFSCancellation:
+    """DFS strategy cancellation tests."""
+
+    @pytest.mark.asyncio
+    async def test_cancel_via_callback(self):
+        """Verify DFS respects should_cancel callback."""
+        pages_crawled = 0
+        cancel_after = 3
+
+        async def check_cancel():
+            return pages_crawled >= cancel_after
+
+        async def track_pages(state: Dict[str, Any]):
+            nonlocal pages_crawled
+            pages_crawled = state.get("pages_crawled", 0)
+
+        strategy = DFSDeepCrawlStrategy(
+            max_depth=5,
+            max_pages=100,
+            should_cancel=check_cancel,
+            on_state_change=track_pages,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=3)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert strategy.cancelled == True
+        assert strategy._pages_crawled >= cancel_after
+        assert strategy._pages_crawled < 100
+
+    @pytest.mark.asyncio
+    async def test_cancel_method_inherited(self):
+        """Verify DFS inherits cancel() from BFS."""
+        strategy = DFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+
+        assert hasattr(strategy, 'cancel')
+        assert hasattr(strategy, 'cancelled')
+        assert hasattr(strategy, '_check_cancellation')
+
+        strategy.cancel()
+        assert strategy.cancelled == True
+
+    @pytest.mark.asyncio
+    async def test_stream_mode_cancellation(self):
+        """Verify DFS stream mode respects cancellation."""
+        results_count = 0
+        cancel_after = 2
+
+        async def check_cancel():
+            return results_count >= cancel_after
+
+        strategy = DFSDeepCrawlStrategy(
+            max_depth=5,
+            max_pages=100,
+            should_cancel=check_cancel,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=3)
+        mock_config = create_mock_config(stream=True)
+
+        async for result in strategy._arun_stream("https://example.com", mock_crawler, mock_config):
+            results_count += 1
+
+        assert strategy.cancelled == True
+        assert results_count >= cancel_after
+        assert results_count < 100
+
+
+class TestBestFirstCancellation:
+    """Best-First strategy cancellation tests."""
+
+    @pytest.mark.asyncio
+    async def test_cancel_via_callback(self):
+        """Verify Best-First respects should_cancel callback."""
+        pages_crawled = 0
+        cancel_after = 3
+
+        async def check_cancel():
+            return pages_crawled >= cancel_after
+
+        async def track_pages(state: Dict[str, Any]):
+            nonlocal pages_crawled
+            pages_crawled = state.get("pages_crawled", 0)
+
+        strategy = BestFirstCrawlingStrategy(
+            max_depth=5,
+            max_pages=100,
+            should_cancel=check_cancel,
+            on_state_change=track_pages,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=3)
+        mock_config = create_mock_config(stream=True)
+
+        async for _ in strategy._arun_stream("https://example.com", mock_crawler, mock_config):
+            pass
+
+        assert strategy.cancelled == True
+        assert strategy._pages_crawled >= cancel_after
+        assert strategy._pages_crawled < 100
+
+    @pytest.mark.asyncio
+    async def test_cancel_method_works(self):
+        """Verify Best-First cancel() method works."""
+        strategy = BestFirstCrawlingStrategy(max_depth=2, max_pages=10)
+
+        assert strategy.cancelled == False
+        strategy.cancel()
+        assert strategy.cancelled == True
+
+    @pytest.mark.asyncio
+    async def test_batch_mode_cancellation(self):
+        """Verify Best-First batch mode respects cancellation."""
+        pages_crawled = 0
+        cancel_after = 2
+
+        async def check_cancel():
+            return pages_crawled >= cancel_after
+
+        async def track_pages(state: Dict[str, Any]):
+            nonlocal pages_crawled
+            pages_crawled = state.get("pages_crawled", 0)
+
+        strategy = BestFirstCrawlingStrategy(
+            max_depth=5,
+            max_pages=100,
+            should_cancel=check_cancel,
+            on_state_change=track_pages,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=3)
+        mock_config = create_mock_config(stream=False)
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert strategy.cancelled == True
+        assert len(results) >= cancel_after
+        assert len(results) < 100
+
+
+class TestCrossStrategyCancellation:
+    """Tests that apply to all strategies."""
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("strategy_class", [
+        BFSDeepCrawlStrategy,
+        DFSDeepCrawlStrategy,
+        BestFirstCrawlingStrategy,
+    ])
+    async def test_no_cancel_callback_means_no_cancellation(self, strategy_class):
+        """Verify crawl completes normally without should_cancel."""
+        strategy = strategy_class(max_depth=1, max_pages=5)
+
+        mock_crawler = create_mock_crawler_with_links(num_links=2)
+
+        if strategy_class == BestFirstCrawlingStrategy:
+            mock_config = create_mock_config(stream=True)
+            results = []
+            async for r in strategy._arun_stream("https://example.com", mock_crawler, mock_config):
+                results.append(r)
+        else:
+            mock_config = create_mock_config()
+            results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert strategy.cancelled == False
+        assert len(results) > 0
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("strategy_class", [
+        BFSDeepCrawlStrategy,
+        DFSDeepCrawlStrategy,
+        BestFirstCrawlingStrategy,
+    ])
+    async def test_cancel_thread_safety(self, strategy_class):
+        """Verify cancel() is thread-safe (doesn't raise)."""
+        strategy = strategy_class(max_depth=2, max_pages=10)
+
+        # Call cancel from multiple "threads" (simulated)
+        for _ in range(10):
+            strategy.cancel()
+
+        # Should be cancelled without errors
+        assert strategy.cancelled == True
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("strategy_class", [
+        BFSDeepCrawlStrategy,
+        DFSDeepCrawlStrategy,
+        BestFirstCrawlingStrategy,
+    ])
+    async def test_should_cancel_param_accepted(self, strategy_class):
+        """Verify should_cancel parameter is accepted by constructor."""
+        async def dummy_cancel():
+            return False
+
+        # Should not raise
+        strategy = strategy_class(
+            max_depth=2,
+            max_pages=10,
+            should_cancel=dummy_cancel,
+        )
+
+        assert strategy._should_cancel == dummy_cancel
+
+
+class TestCancellationEdgeCases:
+    """Edge case tests for cancellation."""
+
+    @pytest.mark.asyncio
+    async def test_cancel_during_batch_processing(self):
+        """Verify cancellation during batch doesn't lose results."""
+        results_count = 0
+
+        async def cancel_mid_batch():
+            # Cancel after receiving first result
+            return results_count >= 1
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=100,
+            should_cancel=cancel_mid_batch,
+        )
+
+        async def track_results(state):
+            nonlocal results_count
+            results_count = state.get("pages_crawled", 0)
+
+        strategy._on_state_change = track_results
+
+        mock_crawler = create_mock_crawler_with_links(num_links=5)
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Should have at least the first batch of results
+        assert len(results) >= 1
+        assert strategy.cancelled == True
+
+    @pytest.mark.asyncio
+    async def test_partial_results_on_cancel(self):
+        """Verify partial results are returned on cancellation."""
+        cancel_after = 5
+
+        async def check_cancel():
+            return strategy._pages_crawled >= cancel_after
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=10,
+            max_pages=1000,
+            should_cancel=check_cancel,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=10)
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Should have results up to cancellation point
+        assert len(results) >= cancel_after
+        assert strategy.cancelled == True
+
+    @pytest.mark.asyncio
+    async def test_cancel_callback_called_once_per_level_bfs(self):
+        """Verify BFS checks cancellation once per level."""
+        check_count = 0
+
+        async def count_checks():
+            nonlocal check_count
+            check_count += 1
+            return False  # Never cancel
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=10,
+            should_cancel=count_checks,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=2)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Should have checked at least once per level
+        assert check_count >= 1
diff --git a/tests/deep_crawling/test_deep_crawl_contextvar.py b/tests/deep_crawling/test_deep_crawl_contextvar.py
new file mode 100644
index 0000000..4f1bfd4
--- /dev/null
+++ b/tests/deep_crawling/test_deep_crawl_contextvar.py
@@ -0,0 +1,340 @@
+"""
+Test Suite: Deep Crawl ContextVar Safety (Issue #1917)
+
+Tests that DeepCrawlDecorator's ContextVar (deep_crawl_active) works correctly
+when the async generator is consumed in a different asyncio context, as happens
+with Starlette's StreamingResponse in the Docker API.
+
+The bug: base_strategy.py used ContextVar.reset(token) in the generator's finally
+block, but reset() requires the same Context that created the token. When Starlette
+consumes the generator in a different Task, the Context changes -> ValueError.
+
+The fix: use ContextVar.set(False) instead of reset(token), which works across
+context boundaries.
+"""
+
+import pytest
+import asyncio
+from unittest.mock import MagicMock, AsyncMock
+
+from crawl4ai.deep_crawling.base_strategy import DeepCrawlDecorator
+
+
+# ============================================================================
+# Helpers
+# ============================================================================
+
+def create_mock_result(url="https://example.com"):
+    result = MagicMock()
+    result.url = url
+    result.success = True
+    result.metadata = {}
+    result.links = {"internal": [], "external": []}
+    return result
+
+
+def create_streaming_strategy(results):
+    """Create a mock deep crawl strategy that streams results."""
+    strategy = MagicMock()
+
+    async def mock_arun(start_url, crawler, config):
+        async def gen():
+            for r in results:
+                yield r
+        return gen()
+
+    strategy.arun = mock_arun
+    return strategy
+
+
+def create_batch_strategy(results):
+    """Create a mock deep crawl strategy that returns results as a list."""
+    strategy = MagicMock()
+
+    async def mock_arun(start_url, crawler, config):
+        return results
+
+    strategy.arun = mock_arun
+    return strategy
+
+
+def create_config(stream=False, strategy=None):
+    config = MagicMock()
+    config.stream = stream
+    config.deep_crawl_strategy = strategy
+    return config
+
+
+# ============================================================================
+# Tests: ContextVar cross-context safety (the core #1917 bug)
+# ============================================================================
+
+class TestContextVarCrossContext:
+    """Tests that deep_crawl_active ContextVar works across task boundaries."""
+
+    @pytest.mark.asyncio
+    async def test_streaming_generator_consumed_in_different_task(self):
+        """
+        Core reproduction of issue #1917:
+        Create the generator in one task, consume it in another.
+        Before the fix, this raised ValueError.
+        """
+        mock_results = [create_mock_result(f"https://example.com/{i}") for i in range(3)]
+        strategy = create_streaming_strategy(mock_results)
+        config = create_config(stream=True, strategy=strategy)
+
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        async def original_arun(url, config=None, **kwargs):
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+
+        # Call wrapped_arun in the current task — sets the token
+        gen = await wrapped("https://example.com", config=config)
+
+        # Consume in a DIFFERENT task (simulates Starlette's StreamingResponse)
+        collected = []
+
+        async def consume_in_new_task():
+            async for result in gen:
+                collected.append(result)
+
+        task = asyncio.create_task(consume_in_new_task())
+        await task
+
+        assert len(collected) == 3
+
+    @pytest.mark.asyncio
+    async def test_batch_mode_in_different_task(self):
+        """Non-streaming mode should also work across task boundaries."""
+        mock_results = [create_mock_result("https://example.com")]
+        strategy = create_batch_strategy(mock_results)
+        config = create_config(stream=False, strategy=strategy)
+
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        async def original_arun(url, config=None, **kwargs):
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+        result = await wrapped("https://example.com", config=config)
+
+        assert result == mock_results
+
+
+# ============================================================================
+# Tests: ContextVar state management
+# ============================================================================
+
+class TestContextVarState:
+    """Tests that deep_crawl_active is properly managed."""
+
+    @pytest.mark.asyncio
+    async def test_flag_is_false_after_streaming_completes(self):
+        """deep_crawl_active should be False after the generator is exhausted."""
+        strategy = create_streaming_strategy([create_mock_result()])
+        config = create_config(stream=True, strategy=strategy)
+
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        async def original_arun(url, config=None, **kwargs):
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+        gen = await wrapped("https://example.com", config=config)
+
+        async for _ in gen:
+            pass
+
+        assert decorator.deep_crawl_active.get() == False
+
+    @pytest.mark.asyncio
+    async def test_flag_is_false_after_batch_completes(self):
+        """deep_crawl_active should be False after batch mode completes."""
+        strategy = create_batch_strategy([create_mock_result()])
+        config = create_config(stream=False, strategy=strategy)
+
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        async def original_arun(url, config=None, **kwargs):
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+        await wrapped("https://example.com", config=config)
+
+        assert decorator.deep_crawl_active.get() == False
+
+    @pytest.mark.asyncio
+    async def test_flag_is_true_during_deep_crawl(self):
+        """deep_crawl_active should be True while the generator is being consumed."""
+        flag_during_yield = None
+
+        async def capturing_arun(start_url, crawler, config):
+            async def gen():
+                nonlocal flag_during_yield
+                flag_during_yield = DeepCrawlDecorator.deep_crawl_active.get()
+                yield create_mock_result(start_url)
+            return gen()
+
+        strategy = MagicMock()
+        strategy.arun = capturing_arun
+        config = create_config(stream=True, strategy=strategy)
+
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        async def original_arun(url, config=None, **kwargs):
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+        gen = await wrapped("https://example.com", config=config)
+
+        async for _ in gen:
+            pass
+
+        assert flag_during_yield == True
+
+    @pytest.mark.asyncio
+    async def test_flag_prevents_recursive_deep_crawl(self):
+        """When deep_crawl_active is True, nested calls should skip deep crawl."""
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        inner_call_hit = False
+
+        async def original_arun(url, config=None, **kwargs):
+            nonlocal inner_call_hit
+            inner_call_hit = True
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+
+        # Manually set the flag to simulate being inside a deep crawl
+        decorator.deep_crawl_active.set(True)
+        try:
+            strategy = create_batch_strategy([create_mock_result()])
+            config = create_config(stream=False, strategy=strategy)
+            # Should call original_arun directly, NOT go through strategy
+            result = await wrapped("https://example.com", config=config)
+            assert inner_call_hit == True
+        finally:
+            decorator.deep_crawl_active.set(False)
+
+    @pytest.mark.asyncio
+    async def test_flag_reset_after_streaming_error(self):
+        """deep_crawl_active should be reset even if the generator raises."""
+        strategy = MagicMock()
+
+        async def failing_arun(start_url, crawler, config):
+            async def gen():
+                yield create_mock_result("https://example.com")
+                raise RuntimeError("simulated error")
+            return gen()
+
+        strategy.arun = failing_arun
+        config = create_config(stream=True, strategy=strategy)
+
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        async def original_arun(url, config=None, **kwargs):
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+        gen = await wrapped("https://example.com", config=config)
+
+        with pytest.raises(RuntimeError, match="simulated error"):
+            async for _ in gen:
+                pass
+
+        assert decorator.deep_crawl_active.get() == False
+
+    @pytest.mark.asyncio
+    async def test_flag_reset_after_streaming_error_in_different_task(self):
+        """
+        Combines #1917 fix with error handling: generator raises in a different task.
+        Both the cross-context issue and error cleanup must work together.
+        """
+        strategy = MagicMock()
+
+        async def failing_arun(start_url, crawler, config):
+            async def gen():
+                yield create_mock_result("https://example.com")
+                raise RuntimeError("simulated error")
+            return gen()
+
+        strategy.arun = failing_arun
+        config = create_config(stream=True, strategy=strategy)
+
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        async def original_arun(url, config=None, **kwargs):
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+        gen = await wrapped("https://example.com", config=config)
+
+        error_caught = False
+
+        async def consume_in_new_task():
+            nonlocal error_caught
+            try:
+                async for _ in gen:
+                    pass
+            except RuntimeError:
+                error_caught = True
+
+        task = asyncio.create_task(consume_in_new_task())
+        await task
+
+        assert error_caught == True
+
+
+# ============================================================================
+# Tests: Concurrent requests
+# ============================================================================
+
+class TestConcurrentRequests:
+    """Tests that multiple concurrent streaming deep crawls don't interfere."""
+
+    @pytest.mark.asyncio
+    async def test_concurrent_streaming_in_separate_tasks(self):
+        """
+        Multiple concurrent streaming requests consumed in separate tasks.
+        This simulates multiple clients hitting /crawl/stream simultaneously.
+        """
+        crawler = MagicMock()
+        decorator = DeepCrawlDecorator(crawler)
+
+        async def original_arun(url, config=None, **kwargs):
+            return create_mock_result(url)
+
+        wrapped = decorator(original_arun)
+        results_per_request = {}
+
+        async def simulate_request(request_id):
+            mock_results = [create_mock_result(f"https://example.com/{request_id}/{i}") for i in range(2)]
+            strategy = create_streaming_strategy(mock_results)
+            config = create_config(stream=True, strategy=strategy)
+
+            gen = await wrapped(f"https://example.com/{request_id}", config=config)
+            results = []
+            async for result in gen:
+                results.append(result)
+                await asyncio.sleep(0.01)  # Interleave with other requests
+            results_per_request[request_id] = results
+
+        tasks = [asyncio.create_task(simulate_request(i)) for i in range(3)]
+        await asyncio.gather(*tasks)
+
+        assert len(results_per_request) == 3
+        for request_id, results in results_per_request.items():
+            assert len(results) == 2
+
+        assert decorator.deep_crawl_active.get() == False
diff --git a/tests/deep_crawling/test_deep_crawl_resume.py b/tests/deep_crawling/test_deep_crawl_resume.py
new file mode 100644
index 0000000..c6d8418
--- /dev/null
+++ b/tests/deep_crawling/test_deep_crawl_resume.py
@@ -0,0 +1,839 @@
+"""
+Test Suite: Deep Crawl Resume/Crash Recovery Tests
+
+Tests that verify:
+1. State export produces valid JSON-serializable data
+2. Resume from checkpoint continues without duplicates
+3. Simulated crash at various points recovers correctly
+4. State callback fires at expected intervals
+5. No damage to existing system behavior (regression tests)
+"""
+
+import pytest
+import asyncio
+import json
+from typing import Dict, Any, List
+from unittest.mock import AsyncMock, MagicMock
+
+from crawl4ai.deep_crawling import (
+    BFSDeepCrawlStrategy,
+    DFSDeepCrawlStrategy,
+    BestFirstCrawlingStrategy,
+    FilterChain,
+    URLPatternFilter,
+    DomainFilter,
+)
+from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
+
+
+# ============================================================================
+# Helper Functions for Mock Crawler
+# ============================================================================
+
+def create_mock_config(stream=False):
+    """Create a mock CrawlerRunConfig."""
+    config = MagicMock()
+    config.clone = MagicMock(return_value=config)
+    config.stream = stream
+    return config
+
+
+def create_mock_crawler_with_links(num_links: int = 3, include_keyword: bool = False):
+    """Create mock crawler that returns results with links."""
+    call_count = 0
+
+    async def mock_arun_many(urls, config):
+        nonlocal call_count
+        results = []
+        for url in urls:
+            call_count += 1
+            result = MagicMock()
+            result.url = url
+            result.success = True
+            result.metadata = {}
+
+            # Generate child links
+            links = []
+            for i in range(num_links):
+                link_url = f"{url}/child{call_count}_{i}"
+                if include_keyword:
+                    link_url = f"{url}/important-child{call_count}_{i}"
+                links.append({"href": link_url})
+
+            result.links = {"internal": links, "external": []}
+            results.append(result)
+
+        # For streaming mode, return async generator
+        if config.stream:
+            async def gen():
+                for r in results:
+                    yield r
+            return gen()
+        return results
+
+    crawler = MagicMock()
+    crawler.arun_many = mock_arun_many
+    return crawler
+
+
+def create_mock_crawler_tracking(crawl_order: List[str], return_no_links: bool = False):
+    """Create mock crawler that tracks crawl order."""
+
+    async def mock_arun_many(urls, config):
+        results = []
+        for url in urls:
+            crawl_order.append(url)
+            result = MagicMock()
+            result.url = url
+            result.success = True
+            result.metadata = {}
+            result.links = {"internal": [], "external": []} if return_no_links else {"internal": [{"href": f"{url}/child"}], "external": []}
+            results.append(result)
+
+        # For streaming mode, return async generator
+        if config.stream:
+            async def gen():
+                for r in results:
+                    yield r
+            return gen()
+        return results
+
+    crawler = MagicMock()
+    crawler.arun_many = mock_arun_many
+    return crawler
+
+
+def create_simple_mock_crawler():
+    """Basic mock crawler returning 1 result with 2 child links."""
+    call_count = 0
+
+    async def mock_arun_many(urls, config):
+        nonlocal call_count
+        results = []
+        for url in urls:
+            call_count += 1
+            result = MagicMock()
+            result.url = url
+            result.success = True
+            result.metadata = {}
+            result.links = {
+                "internal": [
+                    {"href": f"{url}/child1"},
+                    {"href": f"{url}/child2"},
+                ],
+                "external": []
+            }
+            results.append(result)
+
+        if config.stream:
+            async def gen():
+                for r in results:
+                    yield r
+            return gen()
+        return results
+
+    crawler = MagicMock()
+    crawler.arun_many = mock_arun_many
+    return crawler
+
+
+def create_mock_crawler_unlimited_links():
+    """Mock crawler that always returns links (for testing limits)."""
+    async def mock_arun_many(urls, config):
+        results = []
+        for url in urls:
+            result = MagicMock()
+            result.url = url
+            result.success = True
+            result.metadata = {}
+            result.links = {
+                "internal": [{"href": f"{url}/link{i}"} for i in range(10)],
+                "external": []
+            }
+            results.append(result)
+
+        if config.stream:
+            async def gen():
+                for r in results:
+                    yield r
+            return gen()
+        return results
+
+    crawler = MagicMock()
+    crawler.arun_many = mock_arun_many
+    return crawler
+
+
+# ============================================================================
+# TEST SUITE 1: Crash Recovery Tests
+# ============================================================================
+
+class TestBFSResume:
+    """BFS strategy resume tests."""
+
+    @pytest.mark.asyncio
+    async def test_state_export_json_serializable(self):
+        """Verify exported state can be JSON serialized."""
+        captured_states: List[Dict] = []
+
+        async def capture_state(state: Dict[str, Any]):
+            # Verify JSON serializable
+            json_str = json.dumps(state)
+            parsed = json.loads(json_str)
+            captured_states.append(parsed)
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=10,
+            on_state_change=capture_state,
+        )
+
+        # Create mock crawler that returns predictable results
+        mock_crawler = create_mock_crawler_with_links(num_links=3)
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Verify states were captured
+        assert len(captured_states) > 0
+
+        # Verify state structure
+        for state in captured_states:
+            assert state["strategy_type"] == "bfs"
+            assert "visited" in state
+            assert "pending" in state
+            assert "depths" in state
+            assert "pages_crawled" in state
+            assert isinstance(state["visited"], list)
+            assert isinstance(state["pending"], list)
+            assert isinstance(state["depths"], dict)
+            assert isinstance(state["pages_crawled"], int)
+
+    @pytest.mark.asyncio
+    async def test_resume_continues_from_checkpoint(self):
+        """Verify resume starts from saved state, not beginning."""
+        # Simulate state from previous crawl (visited 5 URLs, 3 pending)
+        saved_state = {
+            "strategy_type": "bfs",
+            "visited": [
+                "https://example.com",
+                "https://example.com/page1",
+                "https://example.com/page2",
+                "https://example.com/page3",
+                "https://example.com/page4",
+            ],
+            "pending": [
+                {"url": "https://example.com/page5", "parent_url": "https://example.com/page2"},
+                {"url": "https://example.com/page6", "parent_url": "https://example.com/page3"},
+                {"url": "https://example.com/page7", "parent_url": "https://example.com/page3"},
+            ],
+            "depths": {
+                "https://example.com": 0,
+                "https://example.com/page1": 1,
+                "https://example.com/page2": 1,
+                "https://example.com/page3": 1,
+                "https://example.com/page4": 1,
+                "https://example.com/page5": 2,
+                "https://example.com/page6": 2,
+                "https://example.com/page7": 2,
+            },
+            "pages_crawled": 5,
+        }
+
+        crawled_urls: List[str] = []
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=20,
+            resume_state=saved_state,
+        )
+
+        # Verify internal state was restored
+        assert strategy._resume_state == saved_state
+
+        mock_crawler = create_mock_crawler_tracking(crawled_urls, return_no_links=True)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Should NOT re-crawl already visited URLs
+        for visited_url in saved_state["visited"]:
+            assert visited_url not in crawled_urls, f"Re-crawled already visited: {visited_url}"
+
+        # Should crawl pending URLs
+        for pending in saved_state["pending"]:
+            assert pending["url"] in crawled_urls, f"Did not crawl pending: {pending['url']}"
+
+    @pytest.mark.asyncio
+    async def test_simulated_crash_mid_crawl(self):
+        """Simulate crash at URL N, verify resume continues from pending URLs."""
+        crash_after = 3
+        states_before_crash: List[Dict] = []
+
+        async def capture_until_crash(state: Dict[str, Any]):
+            states_before_crash.append(state)
+            if state["pages_crawled"] >= crash_after:
+                raise Exception("Simulated crash!")
+
+        strategy1 = BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=10,
+            on_state_change=capture_until_crash,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=5)
+        mock_config = create_mock_config()
+
+        # First crawl - crashes
+        with pytest.raises(Exception, match="Simulated crash"):
+            await strategy1._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Get last state before crash
+        last_state = states_before_crash[-1]
+        assert last_state["pages_crawled"] >= crash_after
+
+        # Calculate which URLs were already crawled vs pending
+        pending_urls = {item["url"] for item in last_state["pending"]}
+        visited_urls = set(last_state["visited"])
+        already_crawled_urls = visited_urls - pending_urls
+
+        # Resume from checkpoint
+        crawled_in_resume: List[str] = []
+
+        strategy2 = BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=10,
+            resume_state=last_state,
+        )
+
+        mock_crawler2 = create_mock_crawler_tracking(crawled_in_resume, return_no_links=True)
+
+        await strategy2._arun_batch("https://example.com", mock_crawler2, mock_config)
+
+        # Verify already-crawled URLs are not re-crawled
+        for crawled_url in already_crawled_urls:
+            assert crawled_url not in crawled_in_resume, f"Re-crawled already visited: {crawled_url}"
+
+        # Verify pending URLs are crawled
+        for pending_url in pending_urls:
+            assert pending_url in crawled_in_resume, f"Did not crawl pending: {pending_url}"
+
+    @pytest.mark.asyncio
+    async def test_callback_fires_per_url(self):
+        """Verify callback fires after each URL for maximum granularity."""
+        callback_count = 0
+        pages_crawled_sequence: List[int] = []
+
+        async def count_callbacks(state: Dict[str, Any]):
+            nonlocal callback_count
+            callback_count += 1
+            pages_crawled_sequence.append(state["pages_crawled"])
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=5,
+            on_state_change=count_callbacks,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=2)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Callback should fire once per successful URL
+        assert callback_count == strategy._pages_crawled, \
+            f"Callback fired {callback_count} times, expected {strategy._pages_crawled} (per URL)"
+
+        # pages_crawled should increment by 1 each callback
+        for i, count in enumerate(pages_crawled_sequence):
+            assert count == i + 1, f"Expected pages_crawled={i+1} at callback {i}, got {count}"
+
+    @pytest.mark.asyncio
+    async def test_export_state_returns_last_captured(self):
+        """Verify export_state() returns last captured state."""
+        last_state = None
+
+        async def capture(state):
+            nonlocal last_state
+            last_state = state
+
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=5, on_state_change=capture)
+
+        mock_crawler = create_mock_crawler_with_links(num_links=2)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        exported = strategy.export_state()
+        assert exported == last_state
+
+
+class TestDFSResume:
+    """DFS strategy resume tests."""
+
+    @pytest.mark.asyncio
+    async def test_state_export_includes_stack_and_dfs_seen(self):
+        """Verify DFS state includes stack structure and _dfs_seen."""
+        captured_states: List[Dict] = []
+
+        async def capture_state(state: Dict[str, Any]):
+            captured_states.append(state)
+
+        strategy = DFSDeepCrawlStrategy(
+            max_depth=3,
+            max_pages=10,
+            on_state_change=capture_state,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=2)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert len(captured_states) > 0
+
+        for state in captured_states:
+            assert state["strategy_type"] == "dfs"
+            assert "stack" in state
+            assert "dfs_seen" in state
+            # Stack items should have depth
+            for item in state["stack"]:
+                assert "url" in item
+                assert "parent_url" in item
+                assert "depth" in item
+
+    @pytest.mark.asyncio
+    async def test_resume_restores_stack_order(self):
+        """Verify DFS stack order is preserved on resume."""
+        saved_state = {
+            "strategy_type": "dfs",
+            "visited": ["https://example.com"],
+            "stack": [
+                {"url": "https://example.com/deep3", "parent_url": "https://example.com/deep2", "depth": 3},
+                {"url": "https://example.com/deep2", "parent_url": "https://example.com/deep1", "depth": 2},
+                {"url": "https://example.com/page1", "parent_url": "https://example.com", "depth": 1},
+            ],
+            "depths": {"https://example.com": 0},
+            "pages_crawled": 1,
+            "dfs_seen": ["https://example.com", "https://example.com/deep3", "https://example.com/deep2", "https://example.com/page1"],
+        }
+
+        crawl_order: List[str] = []
+
+        strategy = DFSDeepCrawlStrategy(
+            max_depth=3,
+            max_pages=10,
+            resume_state=saved_state,
+        )
+
+        mock_crawler = create_mock_crawler_tracking(crawl_order, return_no_links=True)
+        mock_config = create_mock_config()
+
+        await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # DFS pops from end of stack, so order should be: page1, deep2, deep3
+        assert crawl_order[0] == "https://example.com/page1"
+        assert crawl_order[1] == "https://example.com/deep2"
+        assert crawl_order[2] == "https://example.com/deep3"
+
+
+class TestBestFirstResume:
+    """Best-First strategy resume tests."""
+
+    @pytest.mark.asyncio
+    async def test_state_export_includes_scored_queue(self):
+        """Verify Best-First state includes queue with scores."""
+        captured_states: List[Dict] = []
+
+        async def capture_state(state: Dict[str, Any]):
+            captured_states.append(state)
+
+        scorer = KeywordRelevanceScorer(keywords=["important"], weight=1.0)
+
+        strategy = BestFirstCrawlingStrategy(
+            max_depth=2,
+            max_pages=10,
+            url_scorer=scorer,
+            on_state_change=capture_state,
+        )
+
+        mock_crawler = create_mock_crawler_with_links(num_links=3, include_keyword=True)
+        mock_config = create_mock_config(stream=True)
+
+        async for _ in strategy._arun_stream("https://example.com", mock_crawler, mock_config):
+            pass
+
+        assert len(captured_states) > 0
+
+        for state in captured_states:
+            assert state["strategy_type"] == "best_first"
+            assert "queue_items" in state
+            for item in state["queue_items"]:
+                assert "score" in item
+                assert "depth" in item
+                assert "url" in item
+                assert "parent_url" in item
+
+    @pytest.mark.asyncio
+    async def test_resume_maintains_priority_order(self):
+        """Verify priority queue order is maintained on resume."""
+        saved_state = {
+            "strategy_type": "best_first",
+            "visited": ["https://example.com"],
+            "queue_items": [
+                {"score": -0.9, "depth": 1, "url": "https://example.com/high-priority", "parent_url": "https://example.com"},
+                {"score": -0.5, "depth": 1, "url": "https://example.com/medium-priority", "parent_url": "https://example.com"},
+                {"score": -0.1, "depth": 1, "url": "https://example.com/low-priority", "parent_url": "https://example.com"},
+            ],
+            "depths": {"https://example.com": 0},
+            "pages_crawled": 1,
+        }
+
+        crawl_order: List[str] = []
+
+        strategy = BestFirstCrawlingStrategy(
+            max_depth=2,
+            max_pages=10,
+            resume_state=saved_state,
+        )
+
+        mock_crawler = create_mock_crawler_tracking(crawl_order, return_no_links=True)
+        mock_config = create_mock_config(stream=True)
+
+        async for _ in strategy._arun_stream("https://example.com", mock_crawler, mock_config):
+            pass
+
+        # Higher negative score = higher priority (min-heap)
+        # So -0.9 should be crawled first
+        assert crawl_order[0] == "https://example.com/high-priority"
+
+    @pytest.mark.asyncio
+    async def test_stream_results_follow_priority_order_with_out_of_order_batch(self):
+        saved_state = {
+            "strategy_type": "best_first",
+            "visited": ["https://example.com"],
+            "queue_items": [
+                {"score": -0.9, "depth": 1, "url": "https://example.com/high", "parent_url": "https://example.com"},
+                {"score": -0.1, "depth": 1, "url": "https://example.com/low", "parent_url": "https://example.com"},
+            ],
+            "depths": {"https://example.com": 0},
+            "pages_crawled": 1,
+        }
+
+        async def mock_arun_many(urls, config):
+            async def gen():
+                for url in reversed(urls):
+                    result = MagicMock(url=url, success=True, metadata={})
+                    result.links = {"internal": [], "external": []}
+                    yield result
+            return gen()
+
+        strategy = BestFirstCrawlingStrategy(max_depth=2, max_pages=10, resume_state=saved_state)
+        mock_crawler = MagicMock(arun_many=mock_arun_many)
+        results = [
+            result.url
+            async for result in strategy._arun_stream("https://example.com", mock_crawler, create_mock_config(stream=True))
+        ]
+
+        assert results[:2] == ["https://example.com/high", "https://example.com/low"]
+
+
+
+class TestCrossStrategyResume:
+    """Tests that apply to all strategies."""
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("strategy_class,strategy_type", [
+        (BFSDeepCrawlStrategy, "bfs"),
+        (DFSDeepCrawlStrategy, "dfs"),
+        (BestFirstCrawlingStrategy, "best_first"),
+    ])
+    async def test_no_callback_means_no_overhead(self, strategy_class, strategy_type):
+        """Verify no state tracking when callback is None."""
+        strategy = strategy_class(max_depth=2, max_pages=5)
+
+        # _queue_shadow should be None for Best-First when no callback
+        if strategy_class == BestFirstCrawlingStrategy:
+            assert strategy._queue_shadow is None
+
+        # _last_state should be None initially
+        assert strategy._last_state is None
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("strategy_class", [
+        BFSDeepCrawlStrategy,
+        DFSDeepCrawlStrategy,
+        BestFirstCrawlingStrategy,
+    ])
+    async def test_export_state_returns_last_captured(self, strategy_class):
+        """Verify export_state() returns last captured state."""
+        last_state = None
+
+        async def capture(state):
+            nonlocal last_state
+            last_state = state
+
+        strategy = strategy_class(max_depth=2, max_pages=5, on_state_change=capture)
+
+        mock_crawler = create_mock_crawler_with_links(num_links=2)
+
+        if strategy_class == BestFirstCrawlingStrategy:
+            mock_config = create_mock_config(stream=True)
+            async for _ in strategy._arun_stream("https://example.com", mock_crawler, mock_config):
+                pass
+        else:
+            mock_config = create_mock_config()
+            await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        exported = strategy.export_state()
+        assert exported == last_state
+
+
+# ============================================================================
+# TEST SUITE 2: Regression Tests (No Damage to Current System)
+# ============================================================================
+
+class TestBFSRegressions:
+    """Ensure BFS works identically when new params not used."""
+
+    @pytest.mark.asyncio
+    async def test_default_params_unchanged(self):
+        """Constructor with only original params works."""
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=2,
+            include_external=False,
+            max_pages=10,
+        )
+
+        assert strategy.max_depth == 2
+        assert strategy.include_external == False
+        assert strategy.max_pages == 10
+        assert strategy._resume_state is None
+        assert strategy._on_state_change is None
+
+    @pytest.mark.asyncio
+    async def test_filter_chain_still_works(self):
+        """FilterChain integration unchanged."""
+        filter_chain = FilterChain([
+            URLPatternFilter(patterns=["*/blog/*"]),
+            DomainFilter(allowed_domains=["example.com"]),
+        ])
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=2,
+            filter_chain=filter_chain,
+        )
+
+        # Test filter still applies
+        assert await strategy.can_process_url("https://example.com/blog/post1", 1) == True
+        assert await strategy.can_process_url("https://other.com/blog/post1", 1) == False
+
+    @pytest.mark.asyncio
+    async def test_url_scorer_still_works(self):
+        """URL scoring integration unchanged."""
+        scorer = KeywordRelevanceScorer(keywords=["python", "tutorial"], weight=1.0)
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=2,
+            url_scorer=scorer,
+            score_threshold=0.5,
+        )
+
+        assert strategy.url_scorer is not None
+        assert strategy.score_threshold == 0.5
+
+        # Scorer should work
+        score = scorer.score("https://example.com/python-tutorial")
+        assert score > 0
+
+    @pytest.mark.asyncio
+    async def test_batch_mode_returns_list(self):
+        """Batch mode still returns List[CrawlResult]."""
+        strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=5)
+
+        mock_crawler = create_simple_mock_crawler()
+        mock_config = create_mock_config(stream=False)
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert isinstance(results, list)
+        assert len(results) > 0
+
+    @pytest.mark.asyncio
+    async def test_max_pages_limit_respected(self):
+        """max_pages limit still enforced."""
+        strategy = BFSDeepCrawlStrategy(max_depth=10, max_pages=3)
+
+        mock_crawler = create_mock_crawler_unlimited_links()
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # Should stop at max_pages
+        assert strategy._pages_crawled <= 3
+
+    @pytest.mark.asyncio
+    async def test_max_depth_limit_respected(self):
+        """max_depth limit still enforced."""
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=100)
+
+        mock_crawler = create_mock_crawler_unlimited_links()
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # All results should have depth <= max_depth
+        for result in results:
+            assert result.metadata.get("depth", 0) <= 2
+
+    @pytest.mark.asyncio
+    async def test_metadata_depth_still_set(self):
+        """Result metadata still includes depth."""
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=5)
+
+        mock_crawler = create_simple_mock_crawler()
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        for result in results:
+            assert "depth" in result.metadata
+            assert isinstance(result.metadata["depth"], int)
+
+    @pytest.mark.asyncio
+    async def test_metadata_parent_url_still_set(self):
+        """Result metadata still includes parent_url."""
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=5)
+
+        mock_crawler = create_simple_mock_crawler()
+        mock_config = create_mock_config()
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        # First result (start URL) should have parent_url = None
+        assert results[0].metadata.get("parent_url") is None
+
+        # Child results should have parent_url set
+        for result in results[1:]:
+            assert "parent_url" in result.metadata
+
+
+class TestDFSRegressions:
+    """Ensure DFS works identically when new params not used."""
+
+    @pytest.mark.asyncio
+    async def test_inherits_bfs_params(self):
+        """DFS still inherits all BFS parameters."""
+        strategy = DFSDeepCrawlStrategy(
+            max_depth=3,
+            include_external=True,
+            max_pages=20,
+            score_threshold=0.5,
+        )
+
+        assert strategy.max_depth == 3
+        assert strategy.include_external == True
+        assert strategy.max_pages == 20
+        assert strategy.score_threshold == 0.5
+
+    @pytest.mark.asyncio
+    async def test_dfs_seen_initialized(self):
+        """DFS _dfs_seen set still initialized."""
+        strategy = DFSDeepCrawlStrategy(max_depth=2)
+
+        assert hasattr(strategy, '_dfs_seen')
+        assert isinstance(strategy._dfs_seen, set)
+
+
+class TestBestFirstRegressions:
+    """Ensure Best-First works identically when new params not used."""
+
+    @pytest.mark.asyncio
+    async def test_default_params_unchanged(self):
+        """Constructor with only original params works."""
+        strategy = BestFirstCrawlingStrategy(
+            max_depth=2,
+            include_external=False,
+            max_pages=10,
+        )
+
+        assert strategy.max_depth == 2
+        assert strategy.include_external == False
+        assert strategy.max_pages == 10
+        assert strategy._resume_state is None
+        assert strategy._on_state_change is None
+        assert strategy._queue_shadow is None  # Not initialized without callback
+
+    @pytest.mark.asyncio
+    async def test_scorer_integration(self):
+        """URL scorer still affects crawl priority."""
+        scorer = KeywordRelevanceScorer(keywords=["important"], weight=1.0)
+
+        strategy = BestFirstCrawlingStrategy(
+            max_depth=2,
+            max_pages=10,
+            url_scorer=scorer,
+        )
+
+        assert strategy.url_scorer is scorer
+
+
+class TestMaxPagesBoundaryYielded:
+    """best_first must YIELD exactly max_pages results (boundary page kept).
+
+    Regression for #859: best_first used to break BEFORE yielding the page that
+    hit the limit, so it returned max_pages-1 results while BFS already returned
+    max_pages. The downstream scan count (discovered_urls) is derived from the
+    yielded results, so the off-by-one surfaced as "11 requested -> 10 found".
+    BFS is included as the working reference. (DFS batch mode has a separate,
+    unrelated overshoot quirk and is intentionally not asserted here.)
+    """
+
+    @pytest.mark.parametrize(
+        "strategy_class",
+        [BFSDeepCrawlStrategy, BestFirstCrawlingStrategy],
+    )
+    @pytest.mark.asyncio
+    async def test_yields_exactly_max_pages(self, strategy_class):
+        max_pages = 11
+        strategy = strategy_class(max_depth=10, max_pages=max_pages)
+
+        mock_crawler = create_mock_crawler_unlimited_links()
+        # _arun_batch is the path the scan worker takes (crawler.arun with the
+        # default stream=False). best_first's _arun_batch re-clones with
+        # stream=True internally, so clone must honour the requested flag.
+        mock_config = create_mock_config()
+        mock_config.clone = lambda **kw: create_mock_config(stream=kw.get("stream", False))
+
+        results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+        assert len(results) == max_pages, (
+            f"{strategy_class.__name__} yielded {len(results)} results, "
+            f"expected exactly max_pages={max_pages}"
+        )
+
+
+class TestAPICompatibility:
+    """Ensure API/serialization compatibility."""
+
+    def test_strategy_signature_backward_compatible(self):
+        """Old code calling with positional/keyword args still works."""
+        # Positional args (old style)
+        s1 = BFSDeepCrawlStrategy(2)
+        assert s1.max_depth == 2
+
+        # Keyword args (old style)
+        s2 = BFSDeepCrawlStrategy(max_depth=3, max_pages=10)
+        assert s2.max_depth == 3
+
+        # Mixed (old style)
+        s3 = BFSDeepCrawlStrategy(2, FilterChain(), None, False, float('-inf'), 100)
+        assert s3.max_depth == 2
+        assert s3.max_pages == 100
+
+    def test_no_required_new_params(self):
+        """New params are optional, not required."""
+        # Should not raise
+        BFSDeepCrawlStrategy(max_depth=2)
+        DFSDeepCrawlStrategy(max_depth=2)
+        BestFirstCrawlingStrategy(max_depth=2)
diff --git a/tests/deep_crawling/test_deep_crawl_resume_integration.py b/tests/deep_crawling/test_deep_crawl_resume_integration.py
new file mode 100644
index 0000000..1c7a864
--- /dev/null
+++ b/tests/deep_crawling/test_deep_crawl_resume_integration.py
@@ -0,0 +1,162 @@
+"""
+Integration Test: Deep Crawl Resume with Real URLs
+
+Tests the crash recovery feature using books.toscrape.com - a site
+designed for scraping practice with a clear hierarchy:
+- Home page → Category pages → Book detail pages
+"""
+
+import pytest
+import asyncio
+import json
+from typing import Dict, Any, List
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+
+class TestBFSResumeIntegration:
+    """Integration tests for BFS resume with real crawling."""
+
+    @pytest.mark.asyncio
+    async def test_real_crawl_state_capture_and_resume(self):
+        """
+        Test crash recovery with real URLs from books.toscrape.com.
+
+        Flow:
+        1. Start crawl with state callback
+        2. Stop after N pages (simulated crash)
+        3. Resume from saved state
+        4. Verify no duplicate crawls
+        """
+        # Phase 1: Initial crawl that "crashes" after 3 pages
+        crash_after = 3
+        captured_states: List[Dict[str, Any]] = []
+        crawled_urls_phase1: List[str] = []
+
+        async def capture_state_until_crash(state: Dict[str, Any]):
+            captured_states.append(state)
+            crawled_urls_phase1.clear()
+            crawled_urls_phase1.extend(state["visited"])
+
+            if state["pages_crawled"] >= crash_after:
+                raise Exception("Simulated crash!")
+
+        strategy1 = BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=10,
+            on_state_change=capture_state_until_crash,
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=strategy1,
+            stream=False,
+            verbose=False,
+        )
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            # First crawl - will crash after 3 pages
+            with pytest.raises(Exception, match="Simulated crash"):
+                await crawler.arun("https://books.toscrape.com", config=config)
+
+        # Verify we captured state before crash
+        assert len(captured_states) > 0, "No states captured before crash"
+        last_state = captured_states[-1]
+
+        print(f"\n=== Phase 1: Crashed after {last_state['pages_crawled']} pages ===")
+        print(f"Visited URLs: {len(last_state['visited'])}")
+        print(f"Pending URLs: {len(last_state['pending'])}")
+
+        # Verify state structure
+        assert last_state["strategy_type"] == "bfs"
+        assert last_state["pages_crawled"] >= crash_after
+        assert len(last_state["visited"]) > 0
+        assert "pending" in last_state
+        assert "depths" in last_state
+
+        # Verify state is JSON serializable (important for Redis/DB storage)
+        json_str = json.dumps(last_state)
+        restored_state = json.loads(json_str)
+        assert restored_state == last_state, "State not JSON round-trip safe"
+
+        # Phase 2: Resume from checkpoint
+        crawled_urls_phase2: List[str] = []
+
+        async def track_resumed_crawl(state: Dict[str, Any]):
+            # Track what's being crawled in phase 2
+            new_visited = set(state["visited"]) - set(last_state["visited"])
+            for url in new_visited:
+                if url not in crawled_urls_phase2:
+                    crawled_urls_phase2.append(url)
+
+        strategy2 = BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=10,
+            resume_state=restored_state,
+            on_state_change=track_resumed_crawl,
+        )
+
+        config2 = CrawlerRunConfig(
+            deep_crawl_strategy=strategy2,
+            stream=False,
+            verbose=False,
+        )
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            results = await crawler.arun("https://books.toscrape.com", config=config2)
+
+        print(f"\n=== Phase 2: Resumed crawl ===")
+        print(f"New URLs crawled: {len(crawled_urls_phase2)}")
+        print(f"Final pages_crawled: {strategy2._pages_crawled}")
+
+        # Verify no duplicates - URLs from phase 1 should not be re-crawled
+        already_crawled = set(last_state["visited"]) - {item["url"] for item in last_state["pending"]}
+        duplicates = set(crawled_urls_phase2) & already_crawled
+
+        assert len(duplicates) == 0, f"Duplicate crawls detected: {duplicates}"
+
+        # Verify we made progress (crawled some of the pending URLs)
+        pending_urls = {item["url"] for item in last_state["pending"]}
+        crawled_pending = set(crawled_urls_phase2) & pending_urls
+
+        print(f"Pending URLs crawled in phase 2: {len(crawled_pending)}")
+
+        # Final state should show more pages crawled than before crash
+        final_state = strategy2.export_state()
+        if final_state:
+            assert final_state["pages_crawled"] >= last_state["pages_crawled"], \
+                "Resume did not make progress"
+
+        print("\n=== Integration test PASSED ===")
+
+    @pytest.mark.asyncio
+    async def test_state_export_method(self):
+        """Test that export_state() returns valid state during crawl."""
+        states_from_callback: List[Dict] = []
+
+        async def capture(state):
+            states_from_callback.append(state)
+
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=1,
+            max_pages=3,
+            on_state_change=capture,
+        )
+
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=strategy,
+            stream=False,
+            verbose=False,
+        )
+
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            await crawler.arun("https://books.toscrape.com", config=config)
+
+        # export_state should return the last captured state
+        exported = strategy.export_state()
+
+        assert exported is not None, "export_state() returned None"
+        assert exported == states_from_callback[-1], "export_state() doesn't match last callback"
+
+        print(f"\n=== export_state() test PASSED ===")
+        print(f"Final state: {exported['pages_crawled']} pages, {len(exported['visited'])} visited")
diff --git a/tests/deep_crwaling/test_filter.py b/tests/deep_crwaling/test_filter.py
new file mode 100644
index 0000000..29ada08
--- /dev/null
+++ b/tests/deep_crwaling/test_filter.py
@@ -0,0 +1,75 @@
+# // File: tests/deep_crawling/test_filters.py
+import pytest
+from urllib.parse import urlparse
+from crawl4ai import ContentTypeFilter, URLFilter 
+
+# Minimal URLFilter base class stub if not already importable directly for tests
+# In a real scenario, this would be imported from the library
+if not hasattr(URLFilter, '_update_stats'): # Check if it's a basic stub
+    class URLFilter: # Basic stub for testing if needed
+        def __init__(self, name=None): self.name = name
+        def apply(self, url: str) -> bool: raise NotImplementedError
+        def _update_stats(self, passed: bool): pass # Mock implementation
+
+# Assume ContentTypeFilter is structured as discussed. If its definition is not fully
+# available for direct import in the test environment, a more elaborate stub or direct
+# instantiation of the real class (if possible) would be needed.
+# For this example, we assume ContentTypeFilter can be imported and used.
+
+class TestContentTypeFilter:
+    @pytest.mark.parametrize(
+        "url, allowed_types, expected",
+        [
+            # Existing tests (examples)
+            ("http://example.com/page.html", ["text/html"], True),
+            ("http://example.com/page.json", ["application/json"], True),
+            ("http://example.com/image.png", ["text/html"], False),
+            ("http://example.com/document.pdf", ["application/pdf"], True),
+            ("http://example.com/page", ["text/html"], True), # No extension, allowed
+            ("http://example.com/page", ["text/html"], False), # No extension, disallowed
+            ("http://example.com/page.unknown", ["text/html"], False), # Unknown extension
+            
+            # Tests for PHP extensions
+            ("http://example.com/index.php", ["application/x-httpd-php"], True),
+            ("http://example.com/script.php3", ["application/x-httpd-php"], True),
+            ("http://example.com/legacy.php4", ["application/x-httpd-php"], True),
+            ("http://example.com/main.php5", ["application/x-httpd-php"], True),
+            ("http://example.com/api.php7", ["application/x-httpd-php"], True),
+            ("http://example.com/index.phtml", ["application/x-httpd-php"], True),
+            ("http://example.com/source.phps", ["application/x-httpd-php-source"], True),
+
+            # Test rejection of PHP extensions
+            ("http://example.com/index.php", ["text/html"], False),
+            ("http://example.com/script.php3", ["text/plain"], False),
+            ("http://example.com/source.phps", ["application/x-httpd-php"], False), # Mismatch MIME
+            ("http://example.com/source.php", ["application/x-httpd-php-source"], False), # Mismatch MIME for .php
+
+            # Test case-insensitivity of extensions in URL
+            ("http://example.com/PAGE.HTML", ["text/html"], True),
+            ("http://example.com/INDEX.PHP", ["application/x-httpd-php"], True),
+            ("http://example.com/SOURCE.PHPS", ["application/x-httpd-php-source"], True),
+
+            # Test case-insensitivity of allowed_types
+            ("http://example.com/index.php", ["APPLICATION/X-HTTPD-PHP"], True),
+        ],
+    )
+    def test_apply(self, url, allowed_types, expected):
+        content_filter = ContentTypeFilter(
+            allowed_types=allowed_types
+        )
+        assert content_filter.apply(url) == expected
+
+    @pytest.mark.parametrize(
+        "url, expected_extension",
+        [
+            ("http://example.com/file.html", "html"),
+            ("http://example.com/file.tar.gz", "gz"),
+            ("http://example.com/path/", ""),
+            ("http://example.com/nodot", ""),
+            ("http://example.com/.config", "config"), # hidden file with extension
+            ("http://example.com/path/to/archive.BIG.zip", "zip"), # Case test
+        ]
+    )
+    def test_extract_extension(self, url, expected_extension):
+        # Test the static method directly
+        assert ContentTypeFilter._extract_extension(url) == expected_extension
diff --git a/tests/docker/simple_api_test.py b/tests/docker/simple_api_test.py
new file mode 100644
index 0000000..10fb232
--- /dev/null
+++ b/tests/docker/simple_api_test.py
@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+"""
+Simple API Test for Crawl4AI Docker Server v0.7.0
+Uses only built-in Python modules to test all endpoints.
+"""
+
+import urllib.request
+import urllib.parse
+import json
+import time
+import sys
+from typing import Dict, List, Optional
+
+# Configuration
+BASE_URL = "http://localhost:11234"  # Change to your server URL
+TEST_TIMEOUT = 30
+
+class SimpleApiTester:
+    def __init__(self, base_url: str = BASE_URL):
+        self.base_url = base_url
+        self.token = None
+        self.results = []
+        
+    def log(self, message: str):
+        print(f"[INFO] {message}")
+    
+    def test_get_endpoint(self, endpoint: str) -> Dict:
+        """Test a GET endpoint"""
+        url = f"{self.base_url}{endpoint}"
+        start_time = time.time()
+        
+        try:
+            req = urllib.request.Request(url)
+            if self.token:
+                req.add_header('Authorization', f'Bearer {self.token}')
+            
+            with urllib.request.urlopen(req, timeout=TEST_TIMEOUT) as response:
+                response_time = time.time() - start_time
+                status_code = response.getcode()
+                content = response.read().decode('utf-8')
+                
+                # Try to parse JSON
+                try:
+                    data = json.loads(content)
+                except:
+                    data = {"raw_response": content[:200]}
+                
+                return {
+                    "endpoint": endpoint,
+                    "method": "GET",
+                    "status": "PASS" if status_code < 400 else "FAIL",
+                    "status_code": status_code,
+                    "response_time": response_time,
+                    "data": data
+                }
+        except Exception as e:
+            response_time = time.time() - start_time
+            return {
+                "endpoint": endpoint,
+                "method": "GET",
+                "status": "FAIL",
+                "status_code": None,
+                "response_time": response_time,
+                "error": str(e)
+            }
+    
+    def test_post_endpoint(self, endpoint: str, payload: Dict) -> Dict:
+        """Test a POST endpoint"""
+        url = f"{self.base_url}{endpoint}"
+        start_time = time.time()
+        
+        try:
+            data = json.dumps(payload).encode('utf-8')
+            req = urllib.request.Request(url, data=data, method='POST')
+            req.add_header('Content-Type', 'application/json')
+            
+            if self.token:
+                req.add_header('Authorization', f'Bearer {self.token}')
+            
+            with urllib.request.urlopen(req, timeout=TEST_TIMEOUT) as response:
+                response_time = time.time() - start_time
+                status_code = response.getcode()
+                content = response.read().decode('utf-8')
+                
+                # Try to parse JSON
+                try:
+                    data = json.loads(content)
+                except:
+                    data = {"raw_response": content[:200]}
+                
+                return {
+                    "endpoint": endpoint,
+                    "method": "POST",
+                    "status": "PASS" if status_code < 400 else "FAIL",
+                    "status_code": status_code,
+                    "response_time": response_time,
+                    "data": data
+                }
+        except Exception as e:
+            response_time = time.time() - start_time
+            return {
+                "endpoint": endpoint,
+                "method": "POST",
+                "status": "FAIL",
+                "status_code": None,
+                "response_time": response_time,
+                "error": str(e)
+            }
+    
+    def print_result(self, result: Dict):
+        """Print a formatted test result"""
+        status_color = {
+            "PASS": "✅",
+            "FAIL": "❌",
+            "SKIP": "⏭️"
+        }
+        
+        print(f"{status_color[result['status']]} {result['method']} {result['endpoint']} "
+              f"| {result['response_time']:.3f}s | Status: {result['status_code'] or 'N/A'}")
+        
+        if result['status'] == 'FAIL' and 'error' in result:
+            print(f"    Error: {result['error']}")
+        
+        self.results.append(result)
+    
+    def run_all_tests(self):
+        """Run all API tests"""
+        print("🚀 Starting Crawl4AI v0.7.0 API Test Suite")
+        print(f"📡 Testing server at: {self.base_url}")
+        print("=" * 60)
+        
+        # # Test basic endpoints
+        # print("\n=== BASIC ENDPOINTS ===")
+        
+        # # Health check
+        # result = self.test_get_endpoint("/health")
+        # self.print_result(result)
+        
+        
+        # # Schema endpoint
+        # result = self.test_get_endpoint("/schema")
+        # self.print_result(result)
+        
+        # # Metrics endpoint
+        # result = self.test_get_endpoint("/metrics")
+        # self.print_result(result)
+        
+        # # Root redirect
+        # result = self.test_get_endpoint("/")
+        # self.print_result(result)
+        
+        # # Test authentication
+        # print("\n=== AUTHENTICATION ===")
+        
+        # # Get token
+        # token_payload = {"email": "test@example.com"}
+        # result = self.test_post_endpoint("/token", token_payload)
+        # self.print_result(result)
+        
+        # # Extract token if successful
+        # if result['status'] == 'PASS' and 'data' in result:
+        #     token = result['data'].get('access_token')
+        #     if token:
+        #         self.token = token
+        #         self.log(f"Successfully obtained auth token: {token[:20]}...")
+        
+        # Test core APIs
+        print("\n=== CORE APIs ===")
+        
+        test_url = "https://example.com"
+        test_raw_html_url = "raw://
      Hello, World!
      "
+        # Test markdown endpoint
+        md_payload = {
+            "url": test_url,
+            "f": "fit",
+            "q": "test query",
+            "c": "0"
+        }
+        result = self.test_post_endpoint("/md", md_payload)
+        # print(result['data'].get('markdown', ''))
+        self.print_result(result)
+        
+        # Test markdown endpoint with raw HTML
+        raw_md_payload = {
+            "url": test_raw_html_url,
+            "f": "fit",
+            "q": "test query",
+            "c": "0"
+        }
+        result = self.test_post_endpoint("/md", raw_md_payload)
+        self.print_result(result)
+        
+
+        # Test HTML endpoint
+        html_payload = {"url": test_url}
+        result = self.test_post_endpoint("/html", html_payload)
+        self.print_result(result)
+        
+        # Test screenshot endpoint
+        screenshot_payload = {
+            "url": test_url,
+            "screenshot_wait_for": 2
+        }
+        result = self.test_post_endpoint("/screenshot", screenshot_payload)
+        self.print_result(result)
+        
+        # Test PDF endpoint
+        pdf_payload = {"url": test_url}
+        result = self.test_post_endpoint("/pdf", pdf_payload)
+        self.print_result(result)
+        
+        # Test JavaScript execution
+        js_payload = {
+            "url": test_url,
+            "scripts": ["(() => document.title)()"]
+        }
+        result = self.test_post_endpoint("/execute_js", js_payload)
+        self.print_result(result)
+        
+        # Test crawl endpoint
+        crawl_payload = {
+            "urls": [test_url],
+            "browser_config": {},
+            "crawler_config": {}
+        }
+        result = self.test_post_endpoint("/crawl", crawl_payload)
+        self.print_result(result)
+        
+        # Test crawl endpoint with raw HTML
+        crawl_payload = {
+            "urls": [test_raw_html_url],
+            "browser_config": {},
+            "crawler_config": {}
+        }
+        result = self.test_post_endpoint("/crawl", crawl_payload)
+        self.print_result(result)
+
+        # Test config dump
+        config_payload = {"code": "CrawlerRunConfig()"}
+        result = self.test_post_endpoint("/config/dump", config_payload)
+        self.print_result(result)
+        
+        # Test LLM endpoint
+        llm_endpoint = f"/llm/{test_url}?q=Extract%20main%20content"
+        result = self.test_get_endpoint(llm_endpoint)
+        self.print_result(result)
+        
+        # Test ask endpoint
+        ask_endpoint = "/ask?context_type=all&query=crawl4ai&max_results=5"
+        result = self.test_get_endpoint(ask_endpoint)
+        print(result)
+        self.print_result(result)
+        
+        # Test job APIs
+        print("\n=== JOB APIs ===")
+        
+        # Test LLM job
+        llm_job_payload = {
+            "url": test_url,
+            "q": "Extract main content",
+            "cache": False
+        }
+        result = self.test_post_endpoint("/llm/job", llm_job_payload)
+        self.print_result(result)
+        
+        # Test crawl job
+        crawl_job_payload = {
+            "urls": [test_url],
+            "browser_config": {},
+            "crawler_config": {}
+        }
+        result = self.test_post_endpoint("/crawl/job", crawl_job_payload)
+        self.print_result(result)
+        
+        # Test MCP
+        print("\n=== MCP APIs ===")
+        
+        # Test MCP schema
+        result = self.test_get_endpoint("/mcp/schema")
+        self.print_result(result)
+        
+        # Test error handling
+        print("\n=== ERROR HANDLING ===")
+        
+        # Test invalid URL
+        invalid_payload = {"url": "invalid-url", "f": "fit"}
+        result = self.test_post_endpoint("/md", invalid_payload)
+        self.print_result(result)
+        
+        # Test invalid endpoint
+        result = self.test_get_endpoint("/nonexistent")
+        self.print_result(result)
+        
+        # Print summary
+        self.print_summary()
+    
+    def print_summary(self):
+        """Print test results summary"""
+        print("\n" + "=" * 60)
+        print("📊 TEST RESULTS SUMMARY")
+        print("=" * 60)
+        
+        total = len(self.results)
+        passed = sum(1 for r in self.results if r['status'] == 'PASS')
+        failed = sum(1 for r in self.results if r['status'] == 'FAIL')
+        
+        print(f"Total Tests: {total}")
+        print(f"✅ Passed: {passed}")
+        print(f"❌ Failed: {failed}")
+        print(f"📈 Success Rate: {(passed/total)*100:.1f}%")
+        
+        if failed > 0:
+            print("\n❌ FAILED TESTS:")
+            for result in self.results:
+                if result['status'] == 'FAIL':
+                    print(f"  • {result['method']} {result['endpoint']}")
+                    if 'error' in result:
+                        print(f"    Error: {result['error']}")
+        
+        # Performance statistics
+        response_times = [r['response_time'] for r in self.results if r['response_time'] > 0]
+        if response_times:
+            avg_time = sum(response_times) / len(response_times)
+            max_time = max(response_times)
+            print(f"\n⏱️  Average Response Time: {avg_time:.3f}s")
+            print(f"⏱️  Max Response Time: {max_time:.3f}s")
+        
+        # Save detailed report
+        report_file = f"crawl4ai_test_report_{int(time.time())}.json"
+        with open(report_file, 'w') as f:
+            json.dump({
+                "timestamp": time.time(),
+                "server_url": self.base_url,
+                "version": "0.7.0",
+                "summary": {
+                    "total": total,
+                    "passed": passed,
+                    "failed": failed
+                },
+                "results": self.results
+            }, f, indent=2)
+        
+        print(f"\n📄 Detailed report saved to: {report_file}")
+
+def main():
+    """Main test runner"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description='Crawl4AI v0.7.0 API Test Suite')
+    parser.add_argument('--url', default=BASE_URL, help='Base URL of the server')
+    
+    args = parser.parse_args()
+    
+    tester = SimpleApiTester(args.url)
+    
+    try:
+        tester.run_all_tests()
+    except KeyboardInterrupt:
+        print("\n🛑 Test suite interrupted by user")
+    except Exception as e:
+        print(f"\n💥 Test suite failed with error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tests/docker/test_config_object.py b/tests/docker/test_config_object.py
new file mode 100644
index 0000000..94a30f0
--- /dev/null
+++ b/tests/docker/test_config_object.py
@@ -0,0 +1,113 @@
+import json
+from crawl4ai import (
+    CrawlerRunConfig,
+    DefaultMarkdownGenerator,
+    RegexChunking,
+    JsonCssExtractionStrategy,
+    BM25ContentFilter,
+    CacheMode
+)
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+from crawl4ai.deep_crawling.filters import FastFilterChain
+from crawl4ai.deep_crawling.filters import FastContentTypeFilter, FastDomainFilter
+from crawl4ai.deep_crawling.scorers import FastKeywordRelevanceScorer
+
+def create_test_config() -> CrawlerRunConfig:
+    # Set up content filtering and markdown generation
+    content_filter = BM25ContentFilter(
+        user_query="technology articles",
+    )
+    
+    markdown_generator = DefaultMarkdownGenerator(
+        content_filter=content_filter,
+        options={"ignore_links": False, "body_width": 0}
+    )
+
+    # Set up extraction strategy
+    extraction_schema = {
+        "name": "ArticleExtractor",
+        "baseSelector": "article.content",
+        "fields": [
+            {"name": "title", "selector": "h1", "type": "text"},
+            {"name": "content", "selector": ".article-body", "type": "html"}
+        ]
+    }
+    extraction_strategy = JsonCssExtractionStrategy(schema=extraction_schema)
+
+    # Set up deep crawling
+    filter_chain = FastFilterChain([
+        FastContentTypeFilter(["text/html"]),
+        FastDomainFilter(blocked_domains=["ads.*"])
+    ])
+
+    url_scorer = FastKeywordRelevanceScorer(
+        keywords=["article", "blog"],
+        weight=1.0
+    )
+
+    deep_crawl_strategy = BFSDeepCrawlStrategy(
+        max_depth=3,
+        filter_chain=filter_chain,
+        url_scorer=url_scorer
+    )
+
+    # Create the config
+    config = CrawlerRunConfig(
+        word_count_threshold=200,
+        extraction_strategy=extraction_strategy,
+        chunking_strategy=RegexChunking(patterns=[r"\n\n"]),
+        markdown_generator=markdown_generator,
+        css_selector="main.content",
+        excluded_tags=["nav", "footer"],
+        keep_attrs=["href", "src"],
+        cache_mode=CacheMode.BYPASS,
+        wait_until="networkidle",
+        page_timeout=30000,
+        scan_full_page=True,
+        deep_crawl_strategy=deep_crawl_strategy,
+        verbose=True,
+        stream=True
+    )
+
+    return config
+
+def test_config_serialization_cycle():
+    # Create original config
+    original_config = create_test_config()
+    
+    # Dump to serializable dictionary
+    serialized = original_config.dump()
+
+    print(json.dumps(serialized, indent=2))
+    
+    # Load back into config object
+    deserialized_config = CrawlerRunConfig.load(serialized)
+    
+    # Verify core attributes
+    assert deserialized_config.word_count_threshold == original_config.word_count_threshold
+    assert deserialized_config.css_selector == original_config.css_selector
+    assert deserialized_config.excluded_tags == original_config.excluded_tags
+    assert deserialized_config.keep_attrs == original_config.keep_attrs
+    assert deserialized_config.cache_mode == original_config.cache_mode
+    assert deserialized_config.wait_until == original_config.wait_until
+    assert deserialized_config.page_timeout == original_config.page_timeout
+    assert deserialized_config.scan_full_page == original_config.scan_full_page
+    assert deserialized_config.verbose == original_config.verbose
+    assert deserialized_config.stream == original_config.stream
+
+    # Verify complex objects
+    assert isinstance(deserialized_config.extraction_strategy, JsonCssExtractionStrategy)
+    assert isinstance(deserialized_config.chunking_strategy, RegexChunking)
+    assert isinstance(deserialized_config.markdown_generator, DefaultMarkdownGenerator)
+    assert isinstance(deserialized_config.markdown_generator.content_filter, BM25ContentFilter)
+    assert isinstance(deserialized_config.deep_crawl_strategy, BFSDeepCrawlStrategy)
+    
+    # Verify deep crawl strategy configuration
+    assert deserialized_config.deep_crawl_strategy.max_depth == 3
+    assert isinstance(deserialized_config.deep_crawl_strategy.filter_chain, FastFilterChain)
+    assert isinstance(deserialized_config.deep_crawl_strategy.url_scorer, FastKeywordRelevanceScorer)
+
+    print("Serialization cycle test passed successfully!")
+
+if __name__ == "__main__":
+    test_config_serialization_cycle()
\ No newline at end of file
diff --git a/tests/docker/test_docker.py b/tests/docker/test_docker.py
new file mode 100644
index 0000000..87723a7
--- /dev/null
+++ b/tests/docker/test_docker.py
@@ -0,0 +1,186 @@
+import requests
+import time
+import httpx
+import asyncio
+from typing import Dict, Any
+from crawl4ai import (
+    BrowserConfig, CrawlerRunConfig, DefaultMarkdownGenerator,
+    PruningContentFilter, JsonCssExtractionStrategy, LLMContentFilter, CacheMode
+)
+from crawl4ai import LLMConfig
+from crawl4ai.docker_client import Crawl4aiDockerClient
+
+class Crawl4AiTester:
+    def __init__(self, base_url: str = "http://localhost:11235"):
+        self.base_url = base_url
+
+    def submit_and_wait(
+        self, request_data: Dict[str, Any], timeout: int = 300
+    ) -> Dict[str, Any]:
+        # Submit crawl job
+        response = requests.post(f"{self.base_url}/crawl", json=request_data)
+        task_id = response.json()["task_id"]
+        print(f"Task ID: {task_id}")
+
+        # Poll for result
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Task {task_id} did not complete within {timeout} seconds"
+                )
+
+            result = requests.get(f"{self.base_url}/task/{task_id}")
+            status = result.json()
+
+            if status["status"] == "failed":
+                print("Task failed:", status.get("error"))
+                raise Exception(f"Task failed: {status.get('error')}")
+
+            if status["status"] == "completed":
+                return status
+
+            time.sleep(2)
+
+async def test_direct_api():
+    """Test direct API endpoints without using the client SDK"""
+    print("\n=== Testing Direct API Calls ===")
+    
+    # Test 1: Basic crawl with content filtering
+    browser_config = BrowserConfig(
+        headless=True,
+        viewport_width=1200,
+        viewport_height=800
+    )
+    
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(
+                threshold=0.48,
+                threshold_type="fixed",
+                min_word_threshold=0
+            ),
+            options={"ignore_links": True}
+        )
+    )
+
+    request_data = {
+        "urls": ["https://example.com"],
+        "browser_config": browser_config.dump(),
+        "crawler_config": crawler_config.dump()
+    }
+
+    # Make direct API call
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            "http://localhost:11235/crawl",
+            json=request_data,
+            timeout=300
+        )
+        assert response.status_code == 200
+        result = response.json()
+        print("Basic crawl result:", result["success"])
+
+    # Test 2: Structured extraction with JSON CSS
+    schema = {
+        "baseSelector": "article.post",
+        "fields": [
+            {"name": "title", "selector": "h1", "type": "text"},
+            {"name": "content", "selector": ".content", "type": "html"}
+        ]
+    }
+
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=JsonCssExtractionStrategy(schema=schema)
+    )
+
+    request_data["crawler_config"] = crawler_config.dump()
+
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            "http://localhost:11235/crawl",
+            json=request_data
+        )
+        assert response.status_code == 200
+        result = response.json()
+        print("Structured extraction result:", result["success"])
+
+    # Test 3: Raw HTML
+    request_data["urls"] = ["raw://
      Hello, World!
      Example"]
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            "http://localhost:11235/crawl",
+            json=request_data
+        )
+        assert response.status_code == 200
+        result = response.json()
+        print("Raw HTML result:", result["success"])
+
+    # Test 3: Get schema
+    # async with httpx.AsyncClient() as client:
+    #     response = await client.get("http://localhost:8000/schema")
+    #     assert response.status_code == 200
+    #     schemas = response.json()
+    #     print("Retrieved schemas for:", list(schemas.keys()))
+
+async def test_with_client():
+    """Test using the Crawl4AI Docker client SDK"""
+    print("\n=== Testing Client SDK ===")
+    
+    async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client:
+        # Test 1: Basic crawl
+        browser_config = BrowserConfig(headless=True)
+        crawler_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48,
+                    threshold_type="fixed"
+                )
+            )
+        )
+
+        result = await client.crawl(
+            urls=["https://example.com"],
+            browser_config=browser_config,
+            crawler_config=crawler_config
+        )
+        print("Client SDK basic crawl:", result.success)
+
+        # Test 2: LLM extraction with streaming
+        crawler_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=LLMContentFilter(
+                    llm_config=LLMConfig(provider="openai/gpt-40"),
+                    instruction="Extract key technical concepts"
+                )
+            ),
+            stream=True
+        )
+
+        async for result in await client.crawl(
+            urls=["https://example.com"],
+            browser_config=browser_config,
+            crawler_config=crawler_config
+        ):
+            print(f"Streaming result for: {result.url}")
+
+        # # Test 3: Get schema
+        # schemas = await client.get_schema()
+        # print("Retrieved client schemas for:", list(schemas.keys()))
+
+async def main():
+    """Run all tests"""
+    # Test direct API
+    print("Testing direct API calls...")
+    await test_direct_api()
+
+    # Test client SDK
+    print("\nTesting client SDK...")
+    await test_with_client()
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/docker/test_dockerclient.py b/tests/docker/test_dockerclient.py
new file mode 100644
index 0000000..cba6c4c
--- /dev/null
+++ b/tests/docker/test_dockerclient.py
@@ -0,0 +1,34 @@
+import asyncio
+from crawl4ai.docker_client import Crawl4aiDockerClient
+from crawl4ai import (
+    BrowserConfig,
+    CrawlerRunConfig
+)
+
+async def main():
+    async with Crawl4aiDockerClient(base_url="http://localhost:8000", verbose=True) as client:
+        await client.authenticate("test@example.com")
+        
+        # Non-streaming crawl
+        results = await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig()
+        )
+        print(f"Non-streaming results: {results}")
+        
+        # Streaming crawl
+        crawler_config = CrawlerRunConfig(stream=True)
+        async for result in await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=crawler_config
+        ):
+            print(f"Streamed result: {result}")
+        
+        # Get schema
+        schema = await client.get_schema()
+        print(f"Schema: {schema}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/docker/test_filter_deep_crawl.py b/tests/docker/test_filter_deep_crawl.py
new file mode 100644
index 0000000..9e82073
--- /dev/null
+++ b/tests/docker/test_filter_deep_crawl.py
@@ -0,0 +1,245 @@
+"""
+Test the complete fix for both the filter serialization and JSON serialization issues.
+"""
+import os
+import traceback
+from typing import Any
+
+import asyncio
+import httpx
+
+from crawl4ai import BrowserConfig, CacheMode, CrawlerRunConfig
+from crawl4ai.deep_crawling import (
+    BFSDeepCrawlStrategy,
+    ContentRelevanceFilter,
+    FilterChain,
+    URLFilter,
+    URLPatternFilter,
+)
+
+CRAWL4AI_DOCKER_PORT = os.environ.get("CRAWL4AI_DOCKER_PORT", "11234")
+try:
+    BASE_PORT = int(CRAWL4AI_DOCKER_PORT)
+except TypeError:
+    BASE_PORT = 11234
+BASE_URL = f"http://localhost:{BASE_PORT}/"  # Adjust port as needed
+
+
+async def test_with_docker_client(filter_chain: list[URLFilter], max_pages: int = 20, timeout: int = 30) -> bool:
+    """Test using the Docker client (same as 1419.py)."""
+    from crawl4ai.docker_client import Crawl4aiDockerClient
+    
+    print("=" * 60)
+    print("Testing with Docker Client")
+    print("=" * 60)
+    
+    try:
+        async with Crawl4aiDockerClient(
+            base_url=BASE_URL,
+            verbose=True,
+        ) as client:
+            
+            crawler_config = CrawlerRunConfig(
+                deep_crawl_strategy=BFSDeepCrawlStrategy(
+                    max_depth=2,  # Keep it shallow for testing
+                    max_pages=max_pages,  # Limit pages for testing
+                    filter_chain=FilterChain(filter_chain)
+                ),
+                cache_mode=CacheMode.BYPASS,
+            )
+            
+            print("\n1. Testing crawl with filters...")
+            results = await client.crawl(
+                ["https://docs.crawl4ai.com"],  # Simple test page
+                browser_config=BrowserConfig(headless=True),
+                crawler_config=crawler_config,
+                hooks_timeout=timeout,
+            )
+            
+            if results:
+                print(f"✅ Crawl succeeded! Type: {type(results)}")
+                if hasattr(results, 'success'):
+                    print(f"✅ Results success: {results.success}")
+                    # Test that we can iterate results without JSON errors
+                    if hasattr(results, '__iter__'):
+                        for i, result in enumerate(results):
+                            if hasattr(result, 'url'):
+                                print(f"   Result {i}: {result.url[:50]}...")
+                            else:
+                                print(f"   Result {i}: {str(result)[:50]}...")
+                else:
+                    # Handle list of results
+                    print(f"✅ Got {len(results)} results")
+                    for i, result in enumerate(results[:3]):  # Show first 3
+                        print(f"   Result {i}: {result.url[:50]}...")
+            else:
+                print("❌ Crawl failed - no results returned")
+                return False
+                
+        print("\n✅ Docker client test completed successfully!")
+        return True
+        
+    except Exception as e:
+        print(f"❌ Docker client test failed: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_with_rest_api(filters: list[dict[str, Any]], max_pages: int = 20, timeout: int = 30) -> bool:
+    """Test using REST API directly."""
+    print("\n" + "=" * 60)
+    print("Testing with REST API")
+    print("=" * 60)
+    
+    # Create filter configuration
+    deep_crawl_strategy_payload = {
+        "type": "BFSDeepCrawlStrategy",
+        "params": {
+            "max_depth": 2,
+            "max_pages": max_pages,
+            "filter_chain": {
+                "type": "FilterChain",
+                "params": {
+                    "filters": filters
+                }
+            }
+        }
+    }
+    
+    crawl_payload = {
+        "urls": ["https://docs.crawl4ai.com"],
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "deep_crawl_strategy": deep_crawl_strategy_payload,
+                "cache_mode": "bypass"
+            }
+        }
+    }
+    
+    try:
+        async with httpx.AsyncClient() as client:
+            print("\n1. Sending crawl request to REST API...")
+            response = await client.post(
+                f"{BASE_URL}crawl",
+                json=crawl_payload,
+                timeout=timeout,
+            )
+            
+            if response.status_code == 200:
+                print(f"✅ REST API returned 200 OK")
+                data = response.json()
+                if data.get("success"):
+                    results = data.get("results", [])
+                    print(f"✅ Got {len(results)} results")
+                    for i, result in enumerate(results[:3]):
+                        print(f"   Result {i}: {result.get('url', 'unknown')[:50]}...")
+                else:
+                    print(f"❌ Crawl not successful: {data}")
+                    return False
+            else:
+                print(f"❌ REST API returned {response.status_code}")
+                print(f"   Response: {response.text[:500]}")
+                return False
+                
+        print("\n✅ REST API test completed successfully!")
+        return True
+        
+    except Exception as e:
+        print(f"❌ REST API test failed: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def main():
+    """Run all tests."""
+    print("\n🧪 TESTING COMPLETE FIX FOR DOCKER FILTER AND JSON ISSUES")
+    print("=" * 60)
+    print("Make sure the server is running with the updated code!")
+    print("=" * 60)
+    
+    results = []
+    
+    # Test 1: Docker client
+    max_pages_ = [20, 5]
+    timeouts = [30, 60]
+    filter_chain_test_cases = [
+        [
+            URLPatternFilter(
+                # patterns=["*about*", "*privacy*", "*terms*"],
+                patterns=["*advanced*"],
+                reverse=True
+            ),
+        ],
+        [
+            ContentRelevanceFilter(
+                query="about faq",
+                threshold=0.2,
+            ),
+        ],
+    ]
+    for idx, (filter_chain, max_pages, timeout) in enumerate(zip(filter_chain_test_cases, max_pages_, timeouts)):
+        docker_passed = await test_with_docker_client(filter_chain=filter_chain, max_pages=max_pages, timeout=timeout)
+        results.append((f"Docker Client w/ filter chain {idx}", docker_passed))
+    
+    # Test 2: REST API
+    max_pages_ = [20, 5, 5]
+    timeouts = [30, 60, 60]
+    filters_test_cases = [
+        [
+            {
+                "type": "URLPatternFilter",
+                "params": {
+                    "patterns": ["*advanced*"],
+                    "reverse": True
+                }
+            }
+        ],
+        [
+            {
+                "type": "ContentRelevanceFilter",
+                "params": {
+                    "query": "about faq",
+                    "threshold": 0.2,
+                }
+            }
+        ],
+        [
+            {
+                "type": "ContentRelevanceFilter",
+                "params": {
+                    "query": ["about", "faq"],
+                    "threshold": 0.2,
+                }
+            }
+        ],
+    ]
+    for idx, (filters, max_pages, timeout) in enumerate(zip(filters_test_cases, max_pages_, timeouts)):
+        rest_passed = await test_with_rest_api(filters=filters, max_pages=max_pages, timeout=timeout)
+        results.append((f"REST API w/ filters {idx}", rest_passed))
+    
+    # Summary
+    print("\n" + "=" * 60)
+    print("FINAL TEST SUMMARY")
+    print("=" * 60)
+    
+    all_passed = True
+    for test_name, passed in results:
+        status = "✅ PASSED" if passed else "❌ FAILED"
+        print(f"{test_name:20} {status}")
+        if not passed:
+            all_passed = False
+    
+    print("=" * 60)
+    if all_passed:
+        print("🎉 ALL TESTS PASSED!")
+    else:
+        print("⚠️ Some tests failed. Please check the server logs for details.")
+    
+    return 0 if all_passed else 1
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(asyncio.run(main()))
diff --git a/tests/docker/test_hooks_client.py b/tests/docker/test_hooks_client.py
new file mode 100644
index 0000000..bfac353
--- /dev/null
+++ b/tests/docker/test_hooks_client.py
@@ -0,0 +1,372 @@
+#!/usr/bin/env python3
+"""
+Test client for demonstrating user-provided hooks in Crawl4AI Docker API
+"""
+
+import requests
+import json
+from typing import Dict, Any
+
+
+API_BASE_URL = "http://localhost:11234"  # Adjust if needed
+
+
+def test_hooks_info():
+    """Get information about available hooks"""
+    print("=" * 70)
+    print("Testing: GET /hooks/info")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE_URL}/hooks/info")
+    if response.status_code == 200:
+        data = response.json()
+        print("Available Hook Points:")
+        for hook, info in data['available_hooks'].items():
+            print(f"\n{hook}:")
+            print(f"  Parameters: {', '.join(info['parameters'])}")
+            print(f"  Description: {info['description']}")
+    else:
+        print(f"Error: {response.status_code}")
+        print(response.text)
+
+
+def test_basic_crawl_with_hooks():
+    """Test basic crawling with user-provided hooks"""
+    print("\n" + "=" * 70)
+    print("Testing: POST /crawl with hooks")
+    print("=" * 70)
+    
+    # Define hooks as Python code strings
+    hooks_code = {
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("Hook: Setting up page context")
+    # Block images to speed up crawling
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    print("Hook: Images blocked")
+    return page
+""",
+        
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    print("Hook: Before retrieving HTML")
+    # Scroll to bottom to load lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    print("Hook: Scrolled to bottom")
+    return page
+""",
+        
+        "before_goto": """
+async def hook(page, context, url, **kwargs):
+    print(f"Hook: About to navigate to {url}")
+    # Add custom headers
+    await page.set_extra_http_headers({
+        'X-Test-Header': 'crawl4ai-hooks-test'
+    })
+    return page
+"""
+    }
+    
+    # Create request payload
+    payload = {
+        "urls": ["https://httpbin.org/html"],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 30
+        }
+    }
+    
+    print("Sending request with hooks...")
+    response = requests.post(f"{API_BASE_URL}/crawl", json=payload)
+    
+    if response.status_code == 200:
+        data = response.json()
+        print("\n✅ Crawl successful!")
+        
+        # Check hooks status
+        if 'hooks' in data:
+            hooks_info = data['hooks']
+            print("\nHooks Execution Summary:")
+            print(f"  Status: {hooks_info['status']['status']}")
+            print(f"  Attached hooks: {', '.join(hooks_info['status']['attached_hooks'])}")
+            
+            if hooks_info['status']['validation_errors']:
+                print("\n⚠️ Validation Errors:")
+                for error in hooks_info['status']['validation_errors']:
+                    print(f"  - {error['hook_point']}: {error['error']}")
+            
+            if 'summary' in hooks_info:
+                summary = hooks_info['summary']
+                print(f"\nExecution Statistics:")
+                print(f"  Total executions: {summary['total_executions']}")
+                print(f"  Successful: {summary['successful']}")
+                print(f"  Failed: {summary['failed']}")
+                print(f"  Timed out: {summary['timed_out']}")
+                print(f"  Success rate: {summary['success_rate']:.1f}%")
+            
+            if hooks_info['execution_log']:
+                print("\nExecution Log:")
+                for log_entry in hooks_info['execution_log']:
+                    status_icon = "✅" if log_entry['status'] == 'success' else "❌"
+                    print(f"  {status_icon} {log_entry['hook_point']}: {log_entry['status']} ({log_entry.get('execution_time', 0):.2f}s)")
+            
+            if hooks_info['errors']:
+                print("\n❌ Hook Errors:")
+                for error in hooks_info['errors']:
+                    print(f"  - {error['hook_point']}: {error['error']}")
+        
+        # Show crawl results
+        if 'results' in data:
+            print(f"\nCrawled {len(data['results'])} URL(s)")
+            for result in data['results']:
+                print(f"  - {result['url']}: {'✅' if result['success'] else '❌'}")
+    
+    else:
+        print(f"❌ Error: {response.status_code}")
+        print(response.text)
+
+
+def test_invalid_hook():
+    """Test with an invalid hook to see error handling"""
+    print("\n" + "=" * 70)
+    print("Testing: Invalid hook handling")
+    print("=" * 70)
+    
+    # Intentionally broken hook
+    hooks_code = {
+        "on_page_context_created": """
+def hook(page, context):  # Missing async!
+    return page
+""",
+        
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    # This will cause an error
+    await page.non_existent_method()
+    return page
+"""
+    }
+    
+    payload = {
+        "urls": ["https://httpbin.org/html"],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 5
+        }
+    }
+    
+    print("Sending request with invalid hooks...")
+    response = requests.post(f"{API_BASE_URL}/crawl", json=payload)
+    
+    if response.status_code == 200:
+        data = response.json()
+        
+        if 'hooks' in data:
+            hooks_info = data['hooks']
+            print(f"\nHooks Status: {hooks_info['status']['status']}")
+            
+            if hooks_info['status']['validation_errors']:
+                print("\n✅ Validation caught errors (as expected):")
+                for error in hooks_info['status']['validation_errors']:
+                    print(f"  - {error['hook_point']}: {error['error']}")
+            
+            if hooks_info['errors']:
+                print("\n✅ Runtime errors handled gracefully:")
+                for error in hooks_info['errors']:
+                    print(f"  - {error['hook_point']}: {error['error']}")
+            
+            # The crawl should still succeed despite hook errors
+            if data.get('success'):
+                print("\n✅ Crawl succeeded despite hook errors (error isolation working!)")
+    
+    else:
+        print(f"Error: {response.status_code}")
+        print(response.text)
+
+
+def test_authentication_hook():
+    """Test authentication using hooks"""
+    print("\n" + "=" * 70)
+    print("Testing: Authentication with hooks")
+    print("=" * 70)
+    
+    hooks_code = {
+        "before_goto": """
+async def hook(page, context, url, **kwargs):
+    # For httpbin.org basic auth test, set Authorization header
+    import base64
+    
+    # httpbin.org/basic-auth/user/passwd expects username="user" and password="passwd"
+    credentials = base64.b64encode(b"user:passwd").decode('ascii')
+    
+    await page.set_extra_http_headers({
+        'Authorization': f'Basic {credentials}'
+    })
+    
+    print(f"Hook: Set Authorization header for {url}")
+    return page
+""",
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    # Example: Add cookies for session tracking
+    await context.add_cookies([
+        {
+            'name': 'session_id',
+            'value': 'test_session_123',
+            'domain': '.httpbin.org',
+            'path': '/',
+            'httpOnly': True,
+            'secure': True
+        }
+    ])
+    
+    print("Hook: Added session cookie")
+    return page
+"""
+    }
+    
+    payload = {
+        "urls": ["https://httpbin.org/basic-auth/user/passwd"],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 30
+        }
+    }
+    
+    print("Sending request with authentication hook...")
+    response = requests.post(f"{API_BASE_URL}/crawl", json=payload)
+    
+    if response.status_code == 200:
+        data = response.json()
+        if data.get('success'):
+            print("✅ Crawl with authentication hook successful")
+            
+            # Check if hooks executed
+            if 'hooks' in data:
+                hooks_info = data['hooks']
+                if hooks_info.get('summary', {}).get('successful', 0) > 0:
+                    print(f"✅ Authentication hooks executed: {hooks_info['summary']['successful']} successful")
+                
+                # Check for any hook errors
+                if hooks_info.get('errors'):
+                    print("⚠️ Hook errors:")
+                    for error in hooks_info['errors']:
+                        print(f"  - {error}")
+            
+            # Check if authentication worked by looking at the result
+            if 'results' in data and len(data['results']) > 0:
+                result = data['results'][0]
+                if result.get('success'):
+                    print("✅ Page crawled successfully (authentication worked!)")
+                    # httpbin.org/basic-auth returns JSON with authenticated=true when successful
+                    if 'authenticated' in str(result.get('html', '')):
+                        print("✅ Authentication confirmed in response content")
+                else:
+                    print(f"❌ Crawl failed: {result.get('error_message', 'Unknown error')}")
+        else:
+            print("❌ Request failed")
+            print(f"Response: {json.dumps(data, indent=2)}")
+    else:
+        print(f"❌ Error: {response.status_code}")
+        try:
+            error_data = response.json()
+            print(f"Error details: {json.dumps(error_data, indent=2)}")
+        except:
+            print(f"Error text: {response.text[:500]}")
+
+
+def test_streaming_with_hooks():
+    """Test streaming endpoint with hooks"""
+    print("\n" + "=" * 70)
+    print("Testing: POST /crawl/stream with hooks")
+    print("=" * 70)
+    
+    hooks_code = {
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    await page.evaluate("document.querySelectorAll('img').forEach(img => img.remove())")
+    return page
+"""
+    }
+    
+    payload = {
+        "urls": ["https://httpbin.org/html", "https://httpbin.org/json"],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 10
+        }
+    }
+    
+    print("Sending streaming request with hooks...")
+    
+    with requests.post(f"{API_BASE_URL}/crawl/stream", json=payload, stream=True) as response:
+        if response.status_code == 200:
+            # Check headers for hooks status
+            hooks_status = response.headers.get('X-Hooks-Status')
+            if hooks_status:
+                print(f"Hooks Status (from header): {hooks_status}")
+            
+            print("\nStreaming results:")
+            for line in response.iter_lines():
+                if line:
+                    try:
+                        result = json.loads(line)
+                        if 'url' in result:
+                            print(f"  Received: {result['url']}")
+                        elif 'status' in result:
+                            print(f"  Stream status: {result['status']}")
+                    except json.JSONDecodeError:
+                        print(f"  Raw: {line.decode()}")
+        else:
+            print(f"Error: {response.status_code}")
+
+
+def test_basic_without_hooks():
+    """Test basic crawl without hooks"""
+    print("\n" + "=" * 70)
+    print("Testing: POST /crawl with no hooks")
+    print("=" * 70)
+
+    payload = {
+        "urls": ["https://httpbin.org/html", "https://httpbin.org/json"]
+    }
+
+    response = requests.post(f"{API_BASE_URL}/crawl", json=payload)
+    if response.status_code == 200:
+        data = response.json()
+        print(f"Response: {json.dumps(data, indent=2)}")
+    else:
+        print(f"Error: {response.status_code}")
+
+
+def main():
+    """Run all tests"""
+    print("🔧 Crawl4AI Docker API - Hooks Testing")
+    print("=" * 70)
+    
+    # Test 1: Get hooks information
+    # test_hooks_info()
+    
+    # Test 2: Basic crawl with hooks
+    # test_basic_crawl_with_hooks()
+    
+    # Test 3: Invalid hooks (error handling)
+    test_invalid_hook()
+    
+    # # Test 4: Authentication hook
+    # test_authentication_hook()
+    
+    # # Test 5: Streaming with hooks
+    # test_streaming_with_hooks()
+
+    # # Test 6: Basic crawl without hooks
+    # test_basic_without_hooks()
+
+    print("\n" + "=" * 70)
+    print("✅ All tests completed!")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tests/docker/test_hooks_comprehensive.py b/tests/docker/test_hooks_comprehensive.py
new file mode 100644
index 0000000..2c89271
--- /dev/null
+++ b/tests/docker/test_hooks_comprehensive.py
@@ -0,0 +1,558 @@
+#!/usr/bin/env python3
+"""
+Comprehensive test demonstrating all hook types from hooks_example.py
+adapted for the Docker API with real URLs
+"""
+
+import requests
+import json
+import time
+from typing import Dict, Optional
+
+API_BASE_URL = "http://localhost:11235"
+
+# Global token storage
+_auth_token: Optional[str] = None
+
+
+def get_auth_token(email: str = "test@gmail.com") -> str:
+    """
+    Get a JWT token from the /token endpoint.
+    The email domain must have valid MX records.
+    """
+    global _auth_token
+
+    if _auth_token:
+        return _auth_token
+
+    print(f"🔐 Requesting JWT token for {email}...")
+    response = requests.post(
+        f"{API_BASE_URL}/token",
+        json={"email": email}
+    )
+
+    if response.status_code == 200:
+        data = response.json()
+        _auth_token = data["access_token"]
+        print(f"✅ Token obtained successfully")
+        return _auth_token
+    else:
+        raise Exception(f"Failed to get token: {response.status_code} - {response.text}")
+
+
+def get_auth_headers() -> Dict[str, str]:
+    """Get headers with JWT Bearer token."""
+    token = get_auth_token()
+    return {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json"
+    }
+
+
+def test_all_hooks_demo():
+    """Demonstrate all 8 hook types with practical examples"""
+    print("=" * 70)
+    print("Testing: All Hooks Comprehensive Demo")
+    print("=" * 70)
+    
+    hooks_code = {
+        "on_browser_created": """
+async def hook(browser, **kwargs):
+    # Hook called after browser is created
+    print("[HOOK] on_browser_created - Browser is ready!")
+    # Browser-level configurations would go here
+    return browser
+""",
+        
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    # Hook called after a new page and context are created
+    print("[HOOK] on_page_context_created - New page created!")
+    
+    # Set viewport size for consistent rendering
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    
+    # Add cookies for the session (using httpbin.org domain)
+    await context.add_cookies([
+        {
+            "name": "test_session",
+            "value": "abc123xyz",
+            "domain": ".httpbin.org",
+            "path": "/",
+            "httpOnly": True,
+            "secure": True
+        }
+    ])
+    
+    # Block ads and tracking scripts to speed up crawling
+    await context.route("**/*.{png,jpg,jpeg,gif,webp,svg}", lambda route: route.abort())
+    await context.route("**/analytics/*", lambda route: route.abort())
+    await context.route("**/ads/*", lambda route: route.abort())
+    
+    print("[HOOK] Viewport set, cookies added, and ads blocked")
+    return page
+""",
+        
+        "on_user_agent_updated": """
+async def hook(page, context, user_agent, **kwargs):
+    # Hook called when user agent is updated
+    print(f"[HOOK] on_user_agent_updated - User agent: {user_agent[:50]}...")
+    return page
+""",
+        
+        "before_goto": """
+async def hook(page, context, url, **kwargs):
+    # Hook called before navigating to each URL
+    print(f"[HOOK] before_goto - About to visit: {url}")
+    
+    # Add custom headers for the request
+    await page.set_extra_http_headers({
+        "X-Custom-Header": "crawl4ai-test",
+        "Accept-Language": "en-US,en;q=0.9",
+        "DNT": "1"
+    })
+    
+    return page
+""",
+        
+        "after_goto": """
+async def hook(page, context, url, response, **kwargs):
+    # Hook called after navigating to each URL
+    print(f"[HOOK] after_goto - Successfully loaded: {url}")
+    
+    # Wait a moment for dynamic content to load
+    await page.wait_for_timeout(1000)
+    
+    # Check if specific elements exist (with error handling)
+    try:
+        # For httpbin.org, wait for body element
+        await page.wait_for_selector("body", timeout=2000)
+        print("[HOOK] Body element found and loaded")
+    except:
+        print("[HOOK] Timeout waiting for body, continuing anyway")
+    
+    return page
+""",
+        
+        "on_execution_started": """
+async def hook(page, context, **kwargs):
+    # Hook called after custom JavaScript execution
+    print("[HOOK] on_execution_started - Custom JS executed!")
+    
+    # You could inject additional JavaScript here if needed
+    await page.evaluate("console.log('[INJECTED] Hook JS running');")
+    
+    return page
+""",
+        
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    # Hook called before retrieving the HTML content
+    print("[HOOK] before_retrieve_html - Preparing to get HTML")
+    
+    # Scroll to bottom to trigger lazy loading
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+    await page.wait_for_timeout(500)
+    
+    # Scroll back to top
+    await page.evaluate("window.scrollTo(0, 0);")
+    await page.wait_for_timeout(500)
+    
+    # One more scroll to middle for good measure
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2);")
+    
+    print("[HOOK] Scrolling completed for lazy-loaded content")
+    return page
+""",
+        
+        "before_return_html": """
+async def hook(page, context, html, **kwargs):
+    # Hook called before returning the HTML content
+    print(f"[HOOK] before_return_html - HTML length: {len(html)} characters")
+    
+    # Log some page metrics
+    metrics = await page.evaluate('''() => {
+        return {
+            images: document.images.length,
+            links: document.links.length,
+            scripts: document.scripts.length
+        }
+    }''')
+    
+    print(f"[HOOK] Page metrics - Images: {metrics['images']}, Links: {metrics['links']}, Scripts: {metrics['scripts']}")
+    
+    return page
+"""
+    }
+    
+    # Create request payload
+    payload = {
+        "urls": ["https://httpbin.org/html"],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 30
+        },
+        "crawler_config": {
+            "js_code": "window.scrollTo(0, document.body.scrollHeight);",
+            "wait_for": "body",
+            "cache_mode": "bypass"
+        }
+    }
+    
+    print("\nSending request with all 8 hooks...")
+    start_time = time.time()
+
+    response = requests.post(f"{API_BASE_URL}/crawl", json=payload, headers=get_auth_headers())
+    
+    elapsed_time = time.time() - start_time
+    print(f"Request completed in {elapsed_time:.2f} seconds")
+    
+    if response.status_code == 200:
+        data = response.json()
+        print("\n✅ Request successful!")
+        
+        # Check hooks execution
+        if 'hooks' in data:
+            hooks_info = data['hooks']
+            print("\n📊 Hooks Execution Summary:")
+            print(f"  Status: {hooks_info['status']['status']}")
+            print(f"  Attached hooks: {len(hooks_info['status']['attached_hooks'])}")
+            
+            for hook_name in hooks_info['status']['attached_hooks']:
+                print(f"    ✓ {hook_name}")
+            
+            if 'summary' in hooks_info:
+                summary = hooks_info['summary']
+                print(f"\n📈 Execution Statistics:")
+                print(f"  Total executions: {summary['total_executions']}")
+                print(f"  Successful: {summary['successful']}")
+                print(f"  Failed: {summary['failed']}")
+                print(f"  Timed out: {summary['timed_out']}")
+                print(f"  Success rate: {summary['success_rate']:.1f}%")
+            
+            if hooks_info.get('execution_log'):
+                print(f"\n📝 Execution Log:")
+                for log_entry in hooks_info['execution_log']:
+                    status_icon = "✅" if log_entry['status'] == 'success' else "❌"
+                    exec_time = log_entry.get('execution_time', 0)
+                    print(f"  {status_icon} {log_entry['hook_point']}: {exec_time:.3f}s")
+        
+        # Check crawl results
+        if 'results' in data and len(data['results']) > 0:
+            print(f"\n📄 Crawl Results:")
+            for result in data['results']:
+                print(f"  URL: {result['url']}")
+                print(f"  Success: {result.get('success', False)}")
+                if result.get('html'):
+                    print(f"  HTML length: {len(result['html'])} characters")
+    
+    else:
+        print(f"❌ Error: {response.status_code}")
+        try:
+            error_data = response.json()
+            print(f"Error details: {json.dumps(error_data, indent=2)}")
+        except:
+            print(f"Error text: {response.text[:500]}")
+
+
+def test_authentication_flow():
+    """Test a complete authentication flow with multiple hooks"""
+    print("\n" + "=" * 70)
+    print("Testing: Authentication Flow with Multiple Hooks")
+    print("=" * 70)
+    
+    hooks_code = {
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("[HOOK] Setting up authentication context")
+    
+    # Add authentication cookies
+    await context.add_cookies([
+        {
+            "name": "auth_token",
+            "value": "fake_jwt_token_here",
+            "domain": ".httpbin.org",
+            "path": "/",
+            "httpOnly": True,
+            "secure": True
+        }
+    ])
+    
+    # Set localStorage items (for SPA authentication)
+    await page.evaluate('''
+        localStorage.setItem('user_id', '12345');
+        localStorage.setItem('auth_time', new Date().toISOString());
+    ''')
+    
+    return page
+""",
+        
+        "before_goto": """
+async def hook(page, context, url, **kwargs):
+    print(f"[HOOK] Adding auth headers for {url}")
+    
+    # Add Authorization header
+    import base64
+    credentials = base64.b64encode(b"user:passwd").decode('ascii')
+    
+    await page.set_extra_http_headers({
+        'Authorization': f'Basic {credentials}',
+        'X-API-Key': 'test-api-key-123'
+    })
+    
+    return page
+"""
+    }
+    
+    payload = {
+        "urls": [
+            "https://httpbin.org/basic-auth/user/passwd"
+        ],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 15
+        }
+    }
+    
+    print("\nTesting authentication with httpbin endpoints...")
+    response = requests.post(f"{API_BASE_URL}/crawl", json=payload, headers=get_auth_headers())
+    
+    if response.status_code == 200:
+        data = response.json()
+        print("✅ Authentication test completed")
+        
+        if 'results' in data:
+            for i, result in enumerate(data['results']):
+                print(f"\n  URL {i+1}: {result['url']}")
+                if result.get('success'):
+                    # Check for authentication success indicators
+                    html_content = result.get('html', '')
+                    if '"authenticated"' in html_content and 'true' in html_content:
+                        print("    ✅ Authentication successful! Basic auth worked.")
+                    else:
+                        print("    ⚠️ Page loaded but auth status unclear")
+                else:
+                    print(f"    ❌ Failed: {result.get('error_message', 'Unknown error')}")
+    else:
+        print(f"❌ Error: {response.status_code}")
+
+
+def test_performance_optimization_hooks():
+    """Test hooks for performance optimization"""
+    print("\n" + "=" * 70)
+    print("Testing: Performance Optimization Hooks")
+    print("=" * 70)
+    
+    hooks_code = {
+        "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("[HOOK] Optimizing page for performance")
+    
+    # Block resource-heavy content
+    await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
+    await context.route("**/*.{woff,woff2,ttf,otf}", lambda route: route.abort())
+    await context.route("**/*.{mp4,webm,ogg,mp3,wav}", lambda route: route.abort())
+    await context.route("**/googletagmanager.com/*", lambda route: route.abort())
+    await context.route("**/google-analytics.com/*", lambda route: route.abort())
+    await context.route("**/doubleclick.net/*", lambda route: route.abort())
+    await context.route("**/facebook.com/*", lambda route: route.abort())
+    
+    # Disable animations and transitions
+    await page.add_style_tag(content='''
+        *, *::before, *::after {
+            animation-duration: 0s !important;
+            animation-delay: 0s !important;
+            transition-duration: 0s !important;
+            transition-delay: 0s !important;
+        }
+    ''')
+    
+    print("[HOOK] Performance optimizations applied")
+    return page
+""",
+        
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    print("[HOOK] Removing unnecessary elements before extraction")
+    
+    # Remove ads, popups, and other unnecessary elements
+    await page.evaluate('''() => {
+        // Remove common ad containers
+        const adSelectors = [
+            '.ad', '.ads', '.advertisement', '[id*="ad-"]', '[class*="ad-"]',
+            '.popup', '.modal', '.overlay', '.cookie-banner', '.newsletter-signup'
+        ];
+        
+        adSelectors.forEach(selector => {
+            document.querySelectorAll(selector).forEach(el => el.remove());
+        });
+        
+        // Remove script tags to clean up HTML
+        document.querySelectorAll('script').forEach(el => el.remove());
+        
+        // Remove style tags we don't need
+        document.querySelectorAll('style').forEach(el => el.remove());
+    }''')
+    
+    return page
+"""
+    }
+    
+    payload = {
+        "urls": ["https://httpbin.org/html"],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 10
+        }
+    }
+    
+    print("\nTesting performance optimization hooks...")
+    start_time = time.time()
+
+    response = requests.post(f"{API_BASE_URL}/crawl", json=payload, headers=get_auth_headers())
+    
+    elapsed_time = time.time() - start_time
+    print(f"Request completed in {elapsed_time:.2f} seconds")
+    
+    if response.status_code == 200:
+        data = response.json()
+        print("✅ Performance optimization test completed")
+        
+        if 'results' in data and len(data['results']) > 0:
+            result = data['results'][0]
+            if result.get('html'):
+                print(f"  HTML size: {len(result['html'])} characters")
+                print("  Resources blocked, ads removed, animations disabled")
+    else:
+        print(f"❌ Error: {response.status_code}")
+
+
+def test_content_extraction_hooks():
+    """Test hooks for intelligent content extraction"""
+    print("\n" + "=" * 70)
+    print("Testing: Content Extraction Hooks")
+    print("=" * 70)
+    
+    hooks_code = {
+        "after_goto": """
+async def hook(page, context, url, response, **kwargs):
+    print(f"[HOOK] Waiting for dynamic content on {url}")
+    
+    # Wait for any lazy-loaded content
+    await page.wait_for_timeout(2000)
+    
+    # Trigger any "Load More" buttons
+    try:
+        load_more = await page.query_selector('[class*="load-more"], [class*="show-more"], button:has-text("Load More")')
+        if load_more:
+            await load_more.click()
+            await page.wait_for_timeout(1000)
+            print("[HOOK] Clicked 'Load More' button")
+    except:
+        pass
+    
+    return page
+""",
+        
+        "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    print("[HOOK] Extracting structured data")
+    
+    # Extract metadata
+    metadata = await page.evaluate('''() => {
+        const getMeta = (name) => {
+            const element = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`);
+            return element ? element.getAttribute('content') : null;
+        };
+        
+        return {
+            title: document.title,
+            description: getMeta('description') || getMeta('og:description'),
+            author: getMeta('author'),
+            keywords: getMeta('keywords'),
+            ogTitle: getMeta('og:title'),
+            ogImage: getMeta('og:image'),
+            canonical: document.querySelector('link[rel="canonical"]')?.href,
+            jsonLd: Array.from(document.querySelectorAll('script[type="application/ld+json"]'))
+                .map(el => el.textContent).filter(Boolean)
+        };
+    }''')
+    
+    print(f"[HOOK] Extracted metadata: {json.dumps(metadata, indent=2)}")
+    
+    # Infinite scroll handling
+    for i in range(3):
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+        await page.wait_for_timeout(1000)
+        print(f"[HOOK] Scroll iteration {i+1}/3")
+    
+    return page
+"""
+    }
+    
+    payload = {
+        "urls": ["https://httpbin.org/html", "https://httpbin.org/json"],
+        "hooks": {
+            "code": hooks_code,
+            "timeout": 20
+        }
+    }
+    
+    print("\nTesting content extraction hooks...")
+    response = requests.post(f"{API_BASE_URL}/crawl", json=payload, headers=get_auth_headers())
+    
+    if response.status_code == 200:
+        data = response.json()
+        print("✅ Content extraction test completed")
+        
+        if 'hooks' in data and 'summary' in data['hooks']:
+            summary = data['hooks']['summary']
+            print(f"  Hooks executed: {summary['successful']}/{summary['total_executions']}")
+        
+        if 'results' in data:
+            for result in data['results']:
+                print(f"\n  URL: {result['url']}")
+                print(f"  Success: {result.get('success', False)}")
+    else:
+        print(f"❌ Error: {response.status_code}")
+
+
+def main():
+    """Run comprehensive hook tests"""
+    print("🔧 Crawl4AI Docker API - Comprehensive Hooks Testing")
+    print("Based on docs/examples/hooks_example.py")
+    print("=" * 70)
+
+    # Get JWT token first (required when jwt_enabled=true)
+    try:
+        get_auth_token()
+        print("=" * 70)
+    except Exception as e:
+        print(f"❌ Failed to authenticate: {e}")
+        print("Make sure the server is running and jwt_enabled is configured correctly.")
+        return
+
+    tests = [
+        ("All Hooks Demo", test_all_hooks_demo),
+        ("Authentication Flow", test_authentication_flow),
+        ("Performance Optimization", test_performance_optimization_hooks),
+        ("Content Extraction", test_content_extraction_hooks),
+    ]
+    
+    for i, (name, test_func) in enumerate(tests, 1):
+        print(f"\n📌 Test {i}/{len(tests)}: {name}")
+        try:
+            test_func()
+            print(f"✅ {name} completed")
+        except Exception as e:
+            print(f"❌ {name} failed: {e}")
+            import traceback
+            traceback.print_exc()
+    
+    print("\n" + "=" * 70)
+    print("🎉 All comprehensive hook tests completed!")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tests/docker/test_hooks_utility.py b/tests/docker/test_hooks_utility.py
new file mode 100644
index 0000000..7c820e5
--- /dev/null
+++ b/tests/docker/test_hooks_utility.py
@@ -0,0 +1,193 @@
+"""
+Test script demonstrating the hooks_to_string utility and Docker client integration.
+"""
+import asyncio
+from crawl4ai import Crawl4aiDockerClient, hooks_to_string
+
+
+# Define hook functions as regular Python functions
+async def auth_hook(page, context, **kwargs):
+    """Add authentication cookies."""
+    await context.add_cookies([{
+        'name': 'test_cookie',
+        'value': 'test_value',
+        'domain': '.httpbin.org',
+        'path': '/'
+    }])
+    return page
+
+
+async def scroll_hook(page, context, **kwargs):
+    """Scroll to load lazy content."""
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    return page
+
+
+async def viewport_hook(page, context, **kwargs):
+    """Set custom viewport."""
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+
+
+async def test_hooks_utility():
+    """Test the hooks_to_string utility function."""
+    print("=" * 60)
+    print("Testing hooks_to_string utility")
+    print("=" * 60)
+
+    # Create hooks dictionary with function objects
+    hooks_dict = {
+        "on_page_context_created": auth_hook,
+        "before_retrieve_html": scroll_hook
+    }
+
+    # Convert to string format
+    hooks_string = hooks_to_string(hooks_dict)
+
+    print("\n✓ Successfully converted function objects to strings")
+    print(f"\n✓ Converted {len(hooks_string)} hooks:")
+    for hook_name in hooks_string.keys():
+        print(f"  - {hook_name}")
+
+    print("\n✓ Preview of converted hook:")
+    print("-" * 60)
+    print(hooks_string["on_page_context_created"][:200] + "...")
+    print("-" * 60)
+
+    return hooks_string
+
+
+async def test_docker_client_with_functions():
+    """Test Docker client with function objects (automatic conversion)."""
+    print("\n" + "=" * 60)
+    print("Testing Docker Client with Function Objects")
+    print("=" * 60)
+
+    # Note: This requires a running Crawl4AI Docker server
+    # Uncomment the following to test with actual server:
+
+    async with Crawl4aiDockerClient(base_url="http://localhost:11234", verbose=True) as client:
+        # Pass function objects directly - they'll be converted automatically
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks={
+                "on_page_context_created": auth_hook,
+                "before_retrieve_html": scroll_hook
+            },
+            hooks_timeout=30
+        )
+        print(f"\n✓ Crawl successful: {result.success}")
+        print(f"✓ URL: {result.url}")
+
+    print("\n✓ Docker client accepts function objects directly")
+    print("✓ Automatic conversion happens internally")
+    print("✓ No manual string formatting needed!")
+
+
+async def test_docker_client_with_strings():
+    """Test Docker client with pre-converted strings."""
+    print("\n" + "=" * 60)
+    print("Testing Docker Client with String Hooks")
+    print("=" * 60)
+
+    # Convert hooks to strings first
+    hooks_dict = {
+        "on_page_context_created": viewport_hook,
+        "before_retrieve_html": scroll_hook
+    }
+    hooks_string = hooks_to_string(hooks_dict)
+
+    # Note: This requires a running Crawl4AI Docker server
+    # Uncomment the following to test with actual server:
+
+    async with Crawl4aiDockerClient(base_url="http://localhost:11234", verbose=True) as client:
+        # Pass string hooks - they'll be used as-is
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks=hooks_string,
+            hooks_timeout=30
+        )
+        print(f"\n✓ Crawl successful: {result.success}")
+
+    print("\n✓ Docker client also accepts pre-converted strings")
+    print("✓ Backward compatible with existing code")
+
+
+async def show_usage_patterns():
+    """Show different usage patterns."""
+    print("\n" + "=" * 60)
+    print("Usage Patterns")
+    print("=" * 60)
+
+    print("\n1. Direct function usage (simplest):")
+    print("-" * 60)
+    print("""
+    async def my_hook(page, context, **kwargs):
+        await page.set_viewport_size({"width": 1920, "height": 1080})
+        return page
+
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks={"on_page_context_created": my_hook}
+    )
+    """)
+
+    print("\n2. Convert then use:")
+    print("-" * 60)
+    print("""
+    hooks_dict = {"on_page_context_created": my_hook}
+    hooks_string = hooks_to_string(hooks_dict)
+
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks=hooks_string
+    )
+    """)
+
+    print("\n3. Manual string (backward compatible):")
+    print("-" * 60)
+    print("""
+    hooks_string = {
+        "on_page_context_created": '''
+async def hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+'''
+    }
+
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks=hooks_string
+    )
+    """)
+
+
+async def main():
+    """Run all tests."""
+    print("\n🚀 Crawl4AI Hooks Utility Test Suite\n")
+
+    # Test the utility function
+    # await test_hooks_utility()
+
+    # Show usage with Docker client
+    # await test_docker_client_with_functions()
+    await test_docker_client_with_strings()
+
+    # Show different patterns
+    # await show_usage_patterns()
+
+    # print("\n" + "=" * 60)
+    # print("✓ All tests completed successfully!")
+    # print("=" * 60)
+    # print("\nKey Benefits:")
+    # print("  • Write hooks as regular Python functions")
+    # print("  • IDE support with autocomplete and type checking")
+    # print("  • Automatic conversion to API format")
+    # print("  • Backward compatible with string hooks")
+    # print("  • Same utility used everywhere")
+    # print("\n")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/docker/test_llm_params.py b/tests/docker/test_llm_params.py
new file mode 100755
index 0000000..533c448
--- /dev/null
+++ b/tests/docker/test_llm_params.py
@@ -0,0 +1,349 @@
+#!/usr/bin/env python3
+"""
+Test script for LLM temperature and base_url parameters in Crawl4AI Docker API.
+This demonstrates the new hierarchical configuration system:
+1. Request-level parameters (highest priority)
+2. Provider-specific environment variables
+3. Global environment variables
+4. System defaults (lowest priority)
+"""
+
+import asyncio
+import httpx
+import json
+import os
+from rich.console import Console
+from rich.panel import Panel
+from rich.syntax import Syntax
+from rich.table import Table
+
+
+console = Console()
+
+# Configuration
+BASE_URL = "http://localhost:11235"  # Docker API endpoint
+TEST_URL = "https://httpbin.org/html"     # Simple test page
+
+# --- Helper Functions ---
+
+async def check_server_health(client: httpx.AsyncClient) -> bool:
+    """Check if the server is healthy."""
+    console.print("[bold cyan]Checking server health...[/]", end="")
+    try:
+        response = await client.get("/health", timeout=10.0)
+        response.raise_for_status()
+        console.print(" [bold green]✓ Server is healthy![/]")
+        return True
+    except Exception as e:
+        console.print(f"\n[bold red]✗ Server health check failed: {e}[/]")
+        console.print(f"Is the server running at {BASE_URL}?")
+        return False
+
+def print_request(endpoint: str, payload: dict, title: str = "Request"):
+    """Pretty print the request."""
+    syntax = Syntax(json.dumps(payload, indent=2), "json", theme="monokai")
+    console.print(Panel.fit(
+        f"[cyan]POST {endpoint}[/cyan]\n{syntax}",
+        title=f"[bold blue]{title}[/]",
+        border_style="blue"
+    ))
+
+def print_response(response: dict, title: str = "Response"):
+    """Pretty print relevant parts of the response."""
+    # Extract only the relevant parts
+    relevant = {}
+    if "markdown" in response:
+        relevant["markdown"] = response["markdown"][:200] + "..." if len(response.get("markdown", "")) > 200 else response.get("markdown", "")
+    if "success" in response:
+        relevant["success"] = response["success"]
+    if "url" in response:
+        relevant["url"] = response["url"]
+    if "filter" in response:
+        relevant["filter"] = response["filter"]
+    
+    console.print(Panel.fit(
+        Syntax(json.dumps(relevant, indent=2), "json", theme="monokai"),
+        title=f"[bold green]{title}[/]",
+        border_style="green"
+    ))
+
+# --- Test Functions ---
+
+async def test_default_no_params(client: httpx.AsyncClient):
+    """Test 1: No temperature or base_url specified - uses defaults"""
+    console.rule("[bold yellow]Test 1: Default Configuration (No Parameters)[/]")
+    
+    payload = {
+        "url": TEST_URL,
+        "f": "llm",
+        "q": "What is the main heading of this page? Answer in exactly 5 words."
+    }
+    
+    print_request("/md", payload, "Request without temperature/base_url")
+    
+    try:
+        response = await client.post("/md", json=payload, timeout=30.0)
+        response.raise_for_status()
+        data = response.json()
+        print_response(data, "Response (using system defaults)")
+        console.print("[dim]→ This used system defaults or environment variables if set[/]")
+    except Exception as e:
+        console.print(f"[red]Error: {e}[/]")
+
+async def test_request_temperature(client: httpx.AsyncClient):
+    """Test 2: Request-level temperature (highest priority)"""
+    console.rule("[bold yellow]Test 2: Request-Level Temperature[/]")
+    
+    # Test with low temperature (more focused)
+    payload_low = {
+        "url": TEST_URL,
+        "f": "llm",
+        "q": "What is the main heading? Be creative and poetic.",
+        "temperature": 0.1  # Very low - should be less creative
+    }
+    
+    print_request("/md", payload_low, "Low Temperature (0.1)")
+    
+    try:
+        response = await client.post("/md", json=payload_low, timeout=30.0)
+        response.raise_for_status()
+        data_low = response.json()
+        print_response(data_low, "Response with Low Temperature")
+        console.print("[dim]→ Low temperature (0.1) should produce focused, less creative output[/]")
+    except Exception as e:
+        console.print(f"[red]Error: {e}[/]")
+    
+    console.print()
+    
+    # Test with high temperature (more creative)
+    payload_high = {
+        "url": TEST_URL,
+        "f": "llm",
+        "q": "What is the main heading? Be creative and poetic.",
+        "temperature": 1.5  # High - should be more creative
+    }
+    
+    print_request("/md", payload_high, "High Temperature (1.5)")
+    
+    try:
+        response = await client.post("/md", json=payload_high, timeout=30.0)
+        response.raise_for_status()
+        data_high = response.json()
+        print_response(data_high, "Response with High Temperature")
+        console.print("[dim]→ High temperature (1.5) should produce more creative, varied output[/]")
+    except Exception as e:
+        console.print(f"[red]Error: {e}[/]")
+
+async def test_provider_override(client: httpx.AsyncClient):
+    """Test 3: Provider override with temperature"""
+    console.rule("[bold yellow]Test 3: Provider Override with Temperature[/]")
+    
+    provider = "gemini/gemini-2.5-flash-lite"
+    payload = {
+        "url": TEST_URL,
+        "f": "llm",
+        "q": "Summarize this page in one sentence.",
+        "provider": provider,  # Explicitly set provider
+        "temperature": 0.7
+    }
+    
+    print_request("/md", payload, "Provider + Temperature Override")
+    
+    try:
+        response = await client.post("/md", json=payload, timeout=30.0)
+        response.raise_for_status()
+        data = response.json()
+        print_response(data, "Response with Provider Override")
+        console.print(f"[dim]→ This explicitly uses {provider} with temperature 0.7[/]")
+    except Exception as e:
+        console.print(f"[red]Error: {e}[/]")
+
+async def test_base_url_custom(client: httpx.AsyncClient):
+    """Test 4: Custom base_url (will fail unless you have a custom endpoint)"""
+    console.rule("[bold yellow]Test 4: Custom Base URL (Demo Only)[/]")
+    
+    payload = {
+        "url": TEST_URL,
+        "f": "llm",
+        "q": "What is this page about?",
+        "base_url": "https://api.custom-endpoint.com/v1",  # Custom endpoint
+        "temperature": 0.5
+    }
+    
+    print_request("/md", payload, "Custom Base URL Request")
+    console.print("[yellow]Note: This will fail unless you have a custom endpoint set up[/]")
+    
+    try:
+        response = await client.post("/md", json=payload, timeout=10.0)
+        response.raise_for_status()
+        data = response.json()
+        print_response(data, "Response from Custom Endpoint")
+    except httpx.HTTPStatusError as e:
+        console.print(f"[yellow]Expected failure (no custom endpoint): Status {e.response.status_code}[/]")
+    except Exception as e:
+        console.print(f"[yellow]Expected error: {e}[/]")
+
+async def test_llm_job_endpoint(client: httpx.AsyncClient):
+    """Test 5: Test the /llm/job endpoint with temperature and base_url"""
+    console.rule("[bold yellow]Test 5: LLM Job Endpoint with Parameters[/]")
+    
+    payload = {
+        "url": TEST_URL,
+        "q": "Extract the main title and any key information",
+        "temperature": 0.3,
+        # "base_url": "https://api.openai.com/v1"  # Optional
+    }
+    
+    print_request("/llm/job", payload, "LLM Job with Temperature")
+    
+    try:
+        # Submit the job
+        response = await client.post("/llm/job", json=payload, timeout=30.0)
+        response.raise_for_status()
+        job_data = response.json()
+        
+        if "task_id" in job_data:
+            task_id = job_data["task_id"]
+            console.print(f"[green]Job created with task_id: {task_id}[/]")
+            
+            # Poll for result (simplified - in production use proper polling)
+            await asyncio.sleep(3)
+            
+            status_response = await client.get(f"/llm/job/{task_id}")
+            status_data = status_response.json()
+            
+            if status_data.get("status") == "completed":
+                console.print("[green]Job completed successfully![/]")
+                if "result" in status_data:
+                    console.print(Panel.fit(
+                        Syntax(json.dumps(status_data["result"], indent=2), "json", theme="monokai"),
+                        title="Extraction Result",
+                        border_style="green"
+                    ))
+            else:
+                console.print(f"[yellow]Job status: {status_data.get('status', 'unknown')}[/]")
+        else:
+            console.print(f"[red]Unexpected response: {job_data}[/]")
+            
+    except Exception as e:
+        console.print(f"[red]Error: {e}[/]")
+
+
+async def test_llm_endpoint(client: httpx.AsyncClient):
+    """
+    Quick QA round-trip with /llm.
+    Asks a trivial question against SIMPLE_URL just to show wiring.
+    """
+    import time
+    import urllib.parse
+
+    page_url = "https://kidocode.com"
+    question = "What is the title of this page?"
+
+    enc = urllib.parse.quote_plus(page_url, safe="")
+    console.print(f"GET /llm/{enc}?q={question}")
+
+    try:
+        t0 = time.time()
+        resp = await client.get(f"/llm/{enc}", params={"q": question})
+        dt = time.time() - t0
+        console.print(
+            f"Response Status: [bold {'green' if resp.is_success else 'red'}]{resp.status_code}[/] (took {dt:.2f}s)")
+        resp.raise_for_status()
+        answer = resp.json().get("answer", "")
+        console.print(Panel(answer or "No answer returned",
+                      title="LLM answer", border_style="magenta", expand=False))
+    except Exception as e:
+        console.print(f"[bold red]Error hitting /llm:[/] {e}")
+
+
+async def show_environment_info():
+    """Display current environment configuration"""
+    console.rule("[bold cyan]Current Environment Configuration[/]")
+    
+    table = Table(title="LLM Environment Variables", show_header=True, header_style="bold magenta")
+    table.add_column("Variable", style="cyan", width=30)
+    table.add_column("Value", style="yellow")
+    table.add_column("Description", style="dim")
+    
+    env_vars = [
+        ("LLM_PROVIDER", "Global default provider"),
+        ("LLM_TEMPERATURE", "Global default temperature"),
+        ("LLM_BASE_URL", "Global custom API endpoint"),
+        ("OPENAI_API_KEY", "OpenAI API key"),
+        ("OPENAI_TEMPERATURE", "OpenAI-specific temperature"),
+        ("OPENAI_BASE_URL", "OpenAI-specific endpoint"),
+        ("ANTHROPIC_API_KEY", "Anthropic API key"),
+        ("ANTHROPIC_TEMPERATURE", "Anthropic-specific temperature"),
+        ("GROQ_API_KEY", "Groq API key"),
+        ("GROQ_TEMPERATURE", "Groq-specific temperature"),
+    ]
+    
+    for var, desc in env_vars:
+        value = os.environ.get(var, "[not set]")
+        if "API_KEY" in var and value != "[not set]":
+            # Mask API keys for security
+            value = value[:10] + "..." if len(value) > 10 else "***"
+        table.add_row(var, value, desc)
+    
+    console.print(table)
+    console.print()
+
+# --- Main Test Runner ---
+
+async def main():
+    """Run all tests"""
+    console.print(Panel.fit(
+        "[bold cyan]Crawl4AI LLM Parameters Test Suite[/]\n" +
+        "Testing temperature and base_url configuration hierarchy",
+        border_style="cyan"
+    ))
+    
+    # Show current environment
+    # await show_environment_info()
+    
+    # Create HTTP client
+    async with httpx.AsyncClient(base_url=BASE_URL, timeout=60.0) as client:
+        # Check server health
+        if not await check_server_health(client):
+            console.print("[red]Server is not available. Please ensure the Docker container is running.[/]")
+            return
+        
+        # Run tests
+        tests = [
+            ("Default Configuration", test_default_no_params),
+            ("Request Temperature", test_request_temperature),
+            ("Provider Override", test_provider_override),
+            ("Custom Base URL", test_base_url_custom),
+            ("LLM Job Endpoint", test_llm_job_endpoint),
+            ("LLM Endpoint", test_llm_endpoint),
+        ]
+        
+        for i, (name, test_func) in enumerate(tests, 1):
+            if i > 1:
+                console.print()  # Add spacing between tests
+            
+            try:
+                await test_func(client)
+            except Exception as e:
+                console.print(f"[red]Test '{name}' failed with error: {e}[/]")
+                console.print_exception(show_locals=False)
+        
+        console.rule("[bold green]All Tests Complete![/]", style="green")
+        
+        # Summary
+        console.print("\n[bold cyan]Configuration Hierarchy Summary:[/]")
+        console.print("1. [yellow]Request parameters[/] - Highest priority (temperature, base_url in API call)")
+        console.print("2. [yellow]Provider-specific env[/] - e.g., OPENAI_TEMPERATURE, GROQ_BASE_URL")
+        console.print("3. [yellow]Global env variables[/] - LLM_TEMPERATURE, LLM_BASE_URL")
+        console.print("4. [yellow]System defaults[/] - Lowest priority (provider/litellm defaults)")
+        console.print()
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        console.print("\n[yellow]Tests interrupted by user.[/]")
+    except Exception as e:
+        console.print(f"\n[bold red]An error occurred:[/]")
+        console.print_exception(show_locals=False)
\ No newline at end of file
diff --git a/tests/docker/test_pool_release.py b/tests/docker/test_pool_release.py
new file mode 100644
index 0000000..6c81b3e
--- /dev/null
+++ b/tests/docker/test_pool_release.py
@@ -0,0 +1,155 @@
+"""Tests for crawler pool release_crawler() and active_requests tracking.
+
+These tests validate the pool lifecycle without requiring Docker or a running
+server. They test the release logic directly using mock crawler objects.
+"""
+
+import asyncio
+import pytest
+from unittest.mock import MagicMock
+
+
+# ---------------------------------------------------------------------------
+# Standalone release_crawler implementation for testing
+# (mirrors the logic that will be added to deploy/docker/crawler_pool.py)
+# ---------------------------------------------------------------------------
+
+_TEST_LOCK = asyncio.Lock()
+
+
+async def _release_crawler(crawler, lock=None):
+    """Standalone release logic matching crawler_pool.release_crawler()."""
+    lock = lock or _TEST_LOCK
+    async with lock:
+        if hasattr(crawler, "active_requests"):
+            crawler.active_requests = max(0, crawler.active_requests - 1)
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestReleaseCrawler:
+    """Tests for the release_crawler function."""
+
+    @pytest.mark.asyncio
+    async def test_release_decrements_active_requests(self):
+        """release_crawler should decrement active_requests by 1."""
+        crawler = MagicMock()
+        crawler.active_requests = 3
+
+        await _release_crawler(crawler)
+        assert crawler.active_requests == 2
+
+    @pytest.mark.asyncio
+    async def test_release_floors_at_zero(self):
+        """active_requests should never go below 0."""
+        crawler = MagicMock()
+        crawler.active_requests = 0
+
+        await _release_crawler(crawler)
+        assert crawler.active_requests == 0
+
+    @pytest.mark.asyncio
+    async def test_release_from_one_to_zero(self):
+        """Standard case: single request finishes."""
+        crawler = MagicMock()
+        crawler.active_requests = 1
+
+        await _release_crawler(crawler)
+        assert crawler.active_requests == 0
+
+    @pytest.mark.asyncio
+    async def test_release_handles_missing_attribute(self):
+        """Should not crash if crawler has no active_requests attribute."""
+        crawler = MagicMock(spec=[])  # no attributes at all
+        # Should not raise
+        await _release_crawler(crawler)
+
+    @pytest.mark.asyncio
+    async def test_multiple_releases_decrement_correctly(self):
+        """Multiple sequential releases should each decrement by 1."""
+        crawler = MagicMock()
+        crawler.active_requests = 5
+
+        for expected in [4, 3, 2, 1, 0, 0]:  # last one should floor at 0
+            await _release_crawler(crawler)
+            assert crawler.active_requests == expected
+
+    @pytest.mark.asyncio
+    async def test_concurrent_releases_are_safe(self):
+        """Concurrent releases should not corrupt the counter."""
+        crawler = MagicMock()
+        crawler.active_requests = 100
+        lock = asyncio.Lock()
+
+        async def release_n_times(n):
+            for _ in range(n):
+                await _release_crawler(crawler, lock=lock)
+
+        # 10 concurrent tasks each releasing 10 times = 100 total
+        tasks = [asyncio.create_task(release_n_times(10)) for _ in range(10)]
+        await asyncio.gather(*tasks)
+
+        assert crawler.active_requests == 0
+
+
+class TestActiveRequestsTracking:
+    """Tests for the get/release lifecycle pattern."""
+
+    @pytest.mark.asyncio
+    async def test_get_sets_active_requests(self):
+        """Simulated get_crawler should set active_requests to 1 for new crawlers."""
+        crawler = MagicMock()
+        # Simulate what get_crawler does for a new browser
+        crawler.active_requests = 1
+
+        assert crawler.active_requests == 1
+
+    @pytest.mark.asyncio
+    async def test_get_increments_existing(self):
+        """Simulated get_crawler should increment for existing pooled crawlers."""
+        crawler = MagicMock()
+        crawler.active_requests = 2
+
+        # Simulate another get_crawler call returning same browser
+        crawler.active_requests += 1
+        assert crawler.active_requests == 3
+
+    @pytest.mark.asyncio
+    async def test_full_get_release_lifecycle(self):
+        """Full lifecycle: get -> use -> release -> get -> release."""
+        crawler = MagicMock()
+
+        # First request gets the crawler
+        crawler.active_requests = 1
+
+        # Second concurrent request gets same crawler
+        crawler.active_requests += 1
+        assert crawler.active_requests == 2
+
+        # First request finishes
+        await _release_crawler(crawler)
+        assert crawler.active_requests == 1
+
+        # Second request finishes
+        await _release_crawler(crawler)
+        assert crawler.active_requests == 0
+
+    @pytest.mark.asyncio
+    async def test_janitor_safety_check(self):
+        """Janitor should only close browsers with active_requests == 0."""
+        crawler = MagicMock()
+        crawler.active_requests = 1
+
+        # Janitor check: should NOT close
+        should_close = getattr(crawler, "active_requests", 0) == 0
+        assert should_close is False
+
+        # Request finishes
+        await _release_crawler(crawler)
+
+        # Janitor check: now safe to close
+        should_close = getattr(crawler, "active_requests", 0) == 0
+        assert should_close is True
diff --git a/tests/docker/test_rest_api_deep_crawl.py b/tests/docker/test_rest_api_deep_crawl.py
new file mode 100644
index 0000000..c535727
--- /dev/null
+++ b/tests/docker/test_rest_api_deep_crawl.py
@@ -0,0 +1,596 @@
+# ==== File: test_rest_api_deep_crawl.py ====
+
+import pytest
+import pytest_asyncio
+import httpx
+import json
+import asyncio
+import os
+from typing import List, Dict, Any, AsyncGenerator
+
+from dotenv import load_dotenv
+load_dotenv() # Load environment variables from .env file if present
+
+# --- Test Configuration ---
+BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:11235") # If server is running in Docker, use the host's IP
+BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:8020") # If server is running in dev debug mode
+DEEP_CRAWL_BASE_URL = "https://docs.crawl4ai.com/samples/deepcrawl/"
+DEEP_CRAWL_DOMAIN = "docs.crawl4ai.com" # Used for domain filter
+
+# --- Helper Functions ---
+def load_proxies_from_env() -> List[Dict]:
+    """Load proxies from PROXIES environment variable"""
+    proxies = []
+    proxies_str = os.getenv("PROXIES", "")
+    if not proxies_str:
+        print("PROXIES environment variable not set or empty.")
+        return proxies
+    try:
+        proxy_list = proxies_str.split(",")
+        for proxy in proxy_list:
+            proxy = proxy.strip()
+            if not proxy:
+                continue
+            parts = proxy.split(":")
+            if len(parts) == 4:
+                ip, port, username, password = parts
+                proxies.append({
+                    "server": f"http://{ip}:{port}", # Assuming http, adjust if needed
+                    "username": username,
+                    "password": password,
+                    "ip": ip  # Store original IP if available
+                })
+            elif len(parts) == 2: # ip:port only
+                 ip, port = parts
+                 proxies.append({
+                    "server": f"http://{ip}:{port}",
+                    "ip": ip
+                 })
+            else:
+                 print(f"Skipping invalid proxy string format: {proxy}")
+
+    except Exception as e:
+        print(f"Error loading proxies from environment: {e}")
+    return proxies
+
+
+async def check_server_health(client: httpx.AsyncClient):
+    """Check if the server is healthy before running tests."""
+    try:
+        response = await client.get("/health")
+        response.raise_for_status()
+        print(f"\nServer healthy: {response.json()}")
+        return True
+    except (httpx.RequestError, httpx.HTTPStatusError) as e:
+        pytest.fail(f"Server health check failed: {e}. Is the server running at {BASE_URL}?", pytrace=False)
+
+async def assert_crawl_result_structure(result: Dict[str, Any], check_ssl=False):
+    """Asserts the basic structure of a single crawl result."""
+    assert isinstance(result, dict)
+    assert "url" in result
+    assert "success" in result
+    assert "html" in result # Basic crawls should return HTML
+    assert "metadata" in result
+    assert isinstance(result["metadata"], dict)
+    assert "depth" in result["metadata"] # Deep crawls add depth
+
+    if check_ssl:
+        assert "ssl_certificate" in result # Check if SSL info is present
+        assert isinstance(result["ssl_certificate"], dict) or result["ssl_certificate"] is None
+
+
+async def process_streaming_response(response: httpx.Response) -> List[Dict[str, Any]]:
+    """Processes an NDJSON streaming response."""
+    results = []
+    completed = False
+    async for line in response.aiter_lines():
+        if line:
+            try:
+                data = json.loads(line)
+                if data.get("status") == "completed":
+                    completed = True
+                    break # Stop processing after completion marker
+                elif data.get("url"): # Ensure it looks like a result object
+                    results.append(data)
+                else:
+                    print(f"Received non-result JSON line: {data}") # Log other status messages if needed
+            except json.JSONDecodeError:
+                pytest.fail(f"Failed to decode JSON line: {line}")
+    assert completed, "Streaming response did not end with a completion marker."
+    return results
+
+
+# --- Pytest Fixtures ---
+@pytest_asyncio.fixture(scope="function")
+async def async_client() -> AsyncGenerator[httpx.AsyncClient, None]:
+    """Provides an async HTTP client"""
+    # Increased timeout for potentially longer deep crawls
+    async with httpx.AsyncClient(base_url=BASE_URL, timeout=300.0) as client:
+        yield client
+    # No explicit close needed with 'async with'
+
+# --- Test Class ---
+@pytest.mark.asyncio
+class TestDeepCrawlEndpoints:
+
+    @pytest_asyncio.fixture(autouse=True)
+    async def check_health_before_tests(self, async_client: httpx.AsyncClient):
+        """Fixture to ensure server is healthy before each test in the class."""
+        await check_server_health(async_client)
+
+    # 1. Basic Deep Crawl
+    async def test_deep_crawl_basic_bfs(self, async_client: httpx.AsyncClient):
+        """Test BFS deep crawl with limited depth and pages."""
+        max_depth = 1
+        max_pages = 3 # start_url + 2 more
+        payload = {
+            "urls": [DEEP_CRAWL_BASE_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False,
+                    "cache_mode": "BYPASS", # Use string value for CacheMode
+                    "deep_crawl_strategy": {
+                        "type": "BFSDeepCrawlStrategy",
+                        "params": {
+                            "max_depth": max_depth,
+                            "max_pages": max_pages,
+                            # Minimal filters for basic test
+                            "filter_chain": {
+                                "type": "FilterChain",
+                                "params": {
+                                    "filters": [
+                                        {
+                                            "type": "DomainFilter",
+                                            "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}
+                                        }
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        response = await async_client.post("/crawl", json=payload)
+        response.raise_for_status()
+        data = response.json()
+
+        assert data["success"] is True
+        assert isinstance(data["results"], list)
+        assert len(data["results"]) > 1 # Should be more than just the start URL
+        assert len(data["results"]) <= max_pages # Respect max_pages
+
+        found_depth_0 = False
+        found_depth_1 = False
+        for result in data["results"]:
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            assert DEEP_CRAWL_DOMAIN in result["url"]
+            depth = result["metadata"]["depth"]
+            assert depth <= max_depth
+            if depth == 0: found_depth_0 = True
+            if depth == 1: found_depth_1 = True
+
+        assert found_depth_0
+        assert found_depth_1
+
+    # 2. Deep Crawl with Filtering
+    async def test_deep_crawl_with_filters(self, async_client: httpx.AsyncClient):
+        """Test BFS deep crawl with content type and domain filters."""
+        max_depth = 1
+        max_pages = 5
+        payload = {
+            "urls": [DEEP_CRAWL_BASE_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False,
+                    "cache_mode": "BYPASS",
+                    "deep_crawl_strategy": {
+                        "type": "BFSDeepCrawlStrategy",
+                        "params": {
+                            "max_depth": max_depth,
+                            "max_pages": max_pages,
+                            "filter_chain": {
+                                "type": "FilterChain",
+                                "params": {
+                                    "filters": [
+                                        {
+                                            "type": "DomainFilter",
+                                            "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}
+                                        },
+                                        {
+                                            "type": "ContentTypeFilter",
+                                            "params": {"allowed_types": ["text/html"]}
+                                        },
+                                        # Example: Exclude specific paths using regex
+                                        {
+                                            "type": "URLPatternFilter",
+                                             "params": {
+                                                 "patterns": ["*/category-3/*"], # Block category 3
+                                                 "reverse": True # Block if match
+                                             }
+                                        }
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        response = await async_client.post("/crawl", json=payload)
+        response.raise_for_status()
+        data = response.json()
+
+        assert data["success"] is True
+        assert len(data["results"]) > 0
+        assert len(data["results"]) <= max_pages
+
+        for result in data["results"]:
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            assert DEEP_CRAWL_DOMAIN in result["url"]
+            assert "category-3" not in result["url"] # Check if filter worked
+            assert result["metadata"]["depth"] <= max_depth
+
+    # 3. Deep Crawl with Scoring
+    async def test_deep_crawl_with_scoring(self, async_client: httpx.AsyncClient):
+        """Test BFS deep crawl with URL scoring."""
+        max_depth = 1
+        max_pages = 4
+        payload = {
+            "urls": [DEEP_CRAWL_BASE_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False,
+                    "cache_mode": "BYPASS",
+                    "deep_crawl_strategy": {
+                        "type": "BFSDeepCrawlStrategy",
+                        "params": {
+                            "max_depth": max_depth,
+                            "max_pages": max_pages,
+                            "filter_chain": { # Keep basic domain filter
+                                "type": "FilterChain",
+                                "params": { "filters": [{"type": "DomainFilter", "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}}]}
+                            },
+                            "url_scorer": { # Add scorer
+                                "type": "CompositeScorer",
+                                "params": {
+                                    "scorers": [
+                                        {   # Favor pages with 'product' in the URL
+                                            "type": "KeywordRelevanceScorer",
+                                            "params": {"keywords": ["product"], "weight": 1.0}
+                                        },
+                                        {   # Penalize deep paths slightly
+                                            "type": "PathDepthScorer",
+                                            "params": {"optimal_depth": 2, "weight": -0.2}
+                                        }
+                                    ]
+                                }
+                            },
+                            # Set a threshold if needed: "score_threshold": 0.1
+                        }
+                    }
+                }
+            }
+        }
+        response = await async_client.post("/crawl", json=payload)
+        response.raise_for_status()
+        data = response.json()
+
+        assert data["success"] is True
+        assert len(data["results"]) > 0
+        assert len(data["results"]) <= max_pages
+
+        # Check if results seem biased towards products (harder to assert strictly without knowing exact scores)
+        product_urls_found = any("product_" in result["url"] for result in data["results"] if result["metadata"]["depth"] > 0)
+        print(f"Product URLs found among depth > 0 results: {product_urls_found}")
+        # We expect scoring to prioritize product pages if available within limits
+        # assert product_urls_found # This might be too strict depending on site structure and limits
+
+        for result in data["results"]:
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            assert result["metadata"]["depth"] <= max_depth
+
+    # 4. Deep Crawl with CSS Extraction
+    async def test_deep_crawl_with_css_extraction(self, async_client: httpx.AsyncClient):
+        """Test BFS deep crawl combined with JsonCssExtractionStrategy."""
+        max_depth = 6 # Go deep enough to reach product pages
+        max_pages = 20
+        # Schema to extract product details
+        product_schema = {
+            "name": "ProductDetails",
+            "baseSelector": "div.container", # Base for product page
+            "fields": [
+                {"name": "product_title", "selector": "h1", "type": "text"},
+                {"name": "price", "selector": ".product-price", "type": "text"},
+                {"name": "description", "selector": ".product-description p", "type": "text"},
+                {"name": "specs", "selector": ".product-specs li", "type": "list", "fields":[
+                     {"name": "spec_name", "selector": ".spec-name", "type": "text"},
+                     {"name": "spec_value", "selector": ".spec-value", "type": "text"}
+                ]}
+            ]
+        }
+        payload = {
+            "urls": [DEEP_CRAWL_BASE_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False,
+                    "cache_mode": "BYPASS",
+                    "extraction_strategy": { # Apply extraction to ALL crawled pages
+                        "type": "JsonCssExtractionStrategy",
+                        "params": {"schema": {"type": "dict", "value": product_schema}}
+                    },
+                    "deep_crawl_strategy": {
+                        "type": "BFSDeepCrawlStrategy",
+                        "params": {
+                            "max_depth": max_depth,
+                            "max_pages": max_pages,
+                            "filter_chain": { # Only crawl HTML on our domain
+                                "type": "FilterChain",
+                                "params": {
+                                    "filters": [
+                                        {"type": "DomainFilter", "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}},
+                                        {"type": "ContentTypeFilter", "params": {"allowed_types": ["text/html"]}}
+                                    ]
+                                }
+                            }
+                            # Optional: Add scoring to prioritize product pages for extraction
+                        }
+                    }
+                }
+            }
+        }
+        response = await async_client.post("/crawl", json=payload)
+        response.raise_for_status()
+        data = response.json()
+
+        assert data["success"] is True
+        assert len(data["results"]) > 0
+        # assert len(data["results"]) <= max_pages
+
+        found_extracted_product = False
+        for result in data["results"]:
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            assert "extracted_content" in result
+            if "product_" in result["url"]: # Check product pages specifically
+                 assert result["extracted_content"] is not None
+                 try:
+                     extracted = json.loads(result["extracted_content"])
+                     # Schema returns list even if one base match
+                     assert isinstance(extracted, list)
+                     if extracted:
+                         item = extracted[0]
+                         assert "product_title" in item and item["product_title"]
+                         assert "price" in item and item["price"]
+                         # Specs might be empty list if not found
+                         assert "specs" in item and isinstance(item["specs"], list)
+                         found_extracted_product = True
+                         print(f"Extracted product: {item.get('product_title')}")
+                 except (json.JSONDecodeError, AssertionError, IndexError) as e:
+                      pytest.fail(f"Extraction validation failed for {result['url']}: {e}\nContent: {result['extracted_content']}")
+            # else:
+            #      # Non-product pages might have None or empty list depending on schema match
+            #      assert result["extracted_content"] is None or json.loads(result["extracted_content"]) == []
+
+        assert found_extracted_product, "Did not find any pages where product data was successfully extracted."
+
+    # 5. Deep Crawl with LLM Extraction (Requires Server LLM Setup)
+    async def test_deep_crawl_with_llm_extraction(self, async_client: httpx.AsyncClient):
+        """Test BFS deep crawl combined with LLMExtractionStrategy."""
+        max_depth = 1 # Limit depth to keep LLM calls manageable
+        max_pages = 3
+        payload = {
+            "urls": [DEEP_CRAWL_BASE_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False,
+                    "cache_mode": "BYPASS",
+                    "extraction_strategy": { # Apply LLM extraction to crawled pages
+                        "type": "LLMExtractionStrategy",
+                        "params": {
+                            "instruction": "Extract the main H1 title and the text content of the first paragraph.",
+                            "llm_config": { # Example override, rely on server default if possible
+                               "type": "LLMConfig",
+                               "params": {"provider": "openai/gpt-4.1-mini"} # Use a cheaper model for testing
+                            },
+                             "schema": { # Expected JSON output
+                                "type": "dict",
+                                "value": {
+                                    "title": "PageContent", "type": "object",
+                                    "properties": {
+                                        "h1_title": {"type": "string"},
+                                        "first_paragraph": {"type": "string"}
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    "deep_crawl_strategy": {
+                        "type": "BFSDeepCrawlStrategy",
+                        "params": {
+                            "max_depth": max_depth,
+                            "max_pages": max_pages,
+                            "filter_chain": {
+                                "type": "FilterChain",
+                                "params": {
+                                    "filters": [
+                                        {"type": "DomainFilter", "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}},
+                                        {"type": "ContentTypeFilter", "params": {"allowed_types": ["text/html"]}}
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        try:
+            response = await async_client.post("/crawl", json=payload)
+            response.raise_for_status()
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            pytest.fail(f"Deep Crawl + LLM extraction request failed: {e}. Response: {e.response.text}. Check server logs and LLM API key setup.")
+        except httpx.RequestError as e:
+             pytest.fail(f"Deep Crawl + LLM extraction request failed: {e}.")
+
+
+        assert data["success"] is True
+        assert len(data["results"]) > 0
+        assert len(data["results"]) <= max_pages
+
+        found_llm_extraction = False
+        for result in data["results"]:
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            assert "extracted_content" in result
+            assert result["extracted_content"] is not None
+            try:
+                extracted = json.loads(result["extracted_content"])
+                if isinstance(extracted, list): extracted = extracted[0] # Handle list output
+                assert isinstance(extracted, dict)
+                assert "h1_title" in extracted # Check keys based on schema
+                assert "first_paragraph" in extracted
+                found_llm_extraction = True
+                print(f"LLM extracted from {result['url']}: Title='{extracted.get('h1_title')}'")
+            except (json.JSONDecodeError, AssertionError, IndexError, TypeError) as e:
+                pytest.fail(f"LLM extraction validation failed for {result['url']}: {e}\nContent: {result['extracted_content']}")
+
+        assert found_llm_extraction, "LLM extraction did not yield expected data on any crawled page."
+
+
+    # 6. Deep Crawl with SSL Certificate Fetching
+    async def test_deep_crawl_with_ssl(self, async_client: httpx.AsyncClient):
+        """Test BFS deep crawl with fetch_ssl_certificate enabled."""
+        max_depth = 0 # Only fetch for start URL to keep test fast
+        max_pages = 1
+        payload = {
+            "urls": [DEEP_CRAWL_BASE_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False,
+                    "cache_mode": "BYPASS",
+                    "fetch_ssl_certificate": True, # <-- Enable SSL fetching
+                    "deep_crawl_strategy": {
+                        "type": "BFSDeepCrawlStrategy",
+                        "params": {
+                            "max_depth": max_depth,
+                            "max_pages": max_pages,
+                        }
+                    }
+                }
+            }
+        }
+        response = await async_client.post("/crawl", json=payload)
+        response.raise_for_status()
+        data = response.json()
+
+        assert data["success"] is True
+        assert len(data["results"]) == 1
+        result = data["results"][0]
+
+        await assert_crawl_result_structure(result, check_ssl=True) # <-- Tell helper to check SSL field
+        assert result["success"] is True
+                # Check if SSL info was actually retrieved
+        if result["ssl_certificate"]:
+            # Assert directly using dictionary keys
+            assert isinstance(result["ssl_certificate"], dict) # Verify it's a dict
+            assert "issuer" in result["ssl_certificate"]
+            assert "subject" in result["ssl_certificate"]
+            # --- MODIFIED ASSERTIONS ---
+            assert "not_before" in result["ssl_certificate"] # Check for the actual key
+            assert "not_after" in result["ssl_certificate"]  # Check for the actual key
+            # --- END MODIFICATIONS ---
+            assert "fingerprint" in result["ssl_certificate"] # Check another key
+
+            # This print statement using .get() already works correctly with dictionaries
+            print(f"SSL Issuer Org: {result['ssl_certificate'].get('issuer', {}).get('O', 'N/A')}")
+            print(f"SSL Valid From: {result['ssl_certificate'].get('not_before', 'N/A')}")
+        else:
+            # This part remains the same
+            print("SSL Certificate was null in the result.")
+
+
+    # 7. Deep Crawl with Proxy Rotation (Requires PROXIES env var)
+    async def test_deep_crawl_with_proxies(self, async_client: httpx.AsyncClient):
+        """Test BFS deep crawl using proxy rotation."""
+        proxies = load_proxies_from_env()
+        if not proxies:
+            pytest.skip("Skipping proxy test: PROXIES environment variable not set or empty.")
+
+        print(f"\nTesting with {len(proxies)} proxies loaded from environment.")
+
+        max_depth = 1
+        max_pages = 3
+        payload = {
+            "urls": [DEEP_CRAWL_BASE_URL], # Use the dummy site
+             # Use a BrowserConfig that *might* pick up proxy if set, but rely on CrawlerRunConfig
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False,
+                    "cache_mode": "BYPASS",
+                    "proxy_rotation_strategy": { # <-- Define the strategy
+                        "type": "RoundRobinProxyStrategy",
+                        "params": {
+                             # Convert ProxyConfig dicts back to the serialized format expected by server
+                             "proxies": [{"type": "ProxyConfig", "params": p} for p in proxies]
+                        }
+                    },
+                    "deep_crawl_strategy": {
+                        "type": "BFSDeepCrawlStrategy",
+                        "params": {
+                            "max_depth": max_depth,
+                            "max_pages": max_pages,
+                            "filter_chain": {
+                                "type": "FilterChain",
+                                "params": { "filters": [{"type": "DomainFilter", "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}}]}
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        try:
+            response = await async_client.post("/crawl", json=payload)
+            response.raise_for_status()
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            # Proxies often cause connection errors, catch them
+            pytest.fail(f"Proxy deep crawl failed: {e}. Response: {e.response.text}. Are proxies valid and accessible by the server?")
+        except httpx.RequestError as e:
+             pytest.fail(f"Proxy deep crawl request failed: {e}. Are proxies valid and accessible?")
+
+        assert data["success"] is True
+        assert len(data["results"]) > 0
+        assert len(data["results"]) <= max_pages
+        # Primary assertion is that the crawl succeeded *with* proxy config
+        print(f"Proxy deep crawl completed successfully for {len(data['results'])} pages.")
+
+        # Verifying specific proxy usage requires server logs or custom headers/responses
+
+
+# --- Main Execution Block (for running script directly) ---
+if __name__ == "__main__":
+    pytest_args = ["-v", "-s", __file__]
+    # Example: Run only proxy test
+    # pytest_args.append("-k test_deep_crawl_with_proxies")
+    print(f"Running pytest with args: {pytest_args}")
+    exit_code = pytest.main(pytest_args)
+    print(f"Pytest finished with exit code: {exit_code}")
\ No newline at end of file
diff --git a/tests/docker/test_serialization.py b/tests/docker/test_serialization.py
new file mode 100644
index 0000000..6ce8000
--- /dev/null
+++ b/tests/docker/test_serialization.py
@@ -0,0 +1,255 @@
+import inspect
+from typing import Any, Dict
+from enum import Enum
+
+from crawl4ai import LLMConfig
+
+def to_serializable_dict(obj: Any) -> Dict:
+    """
+    Recursively convert an object to a serializable dictionary using {type, params} structure
+    for complex objects.
+    """
+    if obj is None:
+        return None
+        
+    # Handle basic types
+    if isinstance(obj, (str, int, float, bool)):
+        return obj
+        
+    # Handle Enum
+    if isinstance(obj, Enum):
+        return {
+            "type": obj.__class__.__name__,
+            "params": obj.value
+        }
+        
+    # Handle datetime objects
+    if hasattr(obj, 'isoformat'):
+        return obj.isoformat()
+        
+    # Handle lists, tuples, and sets
+    if isinstance(obj, (list, tuple, set)):
+        return [to_serializable_dict(item) for item in obj]
+        
+    # Handle dictionaries - preserve them as-is
+    if isinstance(obj, dict):
+        return {
+            "type": "dict",  # Mark as plain dictionary
+            "value": {str(k): to_serializable_dict(v) for k, v in obj.items()}
+        }
+    
+    # Handle class instances
+    if hasattr(obj, '__class__'):
+        # Get constructor signature
+        sig = inspect.signature(obj.__class__.__init__)
+        params = sig.parameters
+        
+        # Get current values
+        current_values = {}
+        for name, param in params.items():
+            if name == 'self':
+                continue
+                
+            value = getattr(obj, name, param.default)
+            
+            # Only include if different from default, considering empty values
+            if not (is_empty_value(value) and is_empty_value(param.default)):
+                if value != param.default:
+                    current_values[name] = to_serializable_dict(value)
+        
+        return {
+            "type": obj.__class__.__name__,
+            "params": current_values
+        }
+        
+    return str(obj)
+
+def from_serializable_dict(data: Any) -> Any:
+    """
+    Recursively convert a serializable dictionary back to an object instance.
+    """
+    if data is None:
+        return None
+        
+    # Handle basic types
+    if isinstance(data, (str, int, float, bool)):
+        return data
+        
+    # Handle typed data
+    if isinstance(data, dict) and "type" in data:
+        # Handle plain dictionaries
+        if data["type"] == "dict":
+            return {k: from_serializable_dict(v) for k, v in data["value"].items()}
+            
+        # Import from crawl4ai for class instances
+        import crawl4ai
+        cls = getattr(crawl4ai, data["type"])
+        
+        # Handle Enum
+        if issubclass(cls, Enum):
+            return cls(data["params"])
+            
+        # Handle class instances
+        constructor_args = {
+            k: from_serializable_dict(v) for k, v in data["params"].items()
+        }
+        return cls(**constructor_args)
+        
+    # Handle lists
+    if isinstance(data, list):
+        return [from_serializable_dict(item) for item in data]
+        
+    # Handle raw dictionaries (legacy support)
+    if isinstance(data, dict):
+        return {k: from_serializable_dict(v) for k, v in data.items()}
+        
+    return data
+    
+def is_empty_value(value: Any) -> bool:
+    """Check if a value is effectively empty/null."""
+    if value is None:
+        return True
+    if isinstance(value, (list, tuple, set, dict, str)) and len(value) == 0:
+        return True
+    return False
+
+# if __name__ == "__main__":
+#     from crawl4ai import (
+#         CrawlerRunConfig, CacheMode, DefaultMarkdownGenerator, 
+#         PruningContentFilter, BM25ContentFilter, LLMContentFilter,
+#         JsonCssExtractionStrategy, CosineStrategy, RegexChunking,
+#         WebScrapingStrategy, LXMLWebScrapingStrategy
+#     )
+
+#     # Test Case 1: BM25 content filtering through markdown generator
+#     config1 = CrawlerRunConfig(
+#         cache_mode=CacheMode.BYPASS,
+#         markdown_generator=DefaultMarkdownGenerator(
+#             content_filter=BM25ContentFilter(
+#                 user_query="technology articles",
+#                 bm25_threshold=1.2,
+#                 language="english"
+#             )
+#         ),
+#         chunking_strategy=RegexChunking(patterns=[r"\n\n", r"\.\s+"]),
+#         excluded_tags=["nav", "footer", "aside"],
+#         remove_overlay_elements=True
+#     )
+
+#     # Serialize
+#     serialized = to_serializable_dict(config1)
+#     print("\nSerialized Config:")
+#     print(serialized)
+    
+#     # Example output structure would now look like:
+#     """
+#     {
+#         "type": "CrawlerRunConfig",
+#         "params": {
+#             "cache_mode": {
+#                 "type": "CacheMode",
+#                 "params": "bypass"
+#             },
+#             "markdown_generator": {
+#                 "type": "DefaultMarkdownGenerator",
+#                 "params": {
+#                     "content_filter": {
+#                         "type": "BM25ContentFilter",
+#                         "params": {
+#                             "user_query": "technology articles",
+#                             "bm25_threshold": 1.2,
+#                             "language": "english"
+#                         }
+#                     }
+#                 }
+#             }
+#         }
+#     }
+#     """
+    
+#     # Deserialize
+#     deserialized = from_serializable_dict(serialized)
+#     print("\nDeserialized Config:")
+#     print(to_serializable_dict(deserialized))
+    
+#     # Verify they match
+#     assert to_serializable_dict(config1) == to_serializable_dict(deserialized)
+#     print("\nVerification passed: Configuration matches after serialization/deserialization!")
+
+if __name__ == "__main__":
+    from crawl4ai import (
+        CrawlerRunConfig, CacheMode, DefaultMarkdownGenerator, 
+        PruningContentFilter, BM25ContentFilter, LLMContentFilter,
+        JsonCssExtractionStrategy, RegexChunking,
+        WebScrapingStrategy, LXMLWebScrapingStrategy
+    )
+
+    # Test Case 1: BM25 content filtering through markdown generator
+    config1 = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=BM25ContentFilter(
+                user_query="technology articles",
+                bm25_threshold=1.2,
+                language="english"
+            )
+        ),
+        chunking_strategy=RegexChunking(patterns=[r"\n\n", r"\.\s+"]),
+        excluded_tags=["nav", "footer", "aside"],
+        remove_overlay_elements=True
+    )
+
+    # Test Case 2: LLM-based extraction with pruning filter
+    schema = {
+        "baseSelector": "article.post",
+        "fields": [
+            {"name": "title", "selector": "h1", "type": "text"},
+            {"name": "content", "selector": ".content", "type": "html"}
+        ]
+    }
+    config2 = CrawlerRunConfig(
+        extraction_strategy=JsonCssExtractionStrategy(schema=schema),
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(
+                threshold=0.48,
+                threshold_type="fixed",
+                min_word_threshold=0
+            ),
+            options={"ignore_links": True}
+        ),
+        scraping_strategy=LXMLWebScrapingStrategy()
+    )
+
+    # Test Case 3:LLM content filter
+    config3 = CrawlerRunConfig(
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=LLMContentFilter(
+                llm_config = LLMConfig(provider="openai/gpt-4"),
+                instruction="Extract key technical concepts",
+                chunk_token_threshold=2000,
+                overlap_rate=0.1
+            ),
+            options={"ignore_images": True}
+        ),
+        scraping_strategy=WebScrapingStrategy()
+    )
+
+    # Test all configurations
+    test_configs = [config1, config2, config3]
+    
+    for i, config in enumerate(test_configs, 1):
+        print(f"\nTesting Configuration {i}:")
+        
+        # Serialize
+        serialized = to_serializable_dict(config)
+        print(f"\nSerialized Config {i}:")
+        print(serialized)
+        
+        # Deserialize
+        deserialized = from_serializable_dict(serialized)
+        print(f"\nDeserialized Config {i}:")
+        print(to_serializable_dict(deserialized))  # Convert back to dict for comparison
+        
+        # Verify they match
+        assert to_serializable_dict(config) == to_serializable_dict(deserialized)
+        print(f"\nVerification passed: Configuration {i} matches after serialization/deserialization!")
\ No newline at end of file
diff --git a/tests/docker/test_server.py b/tests/docker/test_server.py
new file mode 100644
index 0000000..7bb0195
--- /dev/null
+++ b/tests/docker/test_server.py
@@ -0,0 +1,146 @@
+import asyncio
+import json
+from typing import Optional
+from urllib.parse import quote
+
+async def test_endpoint(
+    endpoint: str, 
+    url: str, 
+    params: Optional[dict] = None,
+    expected_status: int = 200
+) -> None:
+    """Test an endpoint and print results"""
+    import aiohttp
+    
+    params = params or {}
+    param_str = "&".join(f"{k}={v}" for k, v in params.items())
+    full_url = f"http://localhost:8000/{endpoint}/{quote(url)}"
+    if param_str:
+        full_url += f"?{param_str}"
+        
+    print(f"\nTesting: {full_url}")
+    
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(full_url) as response:
+                status = response.status
+                try:
+                    data = await response.json()
+                except:
+                    data = await response.text()
+                
+                print(f"Status: {status} (Expected: {expected_status})")
+                if isinstance(data, dict):
+                    print(f"Response: {json.dumps(data, indent=2)}")
+                else:
+                    print(f"Response: {data[:500]}...")  # First 500 chars
+                assert status == expected_status
+                return data
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        return None
+
+async def test_llm_task_completion(task_id: str) -> None:
+    """Poll task until completion"""
+    for _ in range(10):  # Try 10 times
+        result = await test_endpoint("llm", task_id)
+        if result and result.get("status") in ["completed", "failed"]:
+            return result
+        print("Task still processing, waiting 5 seconds...")
+        await asyncio.sleep(5)
+    print("Task timed out")
+
+async def run_tests():
+    print("Starting API Tests...")
+    
+    # Test URLs
+    urls = [
+        "example.com",
+        "https://www.python.org",
+        "https://news.ycombinator.com/news",
+        "https://github.com/trending"
+    ]
+    
+    print("\n=== Testing Markdown Endpoint ===")
+    for url in[] : #urls:
+        # Test different filter types
+        for filter_type in ["raw", "fit", "bm25", "llm"]:
+            params = {"f": filter_type}
+            if filter_type in ["bm25", "llm"]:
+                params["q"] = "extract main content"
+            
+            # Test with and without cache
+            for cache in ["0", "1"]:
+                params["c"] = cache
+                await test_endpoint("md", url, params)
+                await asyncio.sleep(1)  # Be nice to the server
+
+    print("\n=== Testing LLM Endpoint ===")
+    for url in []: # urls:
+        # Test basic extraction
+        result = await test_endpoint(
+            "llm", 
+            url, 
+            {"q": "Extract title and main content"}
+        )
+        if result and "task_id" in result:
+            print("\nChecking task completion...")
+            await test_llm_task_completion(result["task_id"])
+        
+        # Test with schema
+        schema = {
+            "type": "object",
+            "properties": {
+                "title": {"type": "string"},
+                "content": {"type": "string"},
+                "links": {"type": "array", "items": {"type": "string"}}
+            }
+        }
+        result = await test_endpoint(
+            "llm", 
+            url, 
+            {
+                "q": "Extract content with links", 
+                "s": json.dumps(schema),
+                "c": "1"  # Test with cache
+            }
+        )
+        if result and "task_id" in result:
+            print("\nChecking schema task completion...")
+            await test_llm_task_completion(result["task_id"])
+        
+        await asyncio.sleep(2)  # Be nice to the server
+    
+    print("\n=== Testing Error Cases ===")
+    # Test invalid URL
+    await test_endpoint(
+        "md", 
+        "not_a_real_url", 
+        expected_status=500
+    )
+    
+    # Test invalid filter type
+    await test_endpoint(
+        "md", 
+        "example.com", 
+        {"f": "invalid"},
+        expected_status=422
+    )
+    
+    # Test LLM without query
+    await test_endpoint(
+        "llm", 
+        "example.com"
+    )
+    
+    # Test invalid task ID
+    await test_endpoint(
+        "llm", 
+        "llm_invalid_task",
+        expected_status=404
+    )
+    
+    print("\nAll tests completed!")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
\ No newline at end of file
diff --git a/tests/docker/test_server_requests.py b/tests/docker/test_server_requests.py
new file mode 100644
index 0000000..ae838c0
--- /dev/null
+++ b/tests/docker/test_server_requests.py
@@ -0,0 +1,890 @@
+import pytest
+import pytest_asyncio
+import httpx
+import json
+import asyncio
+import os
+from typing import List, Dict, Any, AsyncGenerator
+
+from dotenv import load_dotenv
+load_dotenv()
+
+
+# Optional: Import crawl4ai classes directly for reference/easier payload creation aid
+# You don't strictly NEED these imports for the tests to run against the server,
+# but they help in understanding the structure you are mimicking in JSON.
+from crawl4ai import (
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    DefaultMarkdownGenerator,
+    PruningContentFilter,
+    BM25ContentFilter,
+    BFSDeepCrawlStrategy,
+    FilterChain,
+    ContentTypeFilter,
+    DomainFilter,
+    CompositeScorer,
+    KeywordRelevanceScorer,
+    PathDepthScorer,
+    JsonCssExtractionStrategy,
+    LLMExtractionStrategy,
+    LLMConfig
+)
+
+# --- Test Configuration ---
+# BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:8020") # Make base URL configurable
+BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:11235") # Make base URL configurable
+# Use a known simple HTML page for basic tests
+SIMPLE_HTML_URL = "https://httpbin.org/html"
+# Use a site suitable for scraping tests
+SCRAPE_TARGET_URL = "http://books.toscrape.com/"
+# Use a site with internal links for deep crawl tests
+DEEP_CRAWL_URL = "https://python.org"
+
+# --- Pytest Fixtures ---
+
+# Use the built-in event_loop fixture from pytest_asyncio
+# The custom implementation was causing issues with closing the loop
+
+@pytest_asyncio.fixture(scope="function")  # Changed to function scope to avoid event loop issues
+async def async_client() -> AsyncGenerator[httpx.AsyncClient, None]:
+    """Provides an async HTTP client"""
+    client = httpx.AsyncClient(base_url=BASE_URL, timeout=120.0)
+    yield client
+    await client.aclose()
+
+# --- Helper Functions ---
+
+async def check_server_health(client: httpx.AsyncClient):
+    """Check if the server is healthy before running tests."""
+    try:
+        response = await client.get("/health")
+        response.raise_for_status()
+        print(f"\nServer healthy: {response.json()}")
+        return True
+    except (httpx.RequestError, httpx.HTTPStatusError) as e:
+        pytest.fail(f"Server health check failed: {e}. Is the server running at {BASE_URL}?", pytrace=False)
+
+async def assert_crawl_result_structure(result: Dict[str, Any]):
+    """Asserts the basic structure of a single crawl result."""
+    assert isinstance(result, dict)
+    assert "url" in result
+    assert "success" in result
+    assert "html" in result
+    # Add more common checks if needed
+
+async def process_streaming_response(response: httpx.Response) -> List[Dict[str, Any]]:
+    """Processes an NDJSON streaming response."""
+    results = []
+    completed = False
+    async for line in response.aiter_lines():
+        if line:
+            try:
+                data = json.loads(line)
+                if data.get("status") == "completed":
+                    completed = True
+                    break # Stop processing after completion marker
+                else:
+                    results.append(data)
+            except json.JSONDecodeError:
+                pytest.fail(f"Failed to decode JSON line: {line}")
+    assert completed, "Streaming response did not end with a completion marker."
+    return results
+
+
+# --- Test Class ---
+
+@pytest.mark.asyncio
+class TestCrawlEndpoints:
+
+    @pytest_asyncio.fixture(autouse=True)
+    async def check_health_before_tests(self, async_client: httpx.AsyncClient):
+        """Fixture to ensure server is healthy before each test in the class."""
+        await check_server_health(async_client)
+
+    # 1. Simple Requests (Primitives)
+    async def test_simple_crawl_single_url(self, async_client: httpx.AsyncClient):
+        """Test /crawl with a single URL and simple config values."""
+        payload = {
+            "urls": [SIMPLE_HTML_URL],
+            "browser_config": {
+                "type": "BrowserConfig",
+                "params": {
+                    "headless": True,
+                }
+            },
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False, # Explicitly false for /crawl
+                    "screenshot": False,
+                    "cache_mode": CacheMode.BYPASS.value # Use enum value
+                }
+            }
+        }
+        try:
+            response = await async_client.post("/crawl", json=payload)
+            print(f"Response status: {response.status_code}")
+            response.raise_for_status()
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            print(f"Server error: {e}")
+            print(f"Response content: {e.response.text}")
+            raise
+
+        assert data["success"] is True
+        assert isinstance(data["results"], list)
+        assert len(data["results"]) == 1
+        result = data["results"][0]
+        await assert_crawl_result_structure(result)
+        assert result["success"] is True
+        assert result["url"] == SIMPLE_HTML_URL
+        assert "
      Herman Melville - Moby-Dick
      " in result["html"]
+        # We don't specify a markdown generator in this test, so don't make assumptions about markdown field
+        # It might be null, missing, or populated depending on the server's default behavior
+    async def test_crawl_with_stream_direct(self, async_client: httpx.AsyncClient):
+        """Test that /crawl endpoint handles stream=True directly without redirect."""
+        payload = {
+            "urls": [SIMPLE_HTML_URL],
+            "browser_config": {
+                "type": "BrowserConfig",
+                "params": {
+                    "headless": True,
+                }
+            },
+            "crawler_config": {
+                "type": "CrawlerRunConfig", 
+                "params": {
+                    "stream": True,  # Set stream to True for direct streaming
+                    "screenshot": False,
+                    "cache_mode": CacheMode.BYPASS.value
+                }
+            }
+        }
+
+        # Send a request to the /crawl endpoint - should handle streaming directly
+        async with async_client.stream("POST", "/crawl", json=payload) as response:
+            assert response.status_code == 200
+            assert response.headers["content-type"] == "application/x-ndjson"
+            assert response.headers.get("x-stream-status") == "active"
+
+            results = await process_streaming_response(response)
+
+            assert len(results) == 1
+            result = results[0]
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            assert result["url"] == SIMPLE_HTML_URL
+            assert "
      Herman Melville - Moby-Dick
      " in result["html"]
+    async def test_simple_crawl_single_url_streaming(self, async_client: httpx.AsyncClient):
+        """Test /crawl/stream with a single URL and simple config values."""
+        payload = {
+            "urls": [SIMPLE_HTML_URL],
+            "browser_config": {
+                "type": "BrowserConfig",
+                "params": {
+                    "headless": True,
+                }
+            },
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": True, # Must be true for /crawl/stream
+                    "screenshot": False,
+                    "cache_mode": CacheMode.BYPASS.value
+                }
+            }
+        }
+        async with async_client.stream("POST", "/crawl/stream", json=payload) as response:
+            response.raise_for_status()
+            results = await process_streaming_response(response)
+
+        assert len(results) == 1
+        result = results[0]
+        await assert_crawl_result_structure(result)
+        assert result["success"] is True
+        assert result["url"] == SIMPLE_HTML_URL
+        assert "
      Herman Melville - Moby-Dick
      " in result["html"]
+
+
+    # 2. Multi-URL and Dispatcher
+    async def test_multi_url_crawl(self, async_client: httpx.AsyncClient):
+        """Test /crawl with multiple URLs, implicitly testing dispatcher."""
+        urls = [SIMPLE_HTML_URL, "https://httpbin.org/links/10/0"]
+        payload = {
+            "urls": urls,
+            "browser_config": {
+                "type": "BrowserConfig",
+                "params": {"headless": True}
+            },
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {"stream": False, "cache_mode": CacheMode.BYPASS.value}
+            }
+        }
+        try:
+            print(f"Sending deep crawl request to server...")
+            response = await async_client.post("/crawl", json=payload)
+            print(f"Response status: {response.status_code}")
+            
+            if response.status_code >= 400:
+                error_detail = response.json().get('detail', 'No detail provided')
+                print(f"Error detail: {error_detail}")
+                print(f"Full response: {response.text}")
+            
+            response.raise_for_status()
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            print(f"Server error status: {e.response.status_code}")
+            print(f"Server error response: {e.response.text}")
+            try:
+                error_json = e.response.json()
+                print(f"Parsed error: {error_json}")
+            except:
+                print("Could not parse error response as JSON")
+            raise
+
+        assert data["success"] is True
+        assert isinstance(data["results"], list)
+        assert len(data["results"]) == len(urls)
+        for result in data["results"]:
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            assert result["url"] in urls
+
+    async def test_multi_url_crawl_streaming(self, async_client: httpx.AsyncClient):
+        """Test /crawl/stream with multiple URLs."""
+        urls = [SIMPLE_HTML_URL, "https://httpbin.org/links/10/0"]
+        payload = {
+            "urls": urls,
+            "browser_config": {
+                "type": "BrowserConfig",
+                "params": {"headless": True}
+            },
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {"stream": True, "cache_mode": CacheMode.BYPASS.value}
+            }
+        }
+        async with async_client.stream("POST", "/crawl/stream", json=payload) as response:
+            response.raise_for_status()
+            results = await process_streaming_response(response)
+
+        assert len(results) == len(urls)
+        processed_urls = set()
+        for result in results:
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            assert result["url"] in urls
+            processed_urls.add(result["url"])
+        assert processed_urls == set(urls) # Ensure all URLs were processed
+
+
+    # 3. Class Values and Nested Classes (Markdown Generator)
+    async def test_crawl_with_markdown_pruning_filter(self, async_client: httpx.AsyncClient):
+        """Test /crawl with MarkdownGenerator using PruningContentFilter."""
+        payload = {
+            "urls": [SIMPLE_HTML_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "cache_mode": CacheMode.ENABLED.value, # Test different cache mode
+                    "markdown_generator": {
+                        "type": "DefaultMarkdownGenerator",
+                        "params": {
+                            "content_filter": {
+                                "type": "PruningContentFilter",
+                                "params": {
+                                    "threshold": 0.5, # Example param
+                                    "threshold_type": "relative"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        try:
+            print(f"Sending deep crawl request to server...")
+            response = await async_client.post("/crawl", json=payload)
+            print(f"Response status: {response.status_code}")
+            
+            if response.status_code >= 400:
+                error_detail = response.json().get('detail', 'No detail provided')
+                print(f"Error detail: {error_detail}")
+                print(f"Full response: {response.text}")
+            
+            response.raise_for_status()
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            print(f"Server error status: {e.response.status_code}")
+            print(f"Server error response: {e.response.text}")
+            try:
+                error_json = e.response.json()
+                print(f"Parsed error: {error_json}")
+            except:
+                print("Could not parse error response as JSON")
+            raise
+
+        assert data["success"] is True
+        assert len(data["results"]) == 1
+        result = data["results"][0]
+        await assert_crawl_result_structure(result)
+        assert result["success"] is True
+        assert "markdown" in result
+        assert isinstance(result["markdown"], dict)
+        assert "raw_markdown" in result["markdown"]
+        assert "fit_markdown" in result["markdown"] # Pruning creates fit_markdown
+        assert "Moby-Dick" in result["markdown"]["raw_markdown"]
+        # Fit markdown content might be different/shorter due to pruning
+        assert len(result["markdown"]["fit_markdown"]) <= len(result["markdown"]["raw_markdown"])
+
+    async def test_crawl_with_markdown_bm25_filter(self, async_client: httpx.AsyncClient):
+        """Test /crawl with MarkdownGenerator using BM25ContentFilter."""
+        payload = {
+            "urls": [SIMPLE_HTML_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "markdown_generator": {
+                        "type": "DefaultMarkdownGenerator",
+                        "params": {
+                            "content_filter": {
+                                "type": "BM25ContentFilter",
+                                "params": {
+                                    "user_query": "Herman Melville", # Query for BM25
+                                    "bm25_threshold": 0.1, # Lower threshold to increase matches
+                                    "language": "english"  # Valid parameters
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        try:
+            print(f"Payload for BM25 test: {json.dumps(payload)}")
+            response = await async_client.post("/crawl", json=payload)
+            print(f"Response status: {response.status_code}")
+            
+            if response.status_code >= 400:
+                error_detail = response.json().get('detail', 'No detail provided')
+                print(f"Error detail: {error_detail}")
+                print(f"Full response: {response.text}")
+            
+            response.raise_for_status()
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            print(f"Server error status: {e.response.status_code}")
+            print(f"Server error response: {e.response.text}")
+            try:
+                error_json = e.response.json()
+                print(f"Parsed error: {error_json}")
+            except:
+                print("Could not parse error response as JSON")
+            raise
+
+        assert data["success"] is True
+        assert len(data["results"]) == 1
+        result = data["results"][0]
+        await assert_crawl_result_structure(result)
+        assert result["success"] is True
+        assert "markdown" in result
+        assert isinstance(result["markdown"], dict)
+        assert "raw_markdown" in result["markdown"]
+        assert "fit_markdown" in result["markdown"] # BM25 creates fit_markdown
+        
+        # Print values for debug
+        print(f"Raw markdown length: {len(result['markdown']['raw_markdown'])}")
+        print(f"Fit markdown length: {len(result['markdown']['fit_markdown'])}")
+        
+        # Either fit_markdown has content (possibly including our query terms)
+        # or it might be empty if no good BM25 matches were found
+        # Don't assert specific content since it can be environment-dependent
+
+
+    # 4. Deep Crawling
+    async def test_deep_crawl(self, async_client: httpx.AsyncClient):
+        """Test /crawl with a deep crawl strategy."""
+        payload = {
+            "urls": [DEEP_CRAWL_URL], # Start URL
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "stream": False,
+                    "cache_mode": CacheMode.BYPASS.value,
+                    "deep_crawl_strategy": {
+                        "type": "BFSDeepCrawlStrategy",
+                        "params": {
+                            "max_depth": 1, # Limit depth for testing speed
+                            "max_pages": 5, # Limit pages to crawl
+                            "filter_chain": {
+                                "type": "FilterChain",
+                                "params": {
+                                    "filters": [
+                                        {
+                                            "type": "ContentTypeFilter",
+                                            "params": {"allowed_types": ["text/html"]}
+                                        },
+                                        {
+                                            "type": "DomainFilter",
+                                            "params": {"allowed_domains": ["python.org", "docs.python.org"]} # Include important subdomains
+                                        }
+                                    ]
+                                }
+                            },
+                            "url_scorer": {
+                                "type": "CompositeScorer",
+                                "params": {
+                                    "scorers": [
+                                        {
+                                            "type": "KeywordRelevanceScorer",
+                                            "params": {"keywords": ["documentation", "tutorial"]}
+                                        },
+                                        {
+                                            "type": "PathDepthScorer",
+                                            "params": {"weight": 0.5, "optimal_depth": 2}
+                                        }
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        try:
+            print(f"Sending deep crawl request to server...")
+            response = await async_client.post("/crawl", json=payload)
+            print(f"Response status: {response.status_code}")
+            
+            if response.status_code >= 400:
+                error_detail = response.json().get('detail', 'No detail provided')
+                print(f"Error detail: {error_detail}")
+                print(f"Full response: {response.text}")
+            
+            response.raise_for_status()
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            print(f"Server error status: {e.response.status_code}")
+            print(f"Server error response: {e.response.text}")
+            try:
+                error_json = e.response.json()
+                print(f"Parsed error: {error_json}")
+            except:
+                print("Could not parse error response as JSON")
+            raise
+
+        assert data["success"] is True
+        assert isinstance(data["results"], list)
+        # Expect more than 1 result due to deep crawl (start URL + crawled links)
+        assert len(data["results"]) > 1
+        assert len(data["results"]) <= 6 # Start URL + max_links=5
+
+        start_url_found = False
+        crawled_urls_found = False
+        for result in data["results"]:
+            await assert_crawl_result_structure(result)
+            assert result["success"] is True
+            
+            # Print URL for debugging
+            print(f"Crawled URL: {result['url']}")
+            
+            # Allow URLs that contain python.org (including subdomains like docs.python.org)
+            assert "python.org" in result["url"]
+            if result["url"] == DEEP_CRAWL_URL:
+                start_url_found = True
+            else:
+                crawled_urls_found = True
+
+        assert start_url_found
+        assert crawled_urls_found
+
+
+    # 5. Extraction without LLM (JSON/CSS)
+    async def test_json_css_extraction(self, async_client: httpx.AsyncClient):
+        """Test /crawl with JsonCssExtractionStrategy."""
+        payload = {
+            "urls": [SCRAPE_TARGET_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "cache_mode": CacheMode.BYPASS.value,
+                    "extraction_strategy": {
+                        "type": "JsonCssExtractionStrategy",
+                        "params": {
+                            "schema": { 
+                                "type": "dict", # IMPORTANT: Wrap schema dict with type/value structure
+                                "value": {
+                                    "name": "BookList",
+                                    "baseSelector": "ol.row li.col-xs-6", # Select each book item
+                                    "fields": [
+                                        {"name": "title", "selector": "article.product_pod h3 a", "type": "attribute", "attribute": "title"},
+                                        {"name": "price", "selector": "article.product_pod .price_color", "type": "text"},
+                                        {"name": "rating", "selector": "article.product_pod p.star-rating", "type": "attribute", "attribute": "class"}
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        try:
+            print(f"Sending deep crawl request to server...")
+            response = await async_client.post("/crawl", json=payload)
+            print(f"Response status: {response.status_code}")
+            
+            if response.status_code >= 400:
+                error_detail = response.json().get('detail', 'No detail provided')
+                print(f"Error detail: {error_detail}")
+                print(f"Full response: {response.text}")
+            
+            response.raise_for_status()
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            print(f"Server error status: {e.response.status_code}")
+            print(f"Server error response: {e.response.text}")
+            try:
+                error_json = e.response.json()
+                print(f"Parsed error: {error_json}")
+            except:
+                print("Could not parse error response as JSON")
+            raise
+
+        assert data["success"] is True
+        assert len(data["results"]) == 1
+        result = data["results"][0]
+        await assert_crawl_result_structure(result)
+        assert result["success"] is True
+        assert "extracted_content" in result
+        assert result["extracted_content"] is not None
+
+        # Extracted content should be a JSON string representing a list of dicts
+        try:
+            extracted_data = json.loads(result["extracted_content"])
+            assert isinstance(extracted_data, list)
+            assert len(extracted_data) > 0 # Should find some books
+            # Check structure of the first extracted item
+            first_item = extracted_data[0]
+            assert "title" in first_item
+            assert "price" in first_item
+            assert "rating" in first_item
+            assert "star-rating" in first_item["rating"] # e.g., "star-rating Three"
+        except (json.JSONDecodeError, AssertionError) as e:
+            pytest.fail(f"Extracted content parsing or validation failed: {e}\nContent: {result['extracted_content']}")
+
+
+    # 6. Extraction with LLM
+    async def test_llm_extraction(self, async_client: httpx.AsyncClient):
+        """
+        Test /crawl with LLMExtractionStrategy.
+        NOTE: Requires the server to have appropriate LLM API keys (e.g., OPENAI_API_KEY)
+              configured via .llm.env or environment variables.
+              This test uses the default provider configured in the server's config.yml.
+        """
+        payload = {
+            "urls": [SIMPLE_HTML_URL],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "cache_mode": CacheMode.BYPASS.value,
+                    "extraction_strategy": {
+                        "type": "LLMExtractionStrategy",
+                        "params": {
+                            "instruction": "Extract the main title and the author mentioned in the text into JSON.",
+                            # LLMConfig is implicitly defined by server's config.yml and .llm.env
+                            # If you needed to override provider/token PER REQUEST:
+                            "llm_config": {
+                               "type": "LLMConfig",
+                               "params": {
+                                  "provider": "openai/gpt-4o", # Example override
+                                  "api_token": os.getenv("OPENAI_API_KEY") # Example override
+                               }
+                            },
+                            "schema": { # Optional: Provide a schema for structured output
+                                "type": "dict", # IMPORTANT: Wrap schema dict
+                                "value": {
+                                    "title": "Book Info",
+                                    "type": "object",
+                                    "properties": {
+                                        "title": {"type": "string", "description": "The main title of the work"},
+                                        "author": {"type": "string", "description": "The author of the work"}
+                                    },
+                                     "required": ["title", "author"]
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        try:
+            response = await async_client.post("/crawl", json=payload)
+            response.raise_for_status() # Will raise if server returns 500 (e.g., bad API key)
+            data = response.json()
+        except httpx.HTTPStatusError as e:
+            # Catch potential server errors (like 500 due to missing/invalid API keys)
+            pytest.fail(f"LLM extraction request failed: {e}. Response: {e.response.text}. Check server logs and ensure API keys are correctly configured for the server.")
+        except httpx.RequestError as e:
+             pytest.fail(f"LLM extraction request failed: {e}.")
+
+        assert data["success"] is True
+        assert len(data["results"]) == 1
+        result = data["results"][0]
+        await assert_crawl_result_structure(result)
+        assert result["success"] is True
+        assert "extracted_content" in result
+        assert result["extracted_content"] is not None
+
+        # Extracted content should be JSON (because we provided a schema)
+        try:
+            extracted_data = json.loads(result["extracted_content"])
+            print(f"\nLLM Extracted Data: {extracted_data}") # Print for verification
+            
+            # Handle both dict and list formats (server returns a list)
+            if isinstance(extracted_data, list):
+                assert len(extracted_data) > 0
+                extracted_item = extracted_data[0]  # Take first item
+                assert isinstance(extracted_item, dict)
+                assert "title" in extracted_item
+                assert "author" in extracted_item
+                assert "Moby-Dick" in extracted_item.get("title", "")
+                assert "Herman Melville" in extracted_item.get("author", "")
+            else:
+                assert isinstance(extracted_data, dict)
+                assert "title" in extracted_data
+                assert "author" in extracted_data
+                assert "Moby-Dick" in extracted_data.get("title", "")
+                assert "Herman Melville" in extracted_data.get("author", "")
+        except (json.JSONDecodeError, AssertionError) as e:
+            pytest.fail(f"LLM extracted content parsing or validation failed: {e}\nContent: {result['extracted_content']}")
+        except Exception as e: # Catch any other unexpected error
+            pytest.fail(f"An unexpected error occurred during LLM result processing: {e}\nContent: {result['extracted_content']}")
+
+
+    # 7. Error Handling Tests
+    async def test_invalid_url_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for invalid URLs."""
+        payload = {
+            "urls": ["invalid-url", "https://nonexistent-domain-12345.com"],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {"type": "CrawlerRunConfig", "params": {"cache_mode": CacheMode.BYPASS.value}}
+        }
+        
+        response = await async_client.post("/crawl", json=payload)
+        # Should return 200 with failed results, not 500
+        print(f"Status code: {response.status_code}")
+        print(f"Response: {response.text}")
+        assert response.status_code == 500
+        data = response.json()
+        assert data["detail"].startswith("Crawl request failed:")
+
+    async def test_mixed_success_failure_urls(self, async_client: httpx.AsyncClient):
+        """Test handling of mixed success/failure URLs."""
+        payload = {
+            "urls": [
+                SIMPLE_HTML_URL,  # Should succeed
+                "https://nonexistent-domain-12345.com",  # Should fail
+                "https://invalid-url-with-special-chars-!@#$%^&*()",  # Should fail
+            ],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig", 
+                "params": {
+                    "cache_mode": CacheMode.BYPASS.value,
+                    "markdown_generator": {
+                        "type": "DefaultMarkdownGenerator",
+                        "params": {
+                            "content_filter": {
+                                "type": "PruningContentFilter",
+                                "params": {"threshold": 0.5}
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        
+        response = await async_client.post("/crawl", json=payload)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] is True
+        assert len(data["results"]) == 3
+        
+        success_count = 0
+        failure_count = 0
+        
+        for result in data["results"]:
+            if result["success"]:
+                success_count += 1
+            else:
+                failure_count += 1
+                assert "error_message" in result
+                assert len(result["error_message"]) > 0
+                
+        assert success_count >= 1  # At least one should succeed
+        assert failure_count >= 1  # At least one should fail
+
+    async def test_streaming_mixed_urls(self, async_client: httpx.AsyncClient):
+        """Test streaming with mixed success/failure URLs."""
+        payload = {
+            "urls": [
+                SIMPLE_HTML_URL,  # Should succeed
+                "https://nonexistent-domain-12345.com",  # Should fail
+            ],
+            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+            "crawler_config": {
+                "type": "CrawlerRunConfig", 
+                "params": {
+                    "stream": True,
+                    "cache_mode": CacheMode.BYPASS.value
+                }
+            }
+        }
+        
+        async with async_client.stream("POST", "/crawl/stream", json=payload) as response:
+            response.raise_for_status()
+            results = await process_streaming_response(response)
+        
+        assert len(results) == 2
+        
+        success_count = 0
+        failure_count = 0
+        
+        for result in results:
+            if result["success"]:
+                success_count += 1
+                assert result["url"] == SIMPLE_HTML_URL
+            else:
+                failure_count += 1
+                assert "error_message" in result
+                assert result["error_message"] is not None
+        
+        assert success_count == 1
+        assert failure_count == 1
+
+    async def test_markdown_endpoint_error_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for markdown endpoint."""
+        # Test invalid URL
+        invalid_payload = {"url": "invalid-url", "f": "fit"}
+        response = await async_client.post("/md", json=invalid_payload)
+        # Should return 400 for invalid URL format
+        assert response.status_code == 400
+        
+        # Test non-existent URL
+        nonexistent_payload = {"url": "https://nonexistent-domain-12345.com", "f": "fit"}
+        response = await async_client.post("/md", json=nonexistent_payload)
+        # Should return 500 for crawl failure
+        assert response.status_code == 500
+
+    async def test_html_endpoint_error_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for HTML endpoint."""
+        # Test invalid URL
+        invalid_payload = {"url": "invalid-url"}
+        response = await async_client.post("/html", json=invalid_payload)
+        # Should return 500 for crawl failure
+        assert response.status_code == 500
+
+    async def test_screenshot_endpoint_error_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for screenshot endpoint."""
+        # Test invalid URL
+        invalid_payload = {"url": "invalid-url"}
+        response = await async_client.post("/screenshot", json=invalid_payload)
+        # Should return 500 for crawl failure
+        assert response.status_code == 500
+
+    async def test_pdf_endpoint_error_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for PDF endpoint."""
+        # Test invalid URL
+        invalid_payload = {"url": "invalid-url"}
+        response = await async_client.post("/pdf", json=invalid_payload)
+        # Should return 500 for crawl failure
+        assert response.status_code == 500
+
+    async def test_execute_js_endpoint_error_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for execute_js endpoint."""
+        # Test invalid URL
+        invalid_payload = {"url": "invalid-url", "scripts": ["return document.title;"]}
+        response = await async_client.post("/execute_js", json=invalid_payload)
+        # Should return 500 for crawl failure
+        assert response.status_code == 500
+
+    async def test_llm_endpoint_error_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for LLM endpoint."""
+        # Test missing query parameter
+        response = await async_client.get("/llm/https://example.com")
+        assert response.status_code == 422  # FastAPI validation error, not 400
+        
+        # Test invalid URL
+        response = await async_client.get("/llm/invalid-url?q=test")
+        # Should return 500 for crawl failure
+        assert response.status_code == 500
+
+    async def test_ask_endpoint_error_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for ask endpoint."""
+        # Test invalid context_type
+        response = await async_client.get("/ask?context_type=invalid")
+        assert response.status_code == 422  # Validation error
+        
+        # Test invalid score_ratio
+        response = await async_client.get("/ask?score_ratio=2.0")  # > 1.0
+        assert response.status_code == 422  # Validation error
+        
+        # Test invalid max_results
+        response = await async_client.get("/ask?max_results=0")  # < 1
+        assert response.status_code == 422  # Validation error
+
+    async def test_config_dump_error_handling(self, async_client: httpx.AsyncClient):
+        """Test error handling for config dump endpoint."""
+        # Test invalid code
+        invalid_payload = {"code": "invalid_code"}
+        response = await async_client.post("/config/dump", json=invalid_payload)
+        assert response.status_code == 400
+        
+        # Test nested function calls (not allowed)
+        nested_payload = {"code": "CrawlerRunConfig(BrowserConfig())"}
+        response = await async_client.post("/config/dump", json=nested_payload)
+        assert response.status_code == 400
+
+    async def test_malformed_request_handling(self, async_client: httpx.AsyncClient):
+        """Test handling of malformed requests."""
+        # Test missing required fields
+        malformed_payload = {"urls": []}  # Missing browser_config and crawler_config
+        response = await async_client.post("/crawl", json=malformed_payload)
+        print(f"Response: {response.text}")
+        assert response.status_code == 422  # Validation error
+        
+        # Test empty URLs list
+        empty_urls_payload = {
+            "urls": [],
+            "browser_config": {"type": "BrowserConfig", "params": {}},
+            "crawler_config": {"type": "CrawlerRunConfig", "params": {}}
+        }
+        response = await async_client.post("/crawl", json=empty_urls_payload)
+        assert response.status_code == 422  # "At least one URL required"
+
+if __name__ == "__main__":
+    # Define arguments for pytest programmatically
+    # -v: verbose output
+    # -s: show print statements immediately (useful for debugging)
+    # __file__: tells pytest to run tests in the current file
+    pytest_args = ["-v", "-s", __file__]
+
+    # You can add more pytest arguments here if needed, for example:
+    # '-k test_llm_extraction': Run only the LLM test function
+    # pytest_args.append("-k test_llm_extraction")
+
+    print(f"Running pytest with args: {pytest_args}")
+
+    # Execute pytest
+    exit_code = pytest.main(pytest_args)
+
+    print(f"Pytest finished with exit code: {exit_code}")
\ No newline at end of file
diff --git a/tests/docker/test_server_token.py b/tests/docker/test_server_token.py
new file mode 100644
index 0000000..220b6ca
--- /dev/null
+++ b/tests/docker/test_server_token.py
@@ -0,0 +1,212 @@
+import asyncio
+import json
+from typing import Optional
+from urllib.parse import quote
+
+async def get_token(session, email: str = "test@example.com") -> str:
+    """Fetch a JWT token from the /token endpoint."""
+    url = "http://localhost:8000/token"
+    payload = {"email": email}
+    print(f"\nFetching token from {url} with email: {email}")
+    try:
+        async with session.post(url, json=payload) as response:
+            status = response.status
+            data = await response.json()
+            print(f"Token Response Status: {status}")
+            print(f"Token Response: {json.dumps(data, indent=2)}")
+            if status == 200:
+                return data["access_token"]
+            else:
+                raise Exception(f"Failed to get token: {data.get('detail', 'Unknown error')}")
+    except Exception as e:
+        print(f"Error fetching token: {str(e)}")
+        raise
+
+async def test_endpoint(
+    session,
+    endpoint: str,
+    url: str,
+    token: str,
+    params: Optional[dict] = None,
+    expected_status: int = 200
+) -> Optional[dict]:
+    """Test an endpoint with token and print results."""
+    params = params or {}
+    param_str = "&".join(f"{k}={v}" for k, v in params.items())
+    full_url = f"http://localhost:8000/{endpoint}/{quote(url)}"
+    if param_str:
+        full_url += f"?{param_str}"
+    
+    headers = {"Authorization": f"Bearer {token}"}
+    print(f"\nTesting: {full_url}")
+    
+    try:
+        async with session.get(full_url, headers=headers) as response:
+            status = response.status
+            try:
+                data = await response.json()
+            except:
+                data = await response.text()
+            
+            print(f"Status: {status} (Expected: {expected_status})")
+            if isinstance(data, dict):
+                print(f"Response: {json.dumps(data, indent=2)}")
+            else:
+                print(f"Response: {data[:500]}...")  # First 500 chars
+            assert status == expected_status, f"Expected {expected_status}, got {status}"
+            return data
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        return None
+
+
+async def test_stream_crawl(session, token: str):
+    """Test the /crawl/stream endpoint with multiple URLs."""
+    url = "http://localhost:8000/crawl/stream"
+    payload = {
+        "urls": [
+            "https://example.com",
+            "https://example.com/page1",  # Replicated example.com with variation
+            "https://example.com/page2",  # Replicated example.com with variation
+            "https://example.com/page3",  # Replicated example.com with variation
+            # "https://www.python.org",
+            # "https://news.ycombinator.com/news"
+        ],
+        "browser_config": {"headless": True, "viewport": {"width": 1200}},
+        "crawler_config": {"stream": True, "cache_mode": "bypass"}
+    }
+    headers = {"Authorization": f"Bearer {token}"}
+    print(f"\nTesting Streaming Crawl: {url}")
+    print(f"Payload: {json.dumps(payload, indent=2)}")
+    
+    try:
+        async with session.post(url, json=payload, headers=headers) as response:
+            status = response.status
+            print(f"Status: {status} (Expected: 200)")
+            assert status == 200, f"Expected 200, got {status}"
+            
+            # Read streaming response line-by-line (NDJSON)
+            async for line in response.content:
+                if line:
+                    data = json.loads(line.decode('utf-8').strip())
+                    print(f"Streamed Result: {json.dumps(data, indent=2)}")
+    except Exception as e:
+        print(f"Error in streaming crawl test: {str(e)}")
+
+async def run_tests():
+    import aiohttp
+    print("Starting API Tests...")
+    
+    # Test URLs
+    urls = [
+        "example.com",
+        "https://www.python.org",
+        "https://news.ycombinator.com/news",
+        "https://github.com/trending"
+    ]
+    
+    async with aiohttp.ClientSession() as session:
+        # Fetch token once and reuse it
+        token = await get_token(session)
+        if not token:
+            print("Aborting tests due to token failure!")
+            return
+        
+        print("\n=== Testing Crawl Endpoint ===")
+        crawl_payload = {
+            "urls": ["https://example.com"],
+            "browser_config": {"headless": True},
+            "crawler_config": {"stream": False}
+        }
+        async with session.post(
+            "http://localhost:8000/crawl",
+            json=crawl_payload,
+            headers={"Authorization": f"Bearer {token}"}
+        ) as response:
+            status = response.status
+            data = await response.json()
+            print(f"\nCrawl Endpoint Status: {status}")
+            print(f"Crawl Response: {json.dumps(data, indent=2)}")
+        
+
+        print("\n=== Testing Crawl Stream Endpoint ===")
+        await test_stream_crawl(session, token)
+
+        print("\n=== Testing Markdown Endpoint ===")
+        for url in []: #urls:
+            for filter_type in ["raw", "fit", "bm25", "llm"]:
+                params = {"f": filter_type}
+                if filter_type in ["bm25", "llm"]:
+                    params["q"] = "extract main content"
+                
+                for cache in ["0", "1"]:
+                    params["c"] = cache
+                    await test_endpoint(session, "md", url, token, params)
+                    await asyncio.sleep(1)  # Be nice to the server
+
+        print("\n=== Testing LLM Endpoint ===")
+        for url in urls:
+            # Test basic extraction (direct response now)
+            result = await test_endpoint(
+                session,
+                "llm",
+                url,
+                token,
+                {"q": "Extract title and main content"}
+            )
+            
+            # Test with schema (direct response)
+            schema = {
+                "type": "object",
+                "properties": {
+                    "title": {"type": "string"},
+                    "content": {"type": "string"},
+                    "links": {"type": "array", "items": {"type": "string"}}
+                }
+            }
+            result = await test_endpoint(
+                session,
+                "llm",
+                url,
+                token,
+                {
+                    "q": "Extract content with links",
+                    "s": json.dumps(schema),
+                    "c": "1"  # Test with cache
+                }
+            )
+            await asyncio.sleep(2)  # Be nice to the server
+        
+        print("\n=== Testing Error Cases ===")
+        # Test invalid URL
+        await test_endpoint(
+            session,
+            "md",
+            "not_a_real_url",
+            token,
+            expected_status=500
+        )
+        
+        # Test invalid filter type
+        await test_endpoint(
+            session,
+            "md",
+            "example.com",
+            token,
+            {"f": "invalid"},
+            expected_status=422
+        )
+        
+        # Test LLM without query (should fail per your server logic)
+        await test_endpoint(
+            session,
+            "llm",
+            "example.com",
+            token,
+            expected_status=400
+        )
+        
+    print("\nAll tests completed!")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())
\ No newline at end of file
diff --git a/tests/docker_example.py b/tests/docker_example.py
new file mode 100644
index 0000000..f661ecc
--- /dev/null
+++ b/tests/docker_example.py
@@ -0,0 +1,397 @@
+import requests
+import json
+import time
+import sys
+import base64
+import os
+from typing import Dict, Any
+
+class Crawl4AiTester:
+    def __init__(self, base_url: str = "http://localhost:11235"):
+        self.base_url = base_url
+
+
+    def submit_and_wait(
+        self, request_data: Dict[str, Any], timeout: int = 300
+    ) -> Dict[str, Any]:
+        # Submit crawl job using async endpoint
+        response = requests.post(
+            f"{self.base_url}/crawl/job", json=request_data
+        )
+        response.raise_for_status()
+        job_response = response.json()
+        task_id = job_response["task_id"]
+        print(f"Submitted job with task_id: {task_id}")
+
+        # Poll for result
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Task {task_id} did not complete within {timeout} seconds"
+                )
+
+            result = requests.get(
+                f"{self.base_url}/crawl/job/{task_id}"
+            )
+            result.raise_for_status()
+            status = result.json()
+
+            if status["status"] == "failed":
+                print("Task failed:", status.get("error"))
+                raise Exception(f"Task failed: {status.get('error')}")
+
+            if status["status"] == "completed":
+                return status
+
+            time.sleep(2)
+
+    def submit_sync(self, request_data: Dict[str, Any]) -> Dict[str, Any]:
+        # Use synchronous crawl endpoint
+        response = requests.post(
+            f"{self.base_url}/crawl",
+            json=request_data,
+            timeout=60,
+        )
+        if response.status_code == 408:
+            raise TimeoutError("Task did not complete within server timeout")
+        response.raise_for_status()
+        return response.json()
+
+
+def test_docker_deployment(version="basic"):
+    tester = Crawl4AiTester(
+        base_url="http://localhost:11235",
+        #base_url="https://crawl4ai-sby74.ondigitalocean.app",
+    )
+    print(f"Testing Crawl4AI Docker {version} version")
+
+    # Health check with timeout and retry
+    max_retries = 5
+    for i in range(max_retries):
+        try:
+            health = requests.get(f"{tester.base_url}/health", timeout=10)
+            print("Health check:", health.json())
+            break
+        except requests.exceptions.RequestException:
+            if i == max_retries - 1:
+                print(f"Failed to connect after {max_retries} attempts")
+                sys.exit(1)
+            print(f"Waiting for service to start (attempt {i+1}/{max_retries})...")
+            time.sleep(5)
+
+    # Test cases based on version
+    test_basic_crawl(tester)
+    test_basic_crawl_sync(tester)
+
+    if version in ["full", "transformer"]:
+        test_cosine_extraction(tester)
+
+    test_js_execution(tester)
+    test_css_selector(tester)
+    test_structured_extraction(tester)
+    test_llm_extraction(tester)
+    test_llm_with_ollama(tester)
+    test_screenshot(tester)
+
+
+def test_basic_crawl(tester: Crawl4AiTester):
+    print("\n=== Testing Basic Crawl (Async) ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+    }
+
+    result = tester.submit_and_wait(request)
+    print(f"Basic crawl result count: {len(result['result']['results'])}")
+    assert result["result"]["success"]
+    assert len(result["result"]["results"]) > 0
+    assert len(result["result"]["results"][0]["markdown"]) > 0
+
+
+def test_basic_crawl_sync(tester: Crawl4AiTester):
+    print("\n=== Testing Basic Crawl (Sync) ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+    }
+
+    result = tester.submit_sync(request)
+    print(f"Basic crawl result count: {len(result['results'])}")
+    assert result["success"]
+    assert len(result["results"]) > 0
+    assert len(result["results"][0]["markdown"]) > 0
+
+
+def test_js_execution(tester: Crawl4AiTester):
+    print("\n=== Testing JS Execution ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {"headless": True},
+        "crawler_config": {
+            "js_code": [
+                "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); if(loadMoreButton) loadMoreButton.click();"
+            ],
+            "wait_for": "wide-tease-item__wrapper df flex-column flex-row-m flex-nowrap-m enable-new-sports-feed-mobile-design(10)"
+        }
+    }
+
+    result = tester.submit_and_wait(request)
+    print(f"JS execution result count: {len(result['result']['results'])}")
+    assert result["result"]["success"]
+
+
+def test_css_selector(tester: Crawl4AiTester):
+    print("\n=== Testing CSS Selector ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {"headless": True},
+        "crawler_config": {
+            "css_selector": ".wide-tease-item__description",
+            "word_count_threshold": 10
+        }
+    }
+
+    result = tester.submit_and_wait(request)
+    print(f"CSS selector result count: {len(result['result']['results'])}")
+    assert result["result"]["success"]
+
+
+def test_structured_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing Structured Extraction ===")
+    schema = {
+  "name": "Cryptocurrency Prices",
+  "baseSelector": "table[data-testid=\"prices-table\"] tbody tr",
+  "fields": [
+    {
+      "name": "asset_name",
+      "selector": "td:nth-child(2) p.cds-headline-h4steop",
+      "type": "text"
+    },
+    {
+      "name": "asset_symbol",
+      "selector": "td:nth-child(2) p.cds-label2-l1sm09ec",
+      "type": "text"
+    },
+    {
+      "name": "asset_image_url",
+      "selector": "td:nth-child(2) img[alt=\"Asset Symbol\"]",
+      "type": "attribute",
+      "attribute": "src"
+    },
+    {
+      "name": "asset_url",
+      "selector": "td:nth-child(2) a[aria-label^=\"Asset page for\"]",
+      "type": "attribute",
+      "attribute": "href"
+    },
+    {
+      "name": "price",
+      "selector": "td:nth-child(3) div.cds-typographyResets-t6muwls.cds-body-bwup3gq",
+      "type": "text"
+    },
+    {
+      "name": "change",
+      "selector": "td:nth-child(7) p.cds-body-bwup3gq",
+      "type": "text"
+    }
+  ]
+}
+
+
+    request = {
+        "urls": ["https://www.coinbase.com/explore"],
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "JsonCssExtractionStrategy",
+                    "params": {"schema": schema}
+                }
+            }
+        }
+    }
+
+    result = tester.submit_and_wait(request)
+    extracted = json.loads(result["result"]["results"][0]["extracted_content"])
+    print(f"Extracted {len(extracted)} items")
+    if extracted:
+        print("Sample item:", json.dumps(extracted[0], indent=2))
+    assert result["result"]["success"]
+    assert len(extracted) > 0
+
+
+def test_llm_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing LLM Extraction ===")
+    schema = {
+        "type": "object",
+        "properties": {
+            "asset_name": {
+                "type": "string",
+                "description": "Name of the asset.",
+            },
+            "price": {
+                "type": "string",
+                "description": "Price of the asset.",
+            },
+            "change": {
+                "type": "string",
+                "description": "Change in price of the asset.",
+            },
+        },
+        "required": ["asset_name", "price", "change"],
+    }
+
+    request = {
+        "urls": ["https://www.coinbase.com/en-in/explore"],
+        "browser_config": {},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "LLMExtractionStrategy",
+                    "params": {
+                        "llm_config": {
+                            "type": "LLMConfig",
+                            "params": {
+                                "provider": "gemini/gemini-2.5-flash",
+                                "api_token": os.getenv("GEMINI_API_KEY")
+                            }
+                        },
+                        "schema": schema,
+                        "extraction_type": "schema",
+                        "instruction": "From the crawled content    tioned asset names along with their prices and change in price.",
+                    }
+                },
+                "word_count_threshold": 1
+            }
+        }
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["results"][0]["extracted_content"])
+        print(f"Extracted {len(extracted)} model pricing entries")
+        if extracted:
+            print("Sample entry:", json.dumps(extracted[0], indent=2))
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"LLM extraction test failed (might be due to missing API key): {str(e)}")
+
+
+def test_llm_with_ollama(tester: Crawl4AiTester):
+    print("\n=== Testing LLM with Ollama ===")
+    schema = {
+        "type": "object",
+        "properties": {
+            "article_title": {
+                "type": "string",
+                "description": "The main title of the news article",
+            },
+            "summary": {
+                "type": "string",
+                "description": "A brief summary of the article content",
+            },
+            "main_topics": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Main topics or themes discussed in the article",
+            },
+        },
+    }
+
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {"verbose": True},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "LLMExtractionStrategy",
+                    "params": {
+                        "llm_config": {
+                            "type": "LLMConfig",
+                            "params": {
+                                "provider": "ollama/llama3.2:latest",
+                            }
+                        },
+                        "schema": schema,
+                        "extraction_type": "schema",
+                        "instruction": "Extract the main article information including title, summary, and main topics.",
+                    }
+                },
+                "word_count_threshold": 1
+            }
+        }
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["results"][0]["extracted_content"])
+        print("Extracted content:", json.dumps(extracted, indent=2))
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"Ollama extraction test failed: {str(e)}")
+
+
+def test_cosine_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing Cosine Extraction ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "CosineStrategy",
+                    "params": {
+                        "semantic_filter": "business finance economy",
+                        "word_count_threshold": 10,
+                        "max_dist": 0.2,
+                        "top_k": 3,
+                    }
+                }
+            }
+        }
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["results"][0]["extracted_content"])
+        print(f"Extracted {len(extracted)} text clusters")
+        if extracted:
+            print("First cluster tags:", extracted[0]["tags"])
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"Cosine extraction test failed: {str(e)}")
+
+
+def test_screenshot(tester: Crawl4AiTester):
+    print("\n=== Testing Screenshot ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "browser_config": {"headless": True},
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "screenshot": True
+            }
+        }
+    }
+
+    result = tester.submit_and_wait(request)
+    screenshot_data = result["result"]["results"][0]["screenshot"]
+    print("Screenshot captured:", bool(screenshot_data))
+
+    if screenshot_data:
+        # Save screenshot
+        screenshot_bytes = base64.b64decode(screenshot_data)
+        with open("test_screenshot.jpg", "wb") as f:
+            f.write(screenshot_bytes)
+        print("Screenshot saved as test_screenshot.jpg")
+
+    assert result["result"]["success"]
+
+
+if __name__ == "__main__":
+    version = sys.argv[1] if len(sys.argv) > 1 else "basic"
+    # version = "full"
+    test_docker_deployment(version)
diff --git a/tests/general/generate_dummy_site.py b/tests/general/generate_dummy_site.py
new file mode 100644
index 0000000..d4218b6
--- /dev/null
+++ b/tests/general/generate_dummy_site.py
@@ -0,0 +1,335 @@
+# ==== File: build_dummy_site.py ====
+
+import os
+import random
+import argparse
+from pathlib import Path
+from urllib.parse import quote
+
+# --- Configuration ---
+NUM_CATEGORIES = 3
+NUM_SUBCATEGORIES_PER_CAT = 2 # Results in NUM_CATEGORIES * NUM_SUBCATEGORIES_PER_CAT total L2 categories
+NUM_PRODUCTS_PER_SUBCAT = 5 # Products listed on L3 pages
+MAX_DEPTH_TARGET = 5 # Explicitly set target depth
+
+# --- Helper Functions ---
+
+def generate_lorem(words=20):
+    """Generates simple placeholder text."""
+    lorem_words = ["lorem", "ipsum", "dolor", "sit", "amet", "consectetur",
+                   "adipiscing", "elit", "sed", "do", "eiusmod", "tempor",
+                   "incididunt", "ut", "labore", "et", "dolore", "magna", "aliqua"]
+    return " ".join(random.choice(lorem_words) for _ in range(words)).capitalize() + "."
+
+def create_html_page(filepath: Path, title: str, body_content: str, breadcrumbs: list = [], head_extras: str = ""):
+    """Creates an HTML file with basic structure and inline CSS."""
+    os.makedirs(filepath.parent, exist_ok=True)
+
+    # Generate breadcrumb HTML using the 'link' provided in the breadcrumbs list
+    breadcrumb_html = ""
+    if breadcrumbs:
+        links_html = " » ".join(f'{bc["name"]}' for bc in breadcrumbs)
+        breadcrumb_html = f""
+
+    # Basic CSS for structure identification (kept the same)
+    css = """
+
+    """
+    html_content = f"""
+
+
+    
+    
+    {title} - FakeShop
+    {head_extras}
+    {css}
+
+
+    
      
+        {breadcrumb_html}
+        
      {title}
      
+        {body_content}
+    
      
+
+"""
+    with open(filepath, "w", encoding="utf-8") as f:
+        f.write(html_content)
+    # Keep print statement concise for clarity
+    # print(f"Created: {filepath}")
+
+def generate_site(base_dir: Path, site_name: str = "FakeShop", base_path: str = ""):
+    """Generates the dummy website structure."""
+    base_dir.mkdir(parents=True, exist_ok=True)
+
+    # --- Clean and prepare the base path for URL construction ---
+    # Ensure it starts with '/' if not empty, and remove any trailing '/'
+    if base_path:
+        full_base_path = "/" + base_path.strip('/')
+    else:
+        full_base_path = "" # Represents the root
+
+    print(f"Using base path for links: '{full_base_path}'")
+
+    # --- Level 0: Homepage ---
+    home_body = "
      Welcome to FakeShop!
      Your one-stop shop for imaginary items.
      Categories:
      \n
        "
+    # Define the *actual* link path for the homepage breadcrumb
+    home_link_path = f"{full_base_path}/index.html"
+    breadcrumbs_home = [{"name": "Home", "link": home_link_path}] # Base breadcrumb
+
+    # Links *within* the page content should remain relative
+    for i in range(NUM_CATEGORIES):
+        cat_name = f"Category-{i+1}"
+        cat_folder_name = quote(cat_name.lower().replace(" ", "-"))
+        # This path is relative to the current directory (index.html)
+        cat_relative_page_path = f"{cat_folder_name}/index.html"
+        home_body += f'
      • {cat_name} - {generate_lorem(10)}
        • '
+    home_body += "
      "
+    create_html_page(base_dir / "index.html", "Homepage", home_body, []) # No breadcrumbs *on* the homepage itself
+
+    # --- Levels 1-5 ---
+    for i in range(NUM_CATEGORIES):
+        cat_name = f"Category-{i+1}"
+        cat_folder_name = quote(cat_name.lower().replace(" ", "-"))
+        cat_dir = base_dir / cat_folder_name
+        # This is the *absolute* path for the breadcrumb link
+        cat_link_path = f"{full_base_path}/{cat_folder_name}/index.html"
+        # Update breadcrumbs list for this level
+        breadcrumbs_cat = breadcrumbs_home + [{"name": cat_name, "link": cat_link_path}]
+
+        # --- Level 1: Category Page ---
+        cat_body = f"
      {generate_lorem(15)} for {cat_name}.
      Sub-Categories:
      \n
        "
+        for j in range(NUM_SUBCATEGORIES_PER_CAT):
+            subcat_name = f"{cat_name}-Sub-{j+1}"
+            subcat_folder_name = quote(subcat_name.lower().replace(" ", "-"))
+            # Path relative to the category page
+            subcat_relative_page_path = f"{subcat_folder_name}/index.html"
+            cat_body += f'
      • {subcat_name} - {generate_lorem(8)}
        • '
+        cat_body += "
      "
+        # Pass the updated breadcrumbs list
+        create_html_page(cat_dir / "index.html", cat_name, cat_body, breadcrumbs_home) # Parent breadcrumb needed here
+
+        for j in range(NUM_SUBCATEGORIES_PER_CAT):
+            subcat_name = f"{cat_name}-Sub-{j+1}"
+            subcat_folder_name = quote(subcat_name.lower().replace(" ", "-"))
+            subcat_dir = cat_dir / subcat_folder_name
+            # Absolute path for the breadcrumb link
+            subcat_link_path = f"{full_base_path}/{cat_folder_name}/{subcat_folder_name}/index.html"
+            # Update breadcrumbs list for this level
+            breadcrumbs_subcat = breadcrumbs_cat + [{"name": subcat_name, "link": subcat_link_path}]
+
+            # --- Level 2: Sub-Category Page (Product List) ---
+            subcat_body = f"
      Explore products in {subcat_name}. {generate_lorem(12)}
      Products:
      \n
        "
+            for k in range(NUM_PRODUCTS_PER_SUBCAT):
+                prod_id = f"P{i+1}{j+1}{k+1:03d}" # e.g., P11001
+                prod_name = f"{subcat_name} Product {k+1} ({prod_id})"
+                # Filename relative to the subcategory page
+                prod_filename = f"product_{prod_id}.html"
+                # Absolute path for the breadcrumb link
+                prod_link_path = f"{full_base_path}/{cat_folder_name}/{subcat_folder_name}/{prod_filename}"
+
+                # Preview on list page (link remains relative)
+                subcat_body += f"""
+                
      • 
+                    
        
+                        {prod_name}
+                        
        {generate_lorem(10)}
        
+                        £{random.uniform(10, 500):.2f}
+                    
        
+                
        • """
+
+                # --- Level 3: Product Page ---
+                prod_price = random.uniform(10, 500)
+                prod_desc = generate_lorem(40)
+                prod_specs = {f"Spec {s+1}": generate_lorem(3) for s in range(random.randint(3,6))}
+                prod_reviews_count = random.randint(0, 150)
+                # Relative filenames for links on this page
+                details_filename_relative = f"product_{prod_id}_details.html"
+                reviews_filename_relative = f"product_{prod_id}_reviews.html"
+
+                prod_body = f"""
+                
        Price: £{prod_price:.2f}
        
+                
        
+                    
        Description
        
+                    
        {prod_desc}
        
+                
        
+                
        
+                    
        Specifications
        
+                    
          
+                        {''.join(f'
        • {name}: {value}
          • ' for name, value in prod_specs.items())}
+                    
        
+                
        
+                
        
+                    
        Reviews
        
+                    
        Total Reviews: {prod_reviews_count}
        
+                
        
+                
        
+                
        
+                    View More Details |
+                    See All Reviews
+                
        
+                """
+                # Update breadcrumbs list for this level
+                breadcrumbs_prod = breadcrumbs_subcat + [{"name": prod_name, "link": prod_link_path}]
+                # Pass the updated breadcrumbs list
+                create_html_page(subcat_dir / prod_filename, prod_name, prod_body, breadcrumbs_subcat) # Parent breadcrumb needed here
+
+                # --- Level 4: Product Details Page ---
+                details_filename = f"product_{prod_id}_details.html" # Actual filename
+                # Absolute path for the breadcrumb link
+                details_link_path = f"{full_base_path}/{cat_folder_name}/{subcat_folder_name}/{details_filename}"
+                details_body = f"
        This page contains extremely detailed information about {prod_name}.
        {generate_lorem(100)}"
+                # Update breadcrumbs list for this level
+                breadcrumbs_details = breadcrumbs_prod + [{"name": "Details", "link": details_link_path}]
+                # Pass the updated breadcrumbs list
+                create_html_page(subcat_dir / details_filename, f"{prod_name} - Details", details_body, breadcrumbs_prod) # Parent breadcrumb needed here
+
+                # --- Level 5: Product Reviews Page ---
+                reviews_filename = f"product_{prod_id}_reviews.html" # Actual filename
+                # Absolute path for the breadcrumb link
+                reviews_link_path = f"{full_base_path}/{cat_folder_name}/{subcat_folder_name}/{reviews_filename}"
+                reviews_body = f"
        All {prod_reviews_count} reviews for {prod_name} are listed here.
          "
+                for r in range(prod_reviews_count):
+                     reviews_body += f"
        • Review {r+1}: {generate_lorem(random.randint(15, 50))}
          • "
+                reviews_body += "
        "
+                # Update breadcrumbs list for this level
+                breadcrumbs_reviews = breadcrumbs_prod + [{"name": "Reviews", "link": reviews_link_path}]
+                # Pass the updated breadcrumbs list
+                create_html_page(subcat_dir / reviews_filename, f"{prod_name} - Reviews", reviews_body, breadcrumbs_prod) # Parent breadcrumb needed here
+
+
+            subcat_body += "
      " # Close product-list ul
+            # Pass the correct breadcrumbs list for the subcategory index page
+            create_html_page(subcat_dir / "index.html", subcat_name, subcat_body, breadcrumbs_cat) # Parent breadcrumb needed here
+
+
+# --- Main Execution ---
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate a dummy multi-level retail website.")
+    parser.add_argument(
+        "-o", "--output-dir",
+        type=str,
+        default="dummy_retail_site",
+        help="Directory to generate the website in."
+    )
+    parser.add_argument(
+        "-n", "--site-name",
+        type=str,
+        default="FakeShop",
+        help="Name of the fake shop."
+    )
+    parser.add_argument(
+        "-b", "--base-path",
+        type=str,
+        default="",
+        help="Base path for hosting the site (e.g., 'samples/deepcrawl'). Leave empty if hosted at the root."
+    )
+    # Optional: Add more args to configure counts if needed
+
+    args = parser.parse_args()
+
+    output_directory = Path(args.output_dir)
+    site_name = args.site_name
+    base_path = args.base_path
+
+    print(f"Generating dummy site '{site_name}' in '{output_directory}'...")
+    # Pass the base_path to the generation function
+    generate_site(output_directory, site_name, base_path)
+    print(f"\nCreated {sum(1 for _ in output_directory.rglob('*.html'))} HTML pages.")
+    print("Dummy site generation complete.")
+    print(f"To serve locally (example): python -m http.server --directory {output_directory} 8000")
+    if base_path:
+        print(f"Access the site at: http://localhost:8000/{base_path.strip('/')}/index.html")
+    else:
+         print(f"Access the site at: http://localhost:8000/index.html")
\ No newline at end of file
diff --git a/tests/general/test_acyn_crawl_wuth_http_crawler_strategy.py b/tests/general/test_acyn_crawl_wuth_http_crawler_strategy.py
new file mode 100644
index 0000000..2727d1e
--- /dev/null
+++ b/tests/general/test_acyn_crawl_wuth_http_crawler_strategy.py
@@ -0,0 +1,56 @@
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    HTTPCrawlerConfig,
+    CacheMode,
+    DefaultMarkdownGenerator,
+    PruningContentFilter
+)
+from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
+from crawl4ai.async_logger import AsyncLogger
+
+async def main():
+    # Initialize HTTP crawler strategy
+    http_strategy = AsyncHTTPCrawlerStrategy(
+        browser_config=HTTPCrawlerConfig(
+            method="GET",
+            verify_ssl=True,
+            follow_redirects=True
+        ),
+        logger=AsyncLogger(verbose=True)
+    )
+
+    # Initialize web crawler with HTTP strategy
+    async with AsyncWebCrawler(crawler_strategy=http_strategy) as crawler:
+        crawler_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48, 
+                    threshold_type="fixed", 
+                    min_word_threshold=0
+                )
+            )
+        )
+        
+        # Test different URLs
+        urls = [
+            "https://example.com",
+            "https://httpbin.org/get",
+            "raw://Test content"
+        ]
+        
+        for url in urls:
+            print(f"\n=== Testing {url} ===")
+            try:
+                result = await crawler.arun(url=url, config=crawler_config)
+                print(f"Status: {result.status_code}")
+                print(f"Raw HTML length: {len(result.html)}")
+                if hasattr(result, 'markdown'):
+                    print(f"Markdown length: {len(result.markdown.raw_markdown)}")
+            except Exception as e:
+                print(f"Error: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/general/test_advanced_deep_crawl.py b/tests/general/test_advanced_deep_crawl.py
new file mode 100644
index 0000000..dd291f6
--- /dev/null
+++ b/tests/general/test_advanced_deep_crawl.py
@@ -0,0 +1,46 @@
+import asyncio
+import time
+
+
+from crawl4ai import CrawlerRunConfig, AsyncWebCrawler, CacheMode
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+from crawl4ai.deep_crawling.filters import FilterChain, URLPatternFilter, DomainFilter, ContentTypeFilter, ContentRelevanceFilter
+from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
+# from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+
+
+async def main():
+    """Example deep crawl of documentation site."""
+    filter_chain = FilterChain([
+        URLPatternFilter(patterns=["*2025*"]),
+        DomainFilter(allowed_domains=["techcrunch.com"]),
+        ContentRelevanceFilter(query="Use of artificial intelligence in Defence applications", threshold=1),
+        ContentTypeFilter(allowed_types=["text/html","application/javascript"])
+    ])
+    config = CrawlerRunConfig(
+        deep_crawl_strategy = BestFirstCrawlingStrategy(
+            max_depth=2,
+            include_external=False,
+            filter_chain=filter_chain,
+            url_scorer=KeywordRelevanceScorer(keywords=["anduril", "defence", "AI"]),
+        ),
+        stream=False,
+        verbose=True,
+        cache_mode=CacheMode.BYPASS,
+        scraping_strategy=LXMLWebScrapingStrategy()
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        print("Starting deep crawl in streaming mode:")
+        config.stream = True
+        start_time = time.perf_counter()
+        async for result in await crawler.arun(
+            url="https://techcrunch.com",
+            config=config
+        ):
+            print(f"→ {result.url} (Depth: {result.metadata.get('depth', 0)})")
+        print(f"Duration: {time.perf_counter() - start_time:.2f} seconds")
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/general/test_async_crawler_strategy.py b/tests/general/test_async_crawler_strategy.py
new file mode 100644
index 0000000..e979325
--- /dev/null
+++ b/tests/general/test_async_crawler_strategy.py
@@ -0,0 +1,382 @@
+import pytest
+import pytest_asyncio
+import asyncio
+from typing import Dict, Any
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+import os
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+from crawl4ai.models import AsyncCrawlResponse
+from crawl4ai.async_logger import AsyncLogger, LogLevel
+
+CRAWL4AI_HOME_DIR = Path(os.path.expanduser("~")).joinpath(".crawl4ai")
+
+if not CRAWL4AI_HOME_DIR.joinpath("profiles", "test_profile").exists():
+    CRAWL4AI_HOME_DIR.joinpath("profiles", "test_profile").mkdir(parents=True)
+
+@pytest.fixture
+def basic_html():
+    return """
+    
+    
+        Basic HTML
+    
+    
+        
      Main Heading
      
+        
      
+            
      
+                
      Basic HTML document for testing purposes.
      
+            
      
+        
      
+    
+    
+    """
+
+# Test Config Files
+@pytest.fixture
+def basic_browser_config():
+    return BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        verbose=True
+    )
+
+@pytest.fixture
+def advanced_browser_config():
+    return BrowserConfig(
+        browser_type="chromium", 
+        headless=True,
+        use_managed_browser=True,
+        user_data_dir=CRAWL4AI_HOME_DIR.joinpath("profiles", "test_profile"),
+        # proxy="http://localhost:8080",
+        viewport_width=1920,
+        viewport_height=1080,
+        user_agent_mode="random"
+    )
+
+@pytest.fixture
+def basic_crawler_config():
+    return CrawlerRunConfig(
+        word_count_threshold=100,
+        wait_until="domcontentloaded",
+        page_timeout=30000
+    )
+
+@pytest.fixture
+def logger():
+    return AsyncLogger(verbose=True, log_level=LogLevel.DEBUG)
+
+@pytest_asyncio.fixture
+async def crawler_strategy(basic_browser_config, logger):
+    strategy = AsyncPlaywrightCrawlerStrategy(browser_config=basic_browser_config, logger=logger)
+    await strategy.start()
+    yield strategy
+    await strategy.close()
+
+# Browser Configuration Tests
+@pytest.mark.asyncio
+async def test_browser_config_initialization():
+    config = BrowserConfig(
+        browser_type="chromium",
+        user_agent_mode="random"
+    )
+    assert config.browser_type == "chromium"
+    assert config.user_agent is not None
+    assert config.headless is True
+
+@pytest.mark.asyncio 
+async def test_persistent_browser_config():
+    config = BrowserConfig(
+        use_persistent_context=True,
+        user_data_dir="/tmp/test_dir"
+    )
+    assert config.use_managed_browser is True
+    assert config.user_data_dir == "/tmp/test_dir"
+
+# Crawler Strategy Tests
+@pytest.mark.asyncio
+async def test_basic_page_load(crawler_strategy):
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        CrawlerRunConfig()
+    )
+    assert response.status_code == 200
+    assert len(response.html) > 0
+    assert "Example Domain" in response.html
+
+@pytest.mark.asyncio
+async def test_screenshot_capture(crawler_strategy):
+    config = CrawlerRunConfig(screenshot=True)
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    assert response.screenshot is not None
+    assert len(response.screenshot) > 0
+
+@pytest.mark.asyncio
+async def test_pdf_generation(crawler_strategy):
+    config = CrawlerRunConfig(pdf=True)
+    response = await crawler_strategy.crawl(
+        "https://example.com", 
+        config
+    )
+    assert response.pdf_data is not None
+    assert len(response.pdf_data) > 0
+
+@pytest.mark.asyncio
+async def test_handle_js_execution(crawler_strategy):
+    config = CrawlerRunConfig(
+        js_code="document.body.style.backgroundColor = 'red';"
+    )
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    assert response.status_code == 200
+    assert 'background-color: red' in response.html.lower()
+
+@pytest.mark.asyncio
+async def test_multiple_js_commands(crawler_strategy):
+    js_commands = [
+        "document.body.style.backgroundColor = 'blue';",
+        "document.title = 'Modified Title';",
+        "const div = document.createElement('div'); div.id = 'test'; div.textContent = 'Test Content'; document.body.appendChild(div);"
+    ]
+    config = CrawlerRunConfig(js_code=js_commands)
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    assert response.status_code == 200
+    assert 'background-color: blue' in response.html.lower()
+    assert 'id="test"' in response.html
+    assert '>Test Content<' in response.html
+    assert 'Modified Title' in response.html
+
+@pytest.mark.asyncio
+async def test_complex_dom_manipulation(crawler_strategy):
+    js_code = """
+    // Create a complex structure
+    const container = document.createElement('div');
+    container.className = 'test-container';
+    
+    const list = document.createElement('ul');
+    list.className = 'test-list';
+    
+    for (let i = 1; i <= 3; i++) {
+        const item = document.createElement('li');
+        item.textContent = `Item ${i}`;
+        item.className = `item-${i}`;
+        list.appendChild(item);
+    }
+    
+    container.appendChild(list);
+    document.body.appendChild(container);
+    """
+    config = CrawlerRunConfig(js_code=js_code)
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    assert response.status_code == 200
+    assert 'class="test-container"' in response.html
+    assert 'class="test-list"' in response.html
+    assert 'class="item-1"' in response.html
+    assert '>Item 1<' in response.html
+    assert '>Item 2<' in response.html
+    assert '>Item 3<' in response.html
+
+@pytest.mark.asyncio
+async def test_style_modifications(crawler_strategy):
+    js_code = """
+    const testDiv = document.createElement('div');
+    testDiv.id = 'style-test';
+    testDiv.style.cssText = 'color: green; font-size: 20px; margin: 10px;';
+    testDiv.textContent = 'Styled Content';
+    document.body.appendChild(testDiv);
+    """
+    config = CrawlerRunConfig(js_code=js_code)
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    assert response.status_code == 200
+    assert 'id="style-test"' in response.html
+    assert 'color: green' in response.html.lower()
+    assert 'font-size: 20px' in response.html.lower()
+    assert 'margin: 10px' in response.html.lower()
+    assert '>Styled Content<' in response.html
+
+@pytest.mark.asyncio
+async def test_dynamic_content_loading(crawler_strategy):
+    js_code = """
+    // Simulate dynamic content loading
+    setTimeout(() => {
+        const dynamic = document.createElement('div');
+        dynamic.id = 'dynamic-content';
+        dynamic.textContent = 'Dynamically Loaded';
+        document.body.appendChild(dynamic);
+    }, 1000);
+    
+    // Add a loading indicator immediately
+    const loading = document.createElement('div');
+    loading.id = 'loading';
+    loading.textContent = 'Loading...';
+    document.body.appendChild(loading);
+    """
+    config = CrawlerRunConfig(js_code=js_code, delay_before_return_html=2.0)
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    assert response.status_code == 200
+    assert 'id="loading"' in response.html
+    assert '>Loading...Dynamically Loaded<' in response.html
+
+# @pytest.mark.asyncio
+# async def test_js_return_values(crawler_strategy):
+#     js_code = """
+#     return {
+#         title: document.title,
+#         metaCount: document.getElementsByTagName('meta').length,
+#         bodyClass: document.body.className
+#     };
+#     """
+#     config = CrawlerRunConfig(js_code=js_code)
+#     response = await crawler_strategy.crawl(
+#         "https://example.com",
+#         config
+#     )
+#     assert response.status_code == 200
+#     assert 'Example Domain' in response.html
+#     assert 'meta name="viewport"' in response.html
+#     assert 'class="main"' in response.html
+
+@pytest.mark.asyncio
+async def test_async_js_execution(crawler_strategy):
+    js_code = """
+    await new Promise(resolve => setTimeout(resolve, 1000));
+    document.body.style.color = 'green';
+    const computedStyle = window.getComputedStyle(document.body);
+    return computedStyle.color;
+    """
+    config = CrawlerRunConfig(js_code=js_code)
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    assert response.status_code == 200
+    assert 'color: green' in response.html.lower()
+
+# @pytest.mark.asyncio
+# async def test_js_error_handling(crawler_strategy):
+#     js_code = """
+#     // Intentionally cause different types of errors
+#     const results = [];
+#     try {
+#         nonExistentFunction();
+#     } catch (e) {
+#         results.push(e.name);
+#     }
+#     try {
+#         JSON.parse('{invalid}');
+#     } catch (e) {
+#         results.push(e.name);
+#     }
+#     return results;
+#     """
+#     config = CrawlerRunConfig(js_code=js_code)
+#     response = await crawler_strategy.crawl(
+#         "https://example.com",
+#         config
+#     )
+#     assert response.status_code == 200
+#     assert 'ReferenceError' in response.html
+#     assert 'SyntaxError' in response.html
+
+@pytest.mark.asyncio
+async def test_handle_navigation_timeout():
+    config = CrawlerRunConfig(page_timeout=1)  # 1ms timeout
+    with pytest.raises(Exception):
+        async with AsyncPlaywrightCrawlerStrategy() as strategy:
+            await strategy.crawl("https://example.com", config)
+
+@pytest.mark.asyncio
+async def test_session_management(crawler_strategy):
+    config = CrawlerRunConfig(session_id="test_session")
+    response1 = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    response2 = await crawler_strategy.crawl(
+        "https://example.com",
+        config
+    )
+    assert response1.status_code == 200
+    assert response2.status_code == 200
+
+@pytest.mark.asyncio
+async def test_process_iframes(crawler_strategy):
+    config = CrawlerRunConfig(
+        process_iframes=True,
+        wait_for_images=True
+    )
+    response = await crawler_strategy.crawl(
+        "https://example.com",
+        config  
+    )
+    assert response.status_code == 200
+
+@pytest.mark.asyncio
+async def test_stealth_mode(crawler_strategy):
+    config = CrawlerRunConfig(
+        simulate_user=True,
+        override_navigator=True
+    )
+    response = await crawler_strategy.crawl(
+        "https://bot.sannysoft.com",
+        config
+    )
+    assert response.status_code == 200
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("prefix", ("raw:", "raw://"))
+async def test_raw_urls(crawler_strategy, basic_html, prefix):
+    url = f"{prefix}{basic_html}"
+    response = await crawler_strategy.crawl(url, CrawlerRunConfig())
+    assert response.html == basic_html
+
+# Error Handling Tests  
+@pytest.mark.asyncio
+async def test_invalid_url():
+    with pytest.raises(ValueError):
+        async with AsyncPlaywrightCrawlerStrategy() as strategy:
+            await strategy.crawl("not_a_url", CrawlerRunConfig())
+
+@pytest.mark.asyncio 
+async def test_network_error_handling():
+    config = CrawlerRunConfig()
+    with pytest.raises(Exception):
+        async with AsyncPlaywrightCrawlerStrategy() as strategy:
+            await strategy.crawl("https://invalid.example.com", config)
+
+@pytest.mark.asyncio
+async def test_remove_overlay_elements(crawler_strategy):
+    config = CrawlerRunConfig(
+        remove_overlay_elements=True,
+        delay_before_return_html=5,
+    )
+    
+    response = await crawler_strategy.crawl(
+        "https://www2.hm.com/en_us/index.html",
+        config
+    )
+    assert response.status_code == 200
+    assert "Accept all cookies" not in response.html
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
\ No newline at end of file
diff --git a/tests/general/test_async_markdown_generator.py b/tests/general/test_async_markdown_generator.py
new file mode 100644
index 0000000..145b98b
--- /dev/null
+++ b/tests/general/test_async_markdown_generator.py
@@ -0,0 +1,171 @@
+import asyncio
+from typing import Dict
+from crawl4ai.content_filter_strategy import BM25ContentFilter, PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+import time
+
+# Test HTML samples
+TEST_HTML_SAMPLES = {
+    "basic": """
+        
+            
      Test Title
      
+            
      This is a test paragraph with a link.
      
+            
      
+                
      Section 1
      
+                
      More content here with bold text.
      
+            
      
+        
+    """,
+    
+    "complex": """
+        
+            
+            
      Header content to remove
      
+            
      
+                
      
+                    
      Main Article
      
+                    
      Important content paragraph with useful link.
      
+                    
      
+                        
      Key Section
      
+                        
      Detailed explanation with multiple sentences. This should be kept 
+                           in the final output. Very important information here.
      
+                    
      
+                
      
+                
+            
      
+            
      Footer content to remove
      
+        
+    """,
+    
+    "edge_cases": """
+        
+            
      
+                
      
+                
         
      
+                
+                
+                
+                
      !!Special>> Characters## Title!!
      
+                
      def test(): pass
      
+            
      
+        
+    """,
+    
+    "links_citations": """
+        
+            
      Document with Links
      
+            
      First link to Example 1
      
+            
      Second link to Test 2
      
+            
      Image link: test image
      
+            
      Repeated link to Example 1 again
      
+        
+    """,
+}
+
+def test_content_filters() -> Dict[str, Dict[str, int]]:
+    """Test various content filtering strategies and return length comparisons."""
+    results = {}
+    
+    # Initialize filters
+    pruning_filter = PruningContentFilter(
+        threshold=0.48,
+        threshold_type="fixed",
+        min_word_threshold=2
+    )
+    
+    bm25_filter = BM25ContentFilter(
+        bm25_threshold=1.0,
+        user_query="test article content important"
+    )
+    
+    # Test each HTML sample
+    for test_name, html in TEST_HTML_SAMPLES.items():
+        # Store results for this test case
+        results[test_name] = {}
+        
+        # Test PruningContentFilter
+        start_time = time.time()
+        pruned_content = pruning_filter.filter_content(html)
+        pruning_time = time.time() - start_time
+        
+        # Test BM25ContentFilter
+        start_time = time.time()
+        bm25_content = bm25_filter.filter_content(html)
+        bm25_time = time.time() - start_time
+        
+        # Store results
+        results[test_name] = {
+            "original_length": len(html),
+            "pruned_length": sum(len(c) for c in pruned_content),
+            "bm25_length": sum(len(c) for c in bm25_content),
+            "pruning_time": pruning_time,
+            "bm25_time": bm25_time
+        }
+        
+    return results
+
+def test_markdown_generation():
+    """Test markdown generation with different configurations."""
+    results = []
+    
+    # Initialize generators with different configurations
+    generators = {
+        "no_filter": DefaultMarkdownGenerator(),
+        "pruning": DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(threshold=0.48)
+        ),
+        "bm25": DefaultMarkdownGenerator(
+            content_filter=BM25ContentFilter(
+                user_query="test article content important"
+            )
+        )
+    }
+    
+    # Test each generator with each HTML sample
+    for test_name, html in TEST_HTML_SAMPLES.items():
+        for gen_name, generator in generators.items():
+            start_time = time.time()
+            result = generator.generate_markdown(
+                html,
+                base_url="http://example.com",
+                citations=True
+            )
+            
+            results.append({
+                "test_case": test_name,
+                "generator": gen_name,
+                "time": time.time() - start_time,
+                "raw_length": len(result.raw_markdown),
+                "fit_length": len(result.fit_markdown) if result.fit_markdown else 0,
+                "citations": len(result.references_markdown)
+            })
+    
+    return results
+
+def main():
+    """Run all tests and print results."""
+    print("Starting content filter tests...")
+    filter_results = test_content_filters()
+    
+    print("\nContent Filter Results:")
+    print("-" * 50)
+    for test_name, metrics in filter_results.items():
+        print(f"\nTest case: {test_name}")
+        print(f"Original length: {metrics['original_length']}")
+        print(f"Pruned length: {metrics['pruned_length']} ({metrics['pruning_time']:.3f}s)")
+        print(f"BM25 length: {metrics['bm25_length']} ({metrics['bm25_time']:.3f}s)")
+        
+    print("\nStarting markdown generation tests...")
+    markdown_results = test_markdown_generation()
+    
+    print("\nMarkdown Generation Results:")
+    print("-" * 50)
+    for result in markdown_results:
+        print(f"\nTest: {result['test_case']} - Generator: {result['generator']}")
+        print(f"Time: {result['time']:.3f}s")
+        print(f"Raw length: {result['raw_length']}")
+        print(f"Fit length: {result['fit_length']}")
+        print(f"Citations: {result['citations']}")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tests/general/test_async_url_seeder_bm25.py b/tests/general/test_async_url_seeder_bm25.py
new file mode 100644
index 0000000..31d6cff
--- /dev/null
+++ b/tests/general/test_async_url_seeder_bm25.py
@@ -0,0 +1,711 @@
+"""
+Comprehensive test cases for AsyncUrlSeeder with BM25 scoring functionality.
+Tests cover all features including query-based scoring, metadata extraction, 
+edge cases, and integration scenarios.
+"""
+
+import asyncio
+import pytest
+from typing import List, Dict, Any
+from crawl4ai import AsyncUrlSeeder, SeedingConfig, AsyncLogger
+import json
+from datetime import datetime
+
+# Test domain - using docs.crawl4ai.com as it has the actual documentation
+TEST_DOMAIN = "kidocode.com"
+TEST_DOMAIN = "docs.crawl4ai.com"
+TEST_DOMAIN = "www.bbc.com/sport"
+
+
+class TestAsyncUrlSeederBM25:
+    """Comprehensive test suite for AsyncUrlSeeder with BM25 scoring."""
+    
+    async def create_seeder(self):
+        """Create an AsyncUrlSeeder instance for testing."""
+        logger = AsyncLogger()
+        return AsyncUrlSeeder(logger=logger)
+
+    # ============================================
+    # Basic BM25 Scoring Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_basic_bm25_scoring(self, seeder):
+        """Test basic BM25 scoring with a simple query."""
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="premier league highlights",
+            scoring_method="bm25",
+            max_urls=200,
+            verbose=True,
+            force=True  # Force fresh fetch
+        )
+        
+        results = await seeder.urls(TEST_DOMAIN, config)
+        
+        # Verify results have relevance scores
+        assert all("relevance_score" in r for r in results)
+        
+        # Verify scores are normalized between 0 and 1
+        scores = [r["relevance_score"] for r in results]
+        assert all(0.0 <= s <= 1.0 for s in scores)
+        
+        # Verify results are sorted by relevance (descending)
+        assert scores == sorted(scores, reverse=True)
+        
+        # Print top 5 results for manual verification
+        print("\nTop 5 results for 'web crawling tutorial':")
+        for i, r in enumerate(results[:5]):
+            print(f"{i+1}. Score: {r['relevance_score']:.3f} - {r['url']}")
+    
+    @pytest.mark.asyncio
+    async def test_query_variations(self, seeder):
+        """Test BM25 scoring with different query variations."""
+        queries = [
+            "VAR controversy",
+            "player ratings",
+            "live score update",
+            "transfer rumours",
+            "post match analysis",
+            "injury news"
+        ]
+        
+        for query in queries:
+            config = SeedingConfig(
+                source="sitemap",
+                extract_head=True,
+                query=query,
+                scoring_method="bm25",
+                max_urls=100,
+                # force=True 
+            )
+            
+            results = await seeder.urls(TEST_DOMAIN, config)
+            
+            # Verify each query produces scored results
+            assert len(results) > 0
+            assert all("relevance_score" in r for r in results)
+            
+            print(f"\nTop result for '{query}':")
+            if results:
+                top = results[0]
+                print(f"  Score: {top['relevance_score']:.3f} - {top['url']}")
+    
+    # ============================================
+    # Score Threshold Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_score_threshold_filtering(self, seeder):
+        """Test filtering results by minimum relevance score."""
+        thresholds = [0.1, 0.3, 0.5, 0.7]
+        
+        for threshold in thresholds:
+            config = SeedingConfig(
+                source="sitemap",
+                extract_head=True,
+                query="league standings",
+                score_threshold=threshold,
+                scoring_method="bm25",
+                max_urls=50
+            )
+            
+            results = await seeder.urls(TEST_DOMAIN, config)
+            
+            # Verify all results meet threshold
+            if results:
+                assert all(r["relevance_score"] >= threshold for r in results)
+            
+            print(f"\nThreshold {threshold}: {len(results)} URLs passed")
+    
+    @pytest.mark.asyncio
+    async def test_extreme_thresholds(self, seeder):
+        """Test edge cases with extreme threshold values."""
+        # Very low threshold - should return many results
+        config_low = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="match",
+            score_threshold=0.001,
+            scoring_method="bm25"
+        )
+        results_low = await seeder.urls(TEST_DOMAIN, config_low)
+        
+        # Very high threshold - might return few or no results
+        config_high = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="match",
+            score_threshold=0.99,
+            scoring_method="bm25"
+        )
+        results_high = await seeder.urls(TEST_DOMAIN, config_high)
+        
+        # Low threshold should return more results than high
+        assert len(results_low) >= len(results_high)
+        print(f"\nLow threshold (0.001): {len(results_low)} results")
+        print(f"High threshold (0.99): {len(results_high)} results")
+    
+    # ============================================
+    # Metadata Extraction Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_comprehensive_metadata_extraction(self, seeder):
+        """Test extraction of all metadata types including JSON-LD."""
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="match report",
+            scoring_method="bm25",
+            max_urls=5,
+            verbose=True
+        )
+        
+        results = await seeder.urls(TEST_DOMAIN, config)
+        
+        for result in results:
+            head_data = result.get("head_data", {})
+            
+            # Check for various metadata fields
+            print(f"\nMetadata for {result['url']}:")
+            print(f"  Title: {head_data.get('title', 'N/A')}")
+            print(f"  Charset: {head_data.get('charset', 'N/A')}")
+            print(f"  Lang: {head_data.get('lang', 'N/A')}")
+            
+            # Check meta tags
+            meta = head_data.get("meta", {})
+            if meta:
+                print("  Meta tags found:")
+                for key in ["description", "keywords", "author", "viewport"]:
+                    if key in meta:
+                        print(f"    {key}: {meta[key][:50]}...")
+            
+            # Check for Open Graph tags
+            og_tags = {k: v for k, v in meta.items() if k.startswith("og:")}
+            if og_tags:
+                print("  Open Graph tags found:")
+                for k, v in list(og_tags.items())[:3]:
+                    print(f"    {k}: {v[:50]}...")
+            
+            # Check JSON-LD
+            if head_data.get("jsonld"):
+                print(f"  JSON-LD schemas found: {len(head_data['jsonld'])}")
+    
+    @pytest.mark.asyncio
+    async def test_jsonld_extraction_scoring(self, seeder):
+        """Test that JSON-LD data contributes to BM25 scoring."""
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="Premier League match report highlights",
+            scoring_method="bm25",
+            max_urls=20
+        )
+        
+        results = await seeder.urls(TEST_DOMAIN, config)
+        
+        # Find results with JSON-LD data
+        jsonld_results = [r for r in results if r.get("head_data", {}).get("jsonld")]
+        
+        if jsonld_results:
+            print(f"\nFound {len(jsonld_results)} URLs with JSON-LD data")
+            for r in jsonld_results[:3]:
+                print(f"  Score: {r['relevance_score']:.3f} - {r['url']}")
+                jsonld_data = r["head_data"]["jsonld"]
+                print(f"    JSON-LD types: {[item.get('@type', 'Unknown') for item in jsonld_data if isinstance(item, dict)]}")
+    
+    # ============================================
+    # Edge Cases and Error Handling
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_empty_query(self, seeder):
+        """Test behavior with empty query string."""
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="",
+            scoring_method="bm25",
+            max_urls=10
+        )
+        
+        results = await seeder.urls(TEST_DOMAIN, config)
+        
+        # Should return results but all with zero scores
+        assert len(results) > 0
+        assert all(r.get("relevance_score", 0) == 0 for r in results)
+    
+    @pytest.mark.asyncio
+    async def test_query_without_extract_head(self, seeder):
+        """Test query scoring when extract_head is False."""
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=False,  # This should trigger a warning
+            query="Premier League match report highlights",
+            scoring_method="bm25",
+            max_urls=10
+        )
+        
+        results = await seeder.urls(TEST_DOMAIN, config)
+        
+        # Results should not have relevance scores
+        assert all("relevance_score" not in r for r in results)
+        print("\nVerified: No scores added when extract_head=False")
+    
+    @pytest.mark.asyncio
+    async def test_special_characters_in_query(self, seeder):
+        """Test queries with special characters and symbols."""
+        special_queries = [
+            "premier league + analytics",
+            "injury/rehab routines",
+            "AI-powered scouting",
+            "match stats & xG",
+            "tactical@breakdown",
+            "transfer-window.yml"
+        ]
+        
+        for query in special_queries:
+            config = SeedingConfig(
+                source="sitemap",
+                extract_head=True,
+                query=query,
+                scoring_method="bm25",
+                max_urls=5
+            )
+            
+            try:
+                results = await seeder.urls(TEST_DOMAIN, config)
+                assert isinstance(results, list)
+                print(f"\n✓ Query '{query}' processed successfully")
+            except Exception as e:
+                pytest.fail(f"Failed on query '{query}': {str(e)}")
+    
+    @pytest.mark.asyncio
+    async def test_unicode_query(self, seeder):
+        """Test queries with Unicode characters."""
+        unicode_queries = [
+            "网页爬虫",  # Chinese
+            "веб-краулер",  # Russian
+            "🚀 crawl4ai",  # Emoji
+            "naïve implementation",  # Accented characters
+        ]
+        
+        for query in unicode_queries:
+            config = SeedingConfig(
+                source="sitemap",
+                extract_head=True,
+                query=query,
+                scoring_method="bm25",
+                max_urls=5
+            )
+            
+            try:
+                results = await seeder.urls(TEST_DOMAIN, config)
+                assert isinstance(results, list)
+                print(f"\n✓ Unicode query '{query}' processed successfully")
+            except Exception as e:
+                print(f"\n✗ Unicode query '{query}' failed: {str(e)}")
+    
+    # ============================================
+    # Performance and Scalability Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_large_scale_scoring(self, seeder):
+        """Test BM25 scoring with many URLs."""
+        config = SeedingConfig(
+            source="cc+sitemap",  # Use both sources for more URLs
+            extract_head=True,
+            query="world cup group standings",
+            scoring_method="bm25",
+            max_urls=100,
+            concurrency=20,
+            hits_per_sec=10
+        )
+        
+        start_time = asyncio.get_event_loop().time()
+        results = await seeder.urls(TEST_DOMAIN, config)
+        elapsed = asyncio.get_event_loop().time() - start_time
+        
+        print(f"\nProcessed {len(results)} URLs in {elapsed:.2f} seconds")
+        print(f"Average time per URL: {elapsed/len(results)*1000:.1f}ms")
+        
+        # Verify scoring worked at scale
+        assert all("relevance_score" in r for r in results)
+        
+        # Check score distribution
+        scores = [r["relevance_score"] for r in results]
+        print(f"Score distribution:")
+        print(f"  Min: {min(scores):.3f}")
+        print(f"  Max: {max(scores):.3f}")
+        print(f"  Avg: {sum(scores)/len(scores):.3f}")
+    
+    @pytest.mark.asyncio
+    async def test_concurrent_scoring_consistency(self, seeder):
+        """Test that concurrent requests produce consistent scores."""
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="live score update",
+            scoring_method="bm25",
+            max_urls=20,
+            concurrency=10
+        )
+        
+        # Run the same query multiple times
+        results_list = []
+        for _ in range(3):
+            results = await seeder.urls(TEST_DOMAIN, config)
+            results_list.append(results)
+        
+        # Compare scores across runs (they should be identical for same URLs)
+        url_scores = {}
+        for results in results_list:
+            for r in results:
+                url = r["url"]
+                score = r["relevance_score"]
+                if url in url_scores:
+                    # Scores should be very close (allowing for tiny float differences)
+                    assert abs(url_scores[url] - score) < 0.001
+                else:
+                    url_scores[url] = score
+        
+        print(f"\n✓ Consistent scores across {len(results_list)} runs")
+    
+    # ============================================
+    # Multi-Domain Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_many_urls_with_scoring(self, seeder):
+        """Test many_urls method with BM25 scoring."""
+        domains = [TEST_DOMAIN, "docs.crawl4ai.com", "example.com"]
+        
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            # live_check=True,
+            query="fixture list",
+            scoring_method="bm25",
+            score_threshold=0.2,
+            max_urls=10,
+            force=True,  # Force fresh fetch
+        )
+        
+        results_dict = await seeder.many_urls(domains, config)
+        
+        for domain, results in results_dict.items():
+            print(f"\nDomain: {domain}")
+            print(f"  Found {len(results)} URLs above threshold")
+            if results:
+                top = results[0]
+                print(f"  Top result: {top['relevance_score']:.3f} - {top['url']}")
+    
+    # ============================================
+    # Complex Query Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_multi_word_complex_queries(self, seeder):
+        """Test complex multi-word queries."""
+        complex_queries = [
+            "how to follow live match commentary",
+            "extract expected goals stats from match data",
+            "premier league match report analysis",
+            "transfer rumours and confirmed signings tracker",
+            "tactical breakdown of high press strategy"
+        ]
+        
+        for query in complex_queries:
+            config = SeedingConfig(
+                source="sitemap",
+                extract_head=True,
+                query=query,
+                scoring_method="bm25",
+                max_urls=5
+            )
+            
+            results = await seeder.urls(TEST_DOMAIN, config)
+            
+            if results:
+                print(f"\nQuery: '{query}'")
+                print(f"Top match: {results[0]['relevance_score']:.3f} - {results[0]['url']}")
+                
+                # Extract matched terms from metadata
+                head_data = results[0].get("head_data", {})
+                title = head_data.get("title", "")
+                description = head_data.get("meta", {}).get("description", "")
+                
+                # Simple term matching for verification
+                query_terms = set(query.lower().split())
+                title_terms = set(title.lower().split())
+                desc_terms = set(description.lower().split())
+                
+                matched_terms = query_terms & (title_terms | desc_terms)
+                if matched_terms:
+                    print(f"Matched terms: {', '.join(matched_terms)}")
+    
+    # ============================================
+    # Cache and Force Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_scoring_with_cache(self, seeder):
+        """Test that scoring works correctly with cached results."""
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="injury update timeline",
+            scoring_method="bm25",
+            max_urls=10,
+            force=False  # Use cache
+        )
+        
+        # First run - populate cache
+        results1 = await seeder.urls(TEST_DOMAIN, config)
+        
+        # Second run - should use cache
+        results2 = await seeder.urls(TEST_DOMAIN, config)
+        
+        # Results should be identical
+        assert len(results1) == len(results2)
+        for r1, r2 in zip(results1, results2):
+            assert r1["url"] == r2["url"]
+            assert abs(r1["relevance_score"] - r2["relevance_score"]) < 0.001
+        
+        print("\n✓ Cache produces consistent scores")
+    
+    @pytest.mark.asyncio
+    async def test_force_refresh_scoring(self, seeder):
+        """Test force=True bypasses cache for fresh scoring."""
+        config_cached = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="transfer window",
+            scoring_method="bm25",
+            max_urls=5,
+            force=False
+        )
+        
+        config_forced = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="transfer window",
+            scoring_method="bm25",
+            max_urls=5,
+            force=True
+        )
+        
+        # Run with cache
+        start1 = asyncio.get_event_loop().time()
+        results1 = await seeder.urls(TEST_DOMAIN, config_cached)
+        time1 = asyncio.get_event_loop().time() - start1
+        
+        # Run with force (should be slower due to fresh fetch)
+        start2 = asyncio.get_event_loop().time()
+        results2 = await seeder.urls(TEST_DOMAIN, config_forced)
+        time2 = asyncio.get_event_loop().time() - start2
+        
+        print(f"\nCached run: {time1:.2f}s")
+        print(f"Forced run: {time2:.2f}s")
+        
+        # Both should produce scored results
+        assert all("relevance_score" in r for r in results1)
+        assert all("relevance_score" in r for r in results2)
+    
+    # ============================================
+    # Source Combination Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_scoring_with_multiple_sources(self, seeder):
+        """Test BM25 scoring with combined sources (cc+sitemap)."""
+        config = SeedingConfig(
+            source="cc+sitemap",
+            extract_head=True,
+            query="match highlights video",
+            scoring_method="bm25",
+            score_threshold=0.3,
+            max_urls=30,
+            concurrency=15
+        )
+        
+        results = await seeder.urls(TEST_DOMAIN, config)
+        
+        # Verify we got results from both sources
+        print(f"\nCombined sources returned {len(results)} URLs above threshold")
+        
+        # Check URL diversity
+        unique_paths = set()
+        for r in results:
+            path = r["url"].replace("https://", "").replace("http://", "").split("/", 1)[-1]
+            unique_paths.add(path.split("?")[0])  # Remove query params
+        
+        print(f"Unique paths found: {len(unique_paths)}")
+        
+        # All should be scored and above threshold
+        assert all(r["relevance_score"] >= 0.3 for r in results)
+    
+    # ============================================
+    # Integration Tests
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_full_workflow_integration(self, seeder):
+        """Test complete workflow: discover -> score -> filter -> use."""
+        # Step 1: Discover and score URLs
+        config = SeedingConfig(
+            source="sitemap",
+            extract_head=True,
+            query="premier league opening fixtures",
+            scoring_method="bm25",
+            score_threshold=0.4,
+            max_urls=10,
+            verbose=True
+        )
+        
+        results = await seeder.urls(TEST_DOMAIN, config)
+        
+        print(f"\nStep 1: Found {len(results)} relevant URLs")
+        
+        # Step 2: Analyze top results
+        if results:
+            top_urls = results[:3]
+            print("\nStep 2: Top 3 URLs for crawling:")
+            for i, r in enumerate(top_urls):
+                print(f"{i+1}. Score: {r['relevance_score']:.3f}")
+                print(f"   URL: {r['url']}")
+                print(f"   Title: {r['head_data'].get('title', 'N/A')}")
+                
+                # Check metadata quality
+                meta = r['head_data'].get('meta', {})
+                if 'description' in meta:
+                    print(f"   Description: {meta['description'][:80]}...")
+        
+        # Step 3: Verify these URLs would be good for actual crawling
+        assert all(r["status"] == "valid" for r in results[:3])
+        print("\nStep 3: All top URLs are valid for crawling ✓")
+    
+    # ============================================
+    # Report Generation
+    # ============================================
+    
+    @pytest.mark.asyncio
+    async def test_generate_scoring_report(self, seeder):
+        """Generate a comprehensive report of BM25 scoring effectiveness."""
+        queries = {
+            "beginner": "match schedule",
+            "advanced": "tactical analysis pressing",
+            "api": "VAR decision explanation",
+            "deployment": "fixture changes due to weather",
+            "extraction": "expected goals statistics"
+        }
+        
+        report = {
+            "timestamp": datetime.now().isoformat(),
+            "domain": TEST_DOMAIN,
+            "results": {}
+        }
+        
+        for category, query in queries.items():
+            config = SeedingConfig(
+                source="sitemap",
+                extract_head=True,
+                query=query,
+                scoring_method="bm25",
+                max_urls=10
+            )
+            
+            results = await seeder.urls(TEST_DOMAIN, config)
+            
+            report["results"][category] = {
+                "query": query,
+                "total_results": len(results),
+                "top_results": [
+                    {
+                        "url": r["url"],
+                        "score": r["relevance_score"],
+                        "title": r["head_data"].get("title", "")
+                    }
+                    for r in results[:3]
+                ],
+                "score_distribution": {
+                    "min": min(r["relevance_score"] for r in results) if results else 0,
+                    "max": max(r["relevance_score"] for r in results) if results else 0,
+                    "avg": sum(r["relevance_score"] for r in results) / len(results) if results else 0
+                }
+            }
+        
+        # Print report
+        print("\n" + "="*60)
+        print("BM25 SCORING EFFECTIVENESS REPORT")
+        print("="*60)
+        print(f"Domain: {report['domain']}")
+        print(f"Timestamp: {report['timestamp']}")
+        print("\nResults by Category:")
+        
+        for category, data in report["results"].items():
+            print(f"\n{category.upper()}: '{data['query']}'")
+            print(f"  Total results: {data['total_results']}")
+            print(f"  Score range: {data['score_distribution']['min']:.3f} - {data['score_distribution']['max']:.3f}")
+            print(f"  Average score: {data['score_distribution']['avg']:.3f}")
+            print("  Top matches:")
+            for i, result in enumerate(data['top_results']):
+                print(f"    {i+1}. [{result['score']:.3f}] {result['title']}")
+
+
+# ============================================
+# Standalone test runner
+# ============================================
+
+async def run_all_tests():
+    """Run all tests standalone (without pytest)."""
+    print("Running AsyncUrlSeeder BM25 Tests...")
+    print("="*60)
+    
+    test_instance = TestAsyncUrlSeederBM25()
+    seeder = await test_instance.create_seeder()
+    
+    # Run each test method
+    test_methods = [
+        # test_instance.test_basic_bm25_scoring,
+        # test_instance.test_query_variations,
+        # test_instance.test_score_threshold_filtering,
+        # test_instance.test_extreme_thresholds,
+        # test_instance.test_comprehensive_metadata_extraction,
+        # test_instance.test_jsonld_extraction_scoring,
+        # test_instance.test_empty_query,
+        # test_instance.test_query_without_extract_head,
+        # test_instance.test_special_characters_in_query,
+        # test_instance.test_unicode_query,
+        # test_instance.test_large_scale_scoring,
+        # test_instance.test_concurrent_scoring_consistency,
+        # test_instance.test_many_urls_with_scoring,
+        test_instance.test_multi_word_complex_queries,
+        test_instance.test_scoring_with_cache,
+        test_instance.test_force_refresh_scoring,
+        test_instance.test_scoring_with_multiple_sources,
+        test_instance.test_full_workflow_integration,
+        test_instance.test_generate_scoring_report
+    ]
+    
+    for test_method in test_methods:
+        try:
+            print(f"\nRunning {test_method.__name__}...")
+            await test_method(seeder)
+            print(f"✓ {test_method.__name__} passed")
+        except Exception as e:
+            import traceback
+            print(f"✗ {test_method.__name__} failed: {str(e)}")
+            print(f"  Error type: {type(e).__name__}")
+            traceback.print_exc()
+    
+    print("\n" + "="*60)
+    print("Test suite completed!")
+
+
+if __name__ == "__main__":
+    # Run tests directly
+    asyncio.run(run_all_tests())
\ No newline at end of file
diff --git a/tests/general/test_async_webcrawler.py b/tests/general/test_async_webcrawler.py
new file mode 100644
index 0000000..80d4acb
--- /dev/null
+++ b/tests/general/test_async_webcrawler.py
@@ -0,0 +1,228 @@
+import asyncio
+import pytest
+from typing import List
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig, 
+    CrawlerRunConfig,
+    MemoryAdaptiveDispatcher,
+    RateLimiter,
+    CacheMode
+)
+from crawl4ai.extraction_strategy import ExtractionStrategy
+
+class MockExtractionStrategy(ExtractionStrategy):
+    """Mock extraction strategy for testing URL parameter handling"""
+
+    def __init__(self):
+        super().__init__()
+        self.run_calls = []
+
+    def extract(self, url: str, html: str, *args, **kwargs):
+        return [{"test": "data"}]
+
+    def run(self, url: str, sections: List[str], *args, **kwargs):
+        self.run_calls.append(url)
+        return super().run(url, sections, *args, **kwargs)
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("viewport", [
+    (800, 600),
+    (1024, 768),
+    (1920, 1080)
+])
+async def test_viewport_config(viewport):
+    """Test different viewport configurations"""
+    width, height = viewport
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        viewport_width=width,
+        viewport_height=height
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=CrawlerRunConfig(
+                # cache_mode=CacheMode.BYPASS,
+                page_timeout=30000  # 30 seconds
+            )
+        )
+        assert result.success
+
+@pytest.mark.asyncio
+async def test_memory_management():
+    """Test memory-adaptive dispatching"""
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        viewport_width=1024,
+        viewport_height=768
+    )
+    
+    dispatcher = MemoryAdaptiveDispatcher(
+        memory_threshold_percent=70.0,
+        check_interval=1.0,
+        max_session_permit=5
+    )
+    
+    urls = ["https://example.com"] * 3  # Test with multiple identical URLs
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        results = await crawler.arun_many(
+            urls=urls,
+            config=CrawlerRunConfig(page_timeout=30000),
+            dispatcher=dispatcher
+        )
+        assert len(results) == len(urls)
+
+@pytest.mark.asyncio
+async def test_rate_limiting():
+    """Test rate limiting functionality"""
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True
+    )
+    
+    dispatcher = MemoryAdaptiveDispatcher(
+        rate_limiter=RateLimiter(
+            base_delay=(1.0, 2.0),
+            max_delay=5.0,
+            max_retries=2
+        ),
+        memory_threshold_percent=70.0
+    )
+    
+    urls = [
+        "https://example.com",
+        "https://example.org",
+        "https://example.net"
+    ]
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        results = await crawler.arun_many(
+            urls=urls,
+            config=CrawlerRunConfig(page_timeout=30000),
+            dispatcher=dispatcher
+        )
+        assert len(results) == len(urls)
+
+@pytest.mark.asyncio
+async def test_javascript_execution():
+    """Test JavaScript execution capabilities"""
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        java_script_enabled=True
+    )
+    
+    js_code = """
+        document.body.style.backgroundColor = 'red';
+        return document.body.style.backgroundColor;
+    """
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=CrawlerRunConfig(
+                js_code=js_code,
+                page_timeout=30000
+            )
+        )
+        assert result.success
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error_url", [
+    "https://invalid.domain.test",
+    "https://httpbin.org/status/404",
+    "https://httpbin.org/status/503",
+    "https://httpbin.org/status/403"
+])
+async def test_error_handling(error_url):
+    """Test error handling for various failure scenarios"""
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=error_url,
+            config=CrawlerRunConfig(
+                page_timeout=10000,  # Short timeout for error cases
+                cache_mode=CacheMode.BYPASS
+            )
+        )
+        assert not result.success
+        assert result.error_message is not None
+
+@pytest.mark.asyncio
+async def test_extraction_strategy_run_with_regular_url():
+    """
+    Regression test for extraction_strategy.run URL parameter handling with regular URLs.
+
+    This test verifies that when is_raw_html=False (regular URL),
+    extraction_strategy.run is called with the actual URL.
+    """
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        mock_strategy = MockExtractionStrategy()
+
+        # Test regular URL (is_raw_html=False)
+        regular_url = "https://example.com"
+        result = await crawler.arun(
+            url=regular_url,
+            config=CrawlerRunConfig(
+                page_timeout=30000,
+                extraction_strategy=mock_strategy,
+                cache_mode=CacheMode.BYPASS
+            )
+        )
+
+        assert result.success
+        assert len(mock_strategy.run_calls) == 1
+        assert mock_strategy.run_calls[0] == regular_url, f"Expected '{regular_url}', got '{mock_strategy.run_calls[0]}'"
+
+@pytest.mark.asyncio
+async def test_extraction_strategy_run_with_raw_html():
+    """
+    Regression test for extraction_strategy.run URL parameter handling with raw HTML.
+
+    This test verifies that when is_raw_html=True (URL starts with "raw:"),
+    extraction_strategy.run is called with "Raw HTML" instead of the actual URL.
+    """
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        mock_strategy = MockExtractionStrategy()
+
+        # Test raw HTML URL (is_raw_html=True automatically set)
+        raw_html_url = "raw:
      Test HTML
      This is a test.
      "
+        result = await crawler.arun(
+            url=raw_html_url,
+            config=CrawlerRunConfig(
+                page_timeout=30000,
+                extraction_strategy=mock_strategy,
+                cache_mode=CacheMode.BYPASS
+            )
+        )
+
+        assert result.success
+        assert len(mock_strategy.run_calls) == 1
+        assert mock_strategy.run_calls[0] == "Raw HTML", f"Expected 'Raw HTML', got '{mock_strategy.run_calls[0]}'"
+
+if __name__ == "__main__":
+    asyncio.run(test_viewport_config((1024, 768)))
+    asyncio.run(test_memory_management())
+    asyncio.run(test_rate_limiting())
+    asyncio.run(test_javascript_execution())
+    asyncio.run(test_extraction_strategy_run_with_regular_url())
+    asyncio.run(test_extraction_strategy_run_with_raw_html())
diff --git a/tests/general/test_bff_scoring.py b/tests/general/test_bff_scoring.py
new file mode 100644
index 0000000..d663d94
--- /dev/null
+++ b/tests/general/test_bff_scoring.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+Simple test to verify BestFirstCrawlingStrategy fixes.
+This test crawls a real website and shows that:
+1. Higher-scoring pages are crawled first (priority queue fix)
+2. Links are scored before truncation (link discovery fix)
+"""
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.deep_crawling import BestFirstCrawlingStrategy
+from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
+
+async def test_best_first_strategy():
+    """Test BestFirstCrawlingStrategy with keyword scoring"""
+    
+    print("=" * 70)
+    print("Testing BestFirstCrawlingStrategy with Real URL")
+    print("=" * 70)
+    print("\nThis test will:")
+    print("1. Crawl Python.org documentation")
+    print("2. Score pages based on keywords: 'tutorial', 'guide', 'reference'")
+    print("3. Show that higher-scoring pages are crawled first")
+    print("-" * 70)
+    
+    # Create a keyword scorer that prioritizes tutorial/guide pages
+    scorer = KeywordRelevanceScorer(
+        keywords=["tutorial", "guide", "reference", "documentation"],
+        weight=1.0,
+        case_sensitive=False
+    )
+    
+    # Create the strategy with scoring
+    strategy = BestFirstCrawlingStrategy(
+        max_depth=2,          # Crawl 2 levels deep
+        max_pages=10,         # Limit to 10 pages total
+        url_scorer=scorer,    # Use keyword scoring
+        include_external=False  # Only internal links
+    )
+    
+    # Configure browser and crawler
+    browser_config = BrowserConfig(
+        headless=True,    # Run in background
+        verbose=False     # Reduce output noise
+    )
+    
+    crawler_config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False
+    )
+    
+    print("\nStarting crawl of https://docs.python.org/3/")
+    print("Looking for pages with keywords: tutorial, guide, reference, documentation")
+    print("-" * 70)
+    
+    crawled_urls = []
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Crawl and collect results
+        results = await crawler.arun(
+            url="https://docs.python.org/3/",
+            config=crawler_config
+        )
+        
+        # Process results
+        if isinstance(results, list):
+            for result in results:
+                score = result.metadata.get('score', 0) if result.metadata else 0
+                depth = result.metadata.get('depth', 0) if result.metadata else 0
+                crawled_urls.append({
+                    'url': result.url,
+                    'score': score,
+                    'depth': depth,
+                    'success': result.success
+                })
+    
+    print("\n" + "=" * 70)
+    print("CRAWL RESULTS (in order of crawling)")
+    print("=" * 70)
+    
+    for i, item in enumerate(crawled_urls, 1):
+        status = "✓" if item['success'] else "✗"
+        # Highlight high-scoring pages
+        if item['score'] > 0.5:
+            print(f"{i:2}. [{status}] Score: {item['score']:.2f} | Depth: {item['depth']} | {item['url']}")
+            print(f"     ^ HIGH SCORE - Contains keywords!")
+        else:
+            print(f"{i:2}. [{status}] Score: {item['score']:.2f} | Depth: {item['depth']} | {item['url']}")
+    
+    print("\n" + "=" * 70)
+    print("ANALYSIS")
+    print("=" * 70)
+    
+    # Check if higher scores appear early in the crawl
+    scores = [item['score'] for item in crawled_urls[1:]]  # Skip initial URL
+    high_score_indices = [i for i, s in enumerate(scores) if s > 0.3]
+    
+    if high_score_indices and high_score_indices[0] < len(scores) / 2:
+        print("✅ SUCCESS: Higher-scoring pages (with keywords) were crawled early!")
+        print("   This confirms the priority queue fix is working.")
+    else:
+        print("⚠️  Check the crawl order above - higher scores should appear early")
+    
+    # Show score distribution
+    print(f"\nScore Statistics:")
+    print(f"  - Total pages crawled: {len(crawled_urls)}")
+    print(f"  - Average score: {sum(item['score'] for item in crawled_urls) / len(crawled_urls):.2f}")
+    print(f"  - Max score: {max(item['score'] for item in crawled_urls):.2f}")
+    print(f"  - Pages with keywords: {sum(1 for item in crawled_urls if item['score'] > 0.3)}")
+    
+    print("\n" + "=" * 70)
+    print("TEST COMPLETE")
+    print("=" * 70)
+
+if __name__ == "__main__":
+    print("\n🔍 BestFirstCrawlingStrategy Simple Test\n")
+    asyncio.run(test_best_first_strategy())
\ No newline at end of file
diff --git a/tests/general/test_cache_context.py b/tests/general/test_cache_context.py
new file mode 100644
index 0000000..0f42f9f
--- /dev/null
+++ b/tests/general/test_cache_context.py
@@ -0,0 +1,85 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from playwright.async_api import Page, BrowserContext
+
+async def test_reuse_context_by_config():
+    # We will store each context ID in these maps to confirm reuse
+    context_ids_for_A = []
+    context_ids_for_B = []
+
+    # Create a small hook to track context creation
+    async def on_page_context_created(page: Page, context: BrowserContext, config: CrawlerRunConfig, **kwargs):
+        c_id = id(context)
+        print(f"[HOOK] on_page_context_created - Context ID: {c_id}")
+        # Distinguish which config we used by checking a custom hook param
+        config_label = config.shared_data.get("config_label", "unknown")
+        if config_label == "A":
+            context_ids_for_A.append(c_id)
+        elif config_label == "B":
+            context_ids_for_B.append(c_id)
+        return page
+
+    # Browser config - Headless, verbose so we see logs
+    browser_config = BrowserConfig(headless=True, verbose=True)
+
+    # Two crawler run configs that differ (for example, text_mode):
+    configA = CrawlerRunConfig(
+        only_text=True,
+        cache_mode=CacheMode.BYPASS,
+        wait_until="domcontentloaded",
+        shared_data = {
+            "config_label" : "A"
+        }
+    )
+    configB = CrawlerRunConfig(
+        only_text=False,
+        cache_mode=CacheMode.BYPASS,
+        wait_until="domcontentloaded",
+        shared_data = {
+            "config_label" : "B"
+        }
+    )
+
+    # Create the crawler
+    crawler = AsyncWebCrawler(config=browser_config)
+
+    # Attach our custom hook
+    # Note: "on_page_context_created" will be called each time a new context+page is generated
+    crawler.crawler_strategy.set_hook("on_page_context_created", on_page_context_created)
+
+    # Start the crawler (launches the browser)
+    await crawler.start()
+
+    # For demonstration, we’ll crawl a benign site multiple times with each config
+    test_url = "https://example.com"
+    print("\n--- Crawling with config A (text_mode=True) ---")
+    for _ in range(2):
+        # Pass an extra kwarg to the hook so we know which config is being used
+        await crawler.arun(test_url, config=configA)
+
+    print("\n--- Crawling with config B (text_mode=False) ---")
+    for _ in range(2):
+        await crawler.arun(test_url, config=configB)
+
+    # Close the crawler (shuts down the browser, closes contexts)
+    await crawler.close()
+
+    # Validate and show the results
+    print("\n=== RESULTS ===")
+    print(f"Config A context IDs: {context_ids_for_A}")
+    print(f"Config B context IDs: {context_ids_for_B}")
+    if len(set(context_ids_for_A)) == 1:
+        print("✅ All config A crawls used the SAME BrowserContext.")
+    else:
+        print("❌ Config A crawls created multiple contexts unexpectedly.")
+    if len(set(context_ids_for_B)) == 1:
+        print("✅ All config B crawls used the SAME BrowserContext.")
+    else:
+        print("❌ Config B crawls created multiple contexts unexpectedly.")
+    if set(context_ids_for_A).isdisjoint(context_ids_for_B):
+        print("✅ Config A context is different from Config B context.")
+    else:
+        print("❌ A and B ended up sharing the same context somehow!")
+
+if __name__ == "__main__":
+    asyncio.run(test_reuse_context_by_config())
diff --git a/tests/general/test_content_source_parameter.py b/tests/general/test_content_source_parameter.py
new file mode 100644
index 0000000..e686eaf
--- /dev/null
+++ b/tests/general/test_content_source_parameter.py
@@ -0,0 +1,106 @@
+"""
+Tests for the content_source parameter in markdown generation.
+"""
+import unittest
+import asyncio
+from unittest.mock import patch, MagicMock
+
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator, MarkdownGenerationStrategy
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai.models import MarkdownGenerationResult
+
+HTML_SAMPLE = """
+
+Test Page
+
+    
      Test Content
      
+    
      This is a test paragraph.
      
+    
      
+        
      This is content within a container.
      
+    
      
+
+
+"""
+
+
+class TestContentSourceParameter(unittest.TestCase):
+    """Test cases for the content_source parameter in markdown generation."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self.loop)
+
+    def tearDown(self):
+        """Tear down test fixtures."""
+        self.loop.close()
+
+    def test_default_content_source(self):
+        """Test that the default content_source is 'cleaned_html'."""
+        # Can't directly instantiate abstract class, so just test DefaultMarkdownGenerator
+        generator = DefaultMarkdownGenerator()
+        self.assertEqual(generator.content_source, "cleaned_html")
+
+    def test_custom_content_source(self):
+        """Test that content_source can be customized."""
+        generator = DefaultMarkdownGenerator(content_source="fit_html")
+        self.assertEqual(generator.content_source, "fit_html")
+
+    @patch('crawl4ai.markdown_generation_strategy.CustomHTML2Text')
+    def test_html_processing_using_input_html(self, mock_html2text):
+        """Test that generate_markdown uses input_html parameter."""
+        # Setup mock
+        mock_instance = MagicMock()
+        mock_instance.handle.return_value = "# Test Content\n\nThis is a test paragraph."
+        mock_html2text.return_value = mock_instance
+
+        # Create generator and call generate_markdown
+        generator = DefaultMarkdownGenerator()
+        result = generator.generate_markdown(input_html="
      Test Content
      This is a test paragraph.
      ")
+
+        # Verify input_html was passed to HTML2Text handler
+        mock_instance.handle.assert_called_once()
+        # Get the first positional argument
+        args, _ = mock_instance.handle.call_args
+        self.assertEqual(args[0], "
      Test Content
      This is a test paragraph.
      ")
+        
+        # Check result
+        self.assertIsInstance(result, MarkdownGenerationResult)
+        self.assertEqual(result.raw_markdown, "# Test Content\n\nThis is a test paragraph.")
+
+    def test_html_source_selection_logic(self):
+        """Test that the HTML source selection logic works correctly."""
+        # We'll test the dispatch pattern directly to avoid async complexities
+        
+        # Create test data
+        raw_html = "
      Raw HTML
      "
+        cleaned_html = "
      Cleaned HTML
      "
+        fit_html = "
      Preprocessed HTML
      "
+        
+        # Test the dispatch pattern
+        html_source_selector = {
+            "raw_html": lambda: raw_html,
+            "cleaned_html": lambda: cleaned_html,
+            "fit_html": lambda: fit_html,
+        }
+        
+        # Test Case 1: content_source="cleaned_html"
+        source_lambda = html_source_selector.get("cleaned_html")
+        self.assertEqual(source_lambda(), cleaned_html)
+        
+        # Test Case 2: content_source="raw_html"
+        source_lambda = html_source_selector.get("raw_html")
+        self.assertEqual(source_lambda(), raw_html)
+        
+        # Test Case 3: content_source="fit_html"
+        source_lambda = html_source_selector.get("fit_html")
+        self.assertEqual(source_lambda(), fit_html)
+        
+        # Test Case 4: Invalid content_source falls back to cleaned_html
+        source_lambda = html_source_selector.get("invalid_source", lambda: cleaned_html)
+        self.assertEqual(source_lambda(), cleaned_html)
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file
diff --git a/tests/general/test_crawlers.py b/tests/general/test_crawlers.py
new file mode 100644
index 0000000..45fb8fc
--- /dev/null
+++ b/tests/general/test_crawlers.py
@@ -0,0 +1,17 @@
+
+# example_usageexample_usageexample_usage# example_usage.py
+import asyncio
+from crawl4ai.crawlers import get_crawler
+
+async def main():
+    # Get the registered crawler
+    example_crawler = get_crawler("example_site.content")
+    
+    # Crawl example.com
+    result = await example_crawler(url="https://example.com")
+        
+    print(result)
+            
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/general/test_deep_crawl.py b/tests/general/test_deep_crawl.py
new file mode 100644
index 0000000..2f533cc
--- /dev/null
+++ b/tests/general/test_deep_crawl.py
@@ -0,0 +1,46 @@
+import asyncio
+import time
+
+
+from crawl4ai import CrawlerRunConfig, AsyncWebCrawler, CacheMode
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+# from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+
+
+async def main():
+    """Example deep crawl of documentation site."""
+    config = CrawlerRunConfig(
+        deep_crawl_strategy = BFSDeepCrawlStrategy(
+            max_depth=2,
+            include_external=False
+        ),
+        stream=False,
+        verbose=True,
+        cache_mode=CacheMode.BYPASS,
+        scraping_strategy=LXMLWebScrapingStrategy()
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        start_time = time.perf_counter()
+        print("\nStarting deep crawl in batch mode:")
+        results = await crawler.arun(
+            url="https://docs.crawl4ai.com",
+            config=config
+        )
+        print(f"Crawled {len(results)} pages")
+        print(f"Example page: {results[0].url}")
+        print(f"Duration: {time.perf_counter() - start_time:.2f} seconds\n")
+
+        print("Starting deep crawl in streaming mode:")
+        config.stream = True
+        start_time = time.perf_counter()
+        async for result in await crawler.arun(
+            url="https://docs.crawl4ai.com",
+            config=config
+        ):
+            print(f"→ {result.url} (Depth: {result.metadata.get('depth', 0)})")
+        print(f"Duration: {time.perf_counter() - start_time:.2f} seconds")
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/general/test_deep_crawl_filters.py b/tests/general/test_deep_crawl_filters.py
new file mode 100644
index 0000000..948bbcb
--- /dev/null
+++ b/tests/general/test_deep_crawl_filters.py
@@ -0,0 +1,279 @@
+from crawl4ai.deep_crawling.filters import ContentRelevanceFilter, URLPatternFilter, DomainFilter, ContentTypeFilter, SEOFilter
+async def test_pattern_filter():
+    # Test cases as list of tuples instead of dict for multiple patterns
+    test_cases = [
+        # Simple suffix patterns (*.html)
+        ("*.html", {
+            "https://example.com/page.html": True,
+            "https://example.com/path/doc.html": True,
+            "https://example.com/page.htm": False,
+            "https://example.com/page.html?param=1": True,
+        }),
+        
+        # Path prefix patterns (/foo/*)
+        ("*/article/*", {
+            "https://example.com/article/123": True,
+            "https://example.com/blog/article/456": True,
+            "https://example.com/articles/789": False,
+            "https://example.com/article": False,
+        }),
+        
+        # Complex patterns
+        ("blog-*-[0-9]", {
+            "https://example.com/blog-post-1": True,
+            "https://example.com/blog-test-9": True,
+            "https://example.com/blog-post": False,
+            "https://example.com/blog-post-x": False,
+        }),
+        
+        # Multiple patterns case
+        (["*.pdf", "*/download/*"], {
+            "https://example.com/doc.pdf": True,
+            "https://example.com/download/file.txt": True,
+            "https://example.com/path/download/doc": True,
+            "https://example.com/uploads/file.txt": False,
+        }),
+        
+        # Edge cases
+        ("*", {
+            "https://example.com": True,
+            "": True,
+            "http://test.com/path": True,
+        }),
+        
+        # Complex regex
+        (r"^https?://.*\.example\.com/\d+", {
+            "https://sub.example.com/123": True,
+            "http://test.example.com/456": True,
+            "https://example.com/789": False,
+            "https://sub.example.com/abc": False,
+        })
+    ]
+
+    def run_accuracy_test():
+        print("\nAccuracy Tests:")
+        print("-" * 50)
+        
+        all_passed = True
+        for patterns, test_urls in test_cases:
+            filter_obj = URLPatternFilter(patterns)
+            
+            for url, expected in test_urls.items():
+                result = filter_obj.apply(url)
+                if result != expected:
+                    print(f"❌ Failed: Pattern '{patterns}' with URL '{url}'")
+                    print(f"   Expected: {expected}, Got: {result}")
+                    all_passed = False
+                else:
+                    print(f"✅ Passed: Pattern '{patterns}' with URL '{url}'")
+        
+        return all_passed
+
+    # Run tests
+    print("Running Pattern Filter Tests...")
+    accuracy_passed = run_accuracy_test()
+    
+    if accuracy_passed:
+        print("\n✨ All accuracy tests passed!")
+        
+    else:
+        print("\n❌ Some accuracy tests failed!")
+
+async def test_domain_filter():
+    from itertools import chain
+
+    # Test cases
+    test_cases = [
+        # Allowed domains
+        ({"allowed": "example.com"}, {
+            "https://example.com/page": True,
+            "http://example.com": True,
+            "https://sub.example.com": False,
+            "https://other.com": False,
+        }),
+
+        ({"allowed": ["example.com", "test.com"]}, {
+            "https://example.com/page": True,
+            "https://test.com/home": True,
+            "https://other.com": False,
+        }),
+
+        # Blocked domains
+        ({"blocked": "malicious.com"}, {
+            "https://malicious.com": False,
+            "https://safe.com": True,
+            "http://malicious.com/login": False,
+        }),
+
+        ({"blocked": ["spam.com", "ads.com"]}, {
+            "https://spam.com": False,
+            "https://ads.com/banner": False,
+            "https://example.com": True,
+        }),
+
+        # Allowed and Blocked combination
+        ({"allowed": "example.com", "blocked": "sub.example.com"}, {
+            "https://example.com": True,
+            "https://sub.example.com": False,
+            "https://other.com": False,
+        }),
+    ]
+
+    def run_accuracy_test():
+        print("\nAccuracy Tests:")
+        print("-" * 50)
+        
+        all_passed = True
+        for params, test_urls in test_cases:
+            filter_obj = DomainFilter(
+                allowed_domains=params.get("allowed"),
+                blocked_domains=params.get("blocked"),
+            )
+            
+            for url, expected in test_urls.items():
+                result = filter_obj.apply(url)
+                if result != expected:
+                    print(f"\u274C Failed: Params {params} with URL '{url}'")
+                    print(f"   Expected: {expected}, Got: {result}")
+                    all_passed = False
+                else:
+                    print(f"\u2705 Passed: Params {params} with URL '{url}'")
+        
+        return all_passed
+
+    # Run tests
+    print("Running Domain Filter Tests...")
+    accuracy_passed = run_accuracy_test()
+    
+    if accuracy_passed:
+        print("\n\u2728 All accuracy tests passed!")
+    else:
+        print("\n\u274C Some accuracy tests failed!")
+
+async def test_content_relevance_filter():
+    relevance_filter = ContentRelevanceFilter(
+        query="What was the cause of american civil war?", 
+        threshold=1
+    )
+
+    test_cases = {
+        "https://en.wikipedia.org/wiki/Cricket": False,
+        "https://en.wikipedia.org/wiki/American_Civil_War": True,
+    }
+
+    print("\nRunning Content Relevance Filter Tests...")
+    print("-" * 50)
+    
+    all_passed = True
+    for url, expected in test_cases.items():
+        result = await relevance_filter.apply(url)
+        if result != expected:
+            print(f"\u274C Failed: URL '{url}'")
+            print(f"   Expected: {expected}, Got: {result}")
+            all_passed = False
+        else:
+            print(f"\u2705 Passed: URL '{url}'")
+    
+    if all_passed:
+        print("\n\u2728 All content relevance tests passed!")
+    else:
+        print("\n\u274C Some content relevance tests failed!")
+
+async def test_content_type_filter():
+    from itertools import chain
+
+    # Test cases
+    test_cases = [
+        # Allowed single type
+        ({"allowed": "image/png"}, {
+            "https://example.com/image.png": True,
+            "https://example.com/photo.jpg": False,
+            "https://example.com/document.pdf": False,
+        }),
+
+        # Multiple allowed types
+        ({"allowed": ["image/jpeg", "application/pdf"]}, {
+            "https://example.com/photo.jpg": True,
+            "https://example.com/document.pdf": True,
+            "https://example.com/script.js": False,
+        }),
+
+        # No extension should be allowed
+        ({"allowed": "application/json"}, {
+            "https://example.com/api/data": True,
+            "https://example.com/data.json": True,
+            "https://example.com/page.html": False,
+        }),
+
+        # Unknown extensions should not be allowed
+        ({"allowed": "application/octet-stream"}, {
+            "https://example.com/file.unknown": True,
+            "https://example.com/archive.zip": False,
+            "https://example.com/software.exe": False,
+        }),
+    ]
+
+    def run_accuracy_test():
+        print("\nAccuracy Tests:")
+        print("-" * 50)
+        
+        all_passed = True
+        for params, test_urls in test_cases:
+            filter_obj = ContentTypeFilter(
+                allowed_types=params.get("allowed"),
+            )
+            
+            for url, expected in test_urls.items():
+                result = filter_obj.apply(url)
+                if result != expected:
+                    print(f"\u274C Failed: Params {params} with URL '{url}'")
+                    print(f"   Expected: {expected}, Got: {result}")
+                    all_passed = False
+                else:
+                    print(f"\u2705 Passed: Params {params} with URL '{url}'")
+        
+        return all_passed
+
+    # Run tests
+    print("Running Content Type Filter Tests...")
+    accuracy_passed = run_accuracy_test()
+    
+    if accuracy_passed:
+        print("\n\u2728 All accuracy tests passed!")
+    else:
+        print("\n\u274C Some accuracy tests failed!")
+
+async def test_seo_filter():
+    seo_filter = SEOFilter(threshold=0.5, keywords=["SEO", "search engines", "Optimization"])
+
+    test_cases = {
+        "https://en.wikipedia.org/wiki/Search_engine_optimization": True,
+        "https://en.wikipedia.org/wiki/Randomness": False,
+    }
+
+    print("\nRunning SEO Filter Tests...")
+    print("-" * 50)
+    
+    all_passed = True
+    for url, expected in test_cases.items():
+        result = await seo_filter.apply(url)
+        if result != expected:
+            print(f"\u274C Failed: URL '{url}'")
+            print(f"   Expected: {expected}, Got: {result}")
+            all_passed = False
+        else:
+            print(f"\u2705 Passed: URL '{url}'")
+    
+    if all_passed:
+        print("\n\u2728 All SEO filter tests passed!")
+    else:
+        print("\n\u274C Some SEO filter tests failed!")
+
+import asyncio
+
+if __name__ == "__main__":
+    asyncio.run(test_pattern_filter())
+    asyncio.run(test_domain_filter())
+    asyncio.run(test_content_type_filter())
+    asyncio.run(test_content_relevance_filter())
+    asyncio.run(test_seo_filter())
\ No newline at end of file
diff --git a/tests/general/test_deep_crawl_scorers.py b/tests/general/test_deep_crawl_scorers.py
new file mode 100644
index 0000000..8e68bca
--- /dev/null
+++ b/tests/general/test_deep_crawl_scorers.py
@@ -0,0 +1,179 @@
+from crawl4ai.deep_crawling.scorers import CompositeScorer, ContentTypeScorer, DomainAuthorityScorer, FreshnessScorer, KeywordRelevanceScorer, PathDepthScorer
+
+
+def test_scorers():
+    test_cases = [
+        # Keyword Scorer Tests
+        {
+            "scorer_type": "keyword",
+            "config": {
+                "keywords": ["python", "blog"],
+                "weight": 1.0,
+                "case_sensitive": False
+            },
+            "urls": {
+                "https://example.com/python-blog": 1.0,
+                "https://example.com/PYTHON-BLOG": 1.0,
+                "https://example.com/python-only": 0.5,
+                "https://example.com/other": 0.0
+            }
+        },
+        
+        # Path Depth Scorer Tests
+        {
+            "scorer_type": "path_depth",
+            "config": {
+                "optimal_depth": 2,
+                "weight": 1.0
+            },
+            "urls": {
+                "https://example.com/a/b": 1.0,
+                "https://example.com/a": 0.5,
+                "https://example.com/a/b/c": 0.5,
+                "https://example.com": 0.33333333
+            }
+        },
+        
+        # Content Type Scorer Tests
+        {
+            "scorer_type": "content_type",
+            "config": {
+                "type_weights": {
+                    ".html$": 1.0,
+                    ".pdf$": 0.8,
+                    ".jpg$": 0.6
+                },
+                "weight": 1.0
+            },
+            "urls": {
+                "https://example.com/doc.html": 1.0,
+                "https://example.com/doc.pdf": 0.8,
+                "https://example.com/img.jpg": 0.6,
+                "https://example.com/other.txt": 0.0
+            }
+        },
+        
+        # Freshness Scorer Tests
+        {
+            "scorer_type": "freshness",
+            "config": {
+                "weight": 1.0,  # Remove current_year since original doesn't support it
+            },
+            "urls": {
+                "https://example.com/2024/01/post": 1.0,
+                "https://example.com/2023/12/post": 0.9,
+                "https://example.com/2022/post": 0.8,
+                "https://example.com/no-date": 0.5
+            }
+        },
+        
+        # Domain Authority Scorer Tests
+        {
+            "scorer_type": "domain",
+            "config": {
+                "domain_weights": {
+                    "python.org": 1.0,
+                    "github.com": 0.8,
+                    "medium.com": 0.6
+                },
+                "default_weight": 0.3,
+                "weight": 1.0
+            },
+            "urls": {
+                "https://python.org/about": 1.0,
+                "https://github.com/repo": 0.8,
+                "https://medium.com/post": 0.6,
+                "https://unknown.com": 0.3
+            }
+        }
+    ]
+
+    def create_scorer(scorer_type, config):
+        if scorer_type == "keyword":
+            return KeywordRelevanceScorer(**config)
+        elif scorer_type == "path_depth":
+            return PathDepthScorer(**config)
+        elif scorer_type == "content_type":
+            return ContentTypeScorer(**config)
+        elif scorer_type == "freshness":
+            return FreshnessScorer(**config,current_year=2024)
+        elif scorer_type == "domain":
+            return DomainAuthorityScorer(**config)
+
+    def run_accuracy_test():
+        print("\nAccuracy Tests:")
+        print("-" * 50)
+        
+        all_passed = True
+        for test_case in test_cases:
+            print(f"\nTesting {test_case['scorer_type']} scorer:")
+            scorer = create_scorer(
+                test_case['scorer_type'],
+                test_case['config']
+            )
+            
+            for url, expected in test_case['urls'].items():
+                score = round(scorer.score(url), 8)
+                expected = round(expected, 8)
+                
+                if abs(score - expected) > 0.00001:
+                    print(f"❌ Scorer Failed: URL '{url}'")
+                    print(f"   Expected: {expected}, Got: {score}")
+                    all_passed = False
+                else:
+                    print(f"✅ Scorer Passed: URL '{url}'")
+                    
+                    
+        return all_passed
+
+    def run_composite_test():
+        print("\nTesting Composite Scorer:")
+        print("-" * 50)
+        
+        # Create test data
+        test_urls = {
+            "https://python.org/blog/2024/01/new-release.html":0.86666667,
+            "https://github.com/repo/old-code.pdf": 0.62,
+            "https://unknown.com/random": 0.26
+        }
+        
+        # Create composite scorers with all types
+        scorers = []
+        
+        for test_case in test_cases:
+            scorer = create_scorer(
+                test_case['scorer_type'],
+                test_case['config']
+            )
+            scorers.append(scorer)
+            
+        composite = CompositeScorer(scorers, normalize=True)
+        
+        all_passed = True
+        for url, expected in test_urls.items():
+            score = round(composite.score(url), 8)
+            
+            if abs(score - expected) > 0.00001:
+                print(f"❌ Composite Failed: URL '{url}'")
+                print(f"   Expected: {expected}, Got: {score}")
+                all_passed = False
+            else:
+                print(f"✅ Composite Passed: URL '{url}'")
+                
+        return all_passed
+
+    # Run tests
+    print("Running Scorer Tests...")
+    accuracy_passed = run_accuracy_test()
+    composite_passed = run_composite_test()
+    
+    if accuracy_passed and composite_passed:
+        print("\n✨ All tests passed!")
+        # Note: Already have performance tests in run_scorer_performance_test()
+    else:
+        print("\n❌ Some tests failed!")
+
+    
+
+if __name__ == "__main__":
+    test_scorers()
\ No newline at end of file
diff --git a/tests/general/test_download_file.py b/tests/general/test_download_file.py
new file mode 100644
index 0000000..ca55277
--- /dev/null
+++ b/tests/general/test_download_file.py
@@ -0,0 +1,34 @@
+import asyncio
+from crawl4ai import CrawlerRunConfig, AsyncWebCrawler, BrowserConfig
+from pathlib import Path
+import os
+
+async def test_basic_download():
+    
+    # Custom folder (otherwise defaults to ~/.crawl4ai/downloads)
+    downloads_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
+    os.makedirs(downloads_path, exist_ok=True)
+    browser_config = BrowserConfig(
+        accept_downloads=True,
+        downloads_path=downloads_path
+    )
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        run_config = CrawlerRunConfig(
+            js_code="""
+                const link = document.querySelector('a[href$=".exe"]');
+                if (link) { link.click(); }
+            """,
+            delay_before_return_html=5  
+        )
+        result = await crawler.arun("https://www.python.org/downloads/", config=run_config)
+
+        if result.downloaded_files:
+            print("Downloaded files:")
+            for file_path in result.downloaded_files:
+                print("•", file_path)
+        else:
+            print("No files downloaded.")
+
+if __name__ == "__main__":
+    asyncio.run(test_basic_download())
+ 
\ No newline at end of file
diff --git a/tests/general/test_flatten_shadow_dom.py b/tests/general/test_flatten_shadow_dom.py
new file mode 100644
index 0000000..6b7a236
--- /dev/null
+++ b/tests/general/test_flatten_shadow_dom.py
@@ -0,0 +1,84 @@
+"""Test flatten_shadow_dom feature — full comparison."""
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+URL = "https://store.boschrexroth.com/en/us/p/hydraulic-cylinder-r900999011"
+
+
+async def run_test(label, bc, rc):
+    print(f"\n{'='*70}")
+    print(f"TEST: {label}")
+    print(f"{'='*70}")
+    async with AsyncWebCrawler(config=bc) as crawler:
+        result = await crawler.arun(URL, config=rc)
+
+    html = result.html or ""
+    cleaned = result.cleaned_html or ""
+    md = ""
+    if result.markdown and hasattr(result.markdown, "raw_markdown"):
+        md = result.markdown.raw_markdown or ""
+
+    print(f"  Success:      {result.success}")
+    print(f"  Raw HTML:     {len(html):>8} chars")
+    print(f"  Cleaned HTML: {len(cleaned):>8} chars")
+    print(f"  Markdown:     {len(md):>8} chars")
+
+    checks = {
+        "Product title":               "HYDRAULIC CYLINDER" in md,
+        "Part number (R900999011)":    "R900999011" in md,
+        "Product description":         "mill type design" in md.lower(),
+        "Feature: 6 types of mounting":"6 types of mounting" in md,
+        "Feature: safety vent":        "safety vent" in md.lower(),
+        "Product Description heading": "Product Description" in md,
+        "Technical Specs heading":     "Technical Specs" in md,
+        "Downloads heading":           "Downloads" in md,
+        "Specs table: CDH1":           "CDH1" in md,
+        "Specs table: 250 bar":        "250" in md,
+    }
+    print(f"\n  Content checks:")
+    passes = sum(1 for v in checks.values() if v)
+    for k, v in checks.items():
+        print(f"    {'PASS' if v else 'FAIL'}  {k}")
+    print(f"\n  Result: {passes}/{len(checks)} checks passed")
+
+    # Show product content section
+    for term in ["Product Description"]:
+        idx = md.find(term)
+        if idx >= 0:
+            print(f"\n  --- Product content section ---")
+            print(md[idx:idx+1500])
+    return result
+
+
+async def main():
+    bc = BrowserConfig(headless=True)
+
+    r1 = await run_test(
+        "BASELINE (no shadow flattening)",
+        bc,
+        CrawlerRunConfig(wait_until="load", delay_before_return_html=3.0),
+    )
+
+    r2 = await run_test(
+        "WITH flatten_shadow_dom=True",
+        bc,
+        CrawlerRunConfig(
+            wait_until="load",
+            delay_before_return_html=3.0,
+            flatten_shadow_dom=True,
+        ),
+    )
+
+    # Summary
+    md1 = r1.markdown.raw_markdown if r1.markdown else ""
+    md2 = r2.markdown.raw_markdown if r2.markdown else ""
+    print(f"\n{'='*70}")
+    print(f"SUMMARY")
+    print(f"{'='*70}")
+    print(f"  Baseline markdown:  {len(md1):>6} chars")
+    print(f"  Flattened markdown: {len(md2):>6} chars")
+    print(f"  Improvement:        {len(md2)/max(len(md1),1):.1f}x more content")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/general/test_generate_schema_usage.py b/tests/general/test_generate_schema_usage.py
new file mode 100644
index 0000000..6f7227e
--- /dev/null
+++ b/tests/general/test_generate_schema_usage.py
@@ -0,0 +1,654 @@
+"""Tests for TokenUsage accumulation in generate_schema / agenerate_schema.
+
+Covers:
+- Backward compatibility (usage=None, the default)
+- Single-shot schema generation accumulates usage
+- Validation retry loop accumulates across all LLM calls
+- _infer_target_json accumulates its own LLM call
+- Sync wrapper forwards usage correctly
+- JSON parse failure retry also accumulates usage
+- usage object receives correct cumulative totals
+"""
+
+import asyncio
+import json
+import pytest
+from dataclasses import dataclass
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch, MagicMock
+
+from crawl4ai.extraction_strategy import JsonElementExtractionStrategy, JsonCssExtractionStrategy
+from crawl4ai.models import TokenUsage
+
+# The functions are imported lazily inside method bodies via `from .utils import ...`
+# so we must patch at the source module.
+PATCH_TARGET = "crawl4ai.utils.aperform_completion_with_backoff"
+
+
+# ---------------------------------------------------------------------------
+# Helpers: fake LLM response builder
+# ---------------------------------------------------------------------------
+
+def _make_llm_response(content: str, prompt_tokens: int = 100, completion_tokens: int = 50):
+    """Build a fake litellm-style response with .usage and .choices."""
+    return SimpleNamespace(
+        usage=SimpleNamespace(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+            completion_tokens_details=None,
+            prompt_tokens_details=None,
+        ),
+        choices=[
+            SimpleNamespace(
+                message=SimpleNamespace(content=content)
+            )
+        ],
+    )
+
+
+# A valid CSS schema that will pass validation against SAMPLE_HTML
+VALID_SCHEMA = {
+    "name": "products",
+    "baseSelector": ".product",
+    "fields": [
+        {"name": "title", "selector": ".title", "type": "text"},
+        {"name": "price", "selector": ".price", "type": "text"},
+    ],
+}
+
+SAMPLE_HTML = """
+
      
+    
      
+        Widget
+        $10
+    
      
+    
      
+        Gadget
+        $20
+    
      
+
      
+"""
+
+# A schema with a bad baseSelector — will fail validation and trigger retry
+BAD_SCHEMA = {
+    "name": "products",
+    "baseSelector": ".nonexistent-selector",
+    "fields": [
+        {"name": "title", "selector": ".title", "type": "text"},
+        {"name": "price", "selector": ".price", "type": "text"},
+    ],
+}
+
+# Fake LLMConfig
+@dataclass
+class FakeLLMConfig:
+    provider: str = "fake/model"
+    api_token: str = "fake-token"
+    base_url: str = None
+    backoff_base_delay: float = 0
+    backoff_max_attempts: int = 1
+    backoff_exponential_factor: int = 2
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+class TestGenerateSchemaUsage:
+    """Test suite for usage tracking in generate_schema / agenerate_schema."""
+
+    @pytest.mark.asyncio
+    async def test_backward_compat_usage_none(self):
+        """When usage is not passed (default None), everything works as before."""
+        mock_response = _make_llm_response(json.dumps(VALID_SCHEMA))
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=False,
+            )
+
+        assert isinstance(result, dict)
+        assert result["name"] == "products"
+
+    @pytest.mark.asyncio
+    async def test_single_shot_no_validate(self):
+        """Single LLM call with validate=False populates usage correctly."""
+        usage = TokenUsage()
+        mock_response = _make_llm_response(
+            json.dumps(VALID_SCHEMA), prompt_tokens=200, completion_tokens=80
+        )
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=False,
+                usage=usage,
+            )
+
+        assert result["name"] == "products"
+        assert usage.prompt_tokens == 200
+        assert usage.completion_tokens == 80
+        assert usage.total_tokens == 280
+
+    @pytest.mark.asyncio
+    async def test_validation_success_first_try(self):
+        """With validate=True and schema passes validation on first try, usage reflects 1 call."""
+        usage = TokenUsage()
+        mock_response = _make_llm_response(
+            json.dumps(VALID_SCHEMA), prompt_tokens=300, completion_tokens=120
+        )
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=True,
+                max_refinements=3,
+                usage=usage,
+                # Provide target_json_example to skip _infer_target_json
+                target_json_example='{"title": "x", "price": "y"}',
+            )
+
+        assert result["name"] == "products"
+        # Only 1 LLM call since validation passed
+        assert usage.prompt_tokens == 300
+        assert usage.completion_tokens == 120
+        assert usage.total_tokens == 420
+
+    @pytest.mark.asyncio
+    async def test_validation_retries_accumulate_usage(self):
+        """When validation fails, retry calls accumulate into the same usage object."""
+        usage = TokenUsage()
+
+        # First call returns bad schema (fails validation), second returns good schema
+        responses = [
+            _make_llm_response(json.dumps(BAD_SCHEMA), prompt_tokens=300, completion_tokens=100),
+            _make_llm_response(json.dumps(VALID_SCHEMA), prompt_tokens=350, completion_tokens=120),
+        ]
+        call_count = 0
+
+        async def mock_completion(*args, **kwargs):
+            nonlocal call_count
+            idx = min(call_count, len(responses) - 1)
+            call_count += 1
+            return responses[idx]
+
+        with patch(
+            PATCH_TARGET,
+            side_effect=mock_completion,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=True,
+                max_refinements=3,
+                usage=usage,
+                target_json_example='{"title": "x", "price": "y"}',
+            )
+
+        assert result["name"] == "products"
+        # Two LLM calls: 300+350=650 prompt, 100+120=220 completion
+        assert usage.prompt_tokens == 650
+        assert usage.completion_tokens == 220
+        assert usage.total_tokens == 870
+
+    @pytest.mark.asyncio
+    async def test_infer_target_json_accumulates_usage(self):
+        """When validate=True and no target_json_example, _infer_target_json makes an extra LLM call."""
+        usage = TokenUsage()
+
+        infer_response = _make_llm_response(
+            '{"title": "Widget", "price": "$10"}',
+            prompt_tokens=50,
+            completion_tokens=30,
+        )
+        schema_response = _make_llm_response(
+            json.dumps(VALID_SCHEMA),
+            prompt_tokens=300,
+            completion_tokens=120,
+        )
+
+        call_count = 0
+
+        async def mock_completion(*args, **kwargs):
+            nonlocal call_count
+            call_count += 1
+            # First call is _infer_target_json, second is schema generation
+            if call_count == 1:
+                return infer_response
+            return schema_response
+
+        with patch(
+            PATCH_TARGET,
+            side_effect=mock_completion,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                query="extract product title and price",
+                llm_config=FakeLLMConfig(),
+                validate=True,
+                max_refinements=3,
+                usage=usage,
+                # No target_json_example — triggers _infer_target_json
+            )
+
+        assert result["name"] == "products"
+        # _infer_target_json: 50+30 = 80
+        # schema generation: 300+120 = 420
+        # Total: 350 prompt, 150 completion, 500 total
+        assert usage.prompt_tokens == 350
+        assert usage.completion_tokens == 150
+        assert usage.total_tokens == 500
+
+    @pytest.mark.asyncio
+    async def test_infer_plus_retries_accumulate(self):
+        """Full pipeline: infer + bad schema + good schema = 3 calls accumulated."""
+        usage = TokenUsage()
+
+        infer_resp = _make_llm_response(
+            '{"title": "x", "price": "y"}',
+            prompt_tokens=50, completion_tokens=20,
+        )
+        bad_resp = _make_llm_response(
+            json.dumps(BAD_SCHEMA),
+            prompt_tokens=300, completion_tokens=100,
+        )
+        good_resp = _make_llm_response(
+            json.dumps(VALID_SCHEMA),
+            prompt_tokens=400, completion_tokens=150,
+        )
+
+        call_count = 0
+
+        async def mock_completion(*args, **kwargs):
+            nonlocal call_count
+            call_count += 1
+            if call_count == 1:
+                return infer_resp
+            elif call_count == 2:
+                return bad_resp
+            else:
+                return good_resp
+
+        with patch(
+            PATCH_TARGET,
+            side_effect=mock_completion,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                query="extract products",
+                llm_config=FakeLLMConfig(),
+                validate=True,
+                max_refinements=3,
+                usage=usage,
+            )
+
+        # 3 calls total
+        assert call_count == 3
+        assert usage.prompt_tokens == 750    # 50 + 300 + 400
+        assert usage.completion_tokens == 270  # 20 + 100 + 150
+        assert usage.total_tokens == 1020    # 70 + 400 + 550
+
+    @pytest.mark.asyncio
+    async def test_json_parse_failure_retry_accumulates(self):
+        """When LLM returns invalid JSON, the retry also accumulates usage."""
+        usage = TokenUsage()
+
+        # First response is not valid JSON, second is valid
+        bad_json_resp = _make_llm_response(
+            "this is not json {{{",
+            prompt_tokens=200, completion_tokens=60,
+        )
+        good_resp = _make_llm_response(
+            json.dumps(VALID_SCHEMA),
+            prompt_tokens=250, completion_tokens=80,
+        )
+
+        call_count = 0
+
+        async def mock_completion(*args, **kwargs):
+            nonlocal call_count
+            call_count += 1
+            if call_count == 1:
+                return bad_json_resp
+            return good_resp
+
+        with patch(
+            PATCH_TARGET,
+            side_effect=mock_completion,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=True,
+                max_refinements=3,
+                usage=usage,
+                target_json_example='{"title": "x", "price": "y"}',
+            )
+
+        assert result["name"] == "products"
+        # Both calls tracked: even the one that returned bad JSON
+        assert usage.prompt_tokens == 450   # 200 + 250
+        assert usage.completion_tokens == 140  # 60 + 80
+        assert usage.total_tokens == 590
+
+    @pytest.mark.asyncio
+    async def test_usage_none_does_not_crash(self):
+        """Explicitly passing usage=None should not raise any errors."""
+        mock_response = _make_llm_response(json.dumps(VALID_SCHEMA))
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=False,
+                usage=None,
+            )
+
+        assert isinstance(result, dict)
+
+    @pytest.mark.asyncio
+    async def test_preexisting_usage_values_are_added_to(self):
+        """If usage already has values, new tokens are ADDED, not replaced."""
+        usage = TokenUsage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
+
+        mock_response = _make_llm_response(
+            json.dumps(VALID_SCHEMA), prompt_tokens=200, completion_tokens=80
+        )
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=False,
+                usage=usage,
+            )
+
+        assert usage.prompt_tokens == 1200    # 1000 + 200
+        assert usage.completion_tokens == 580  # 500 + 80
+        assert usage.total_tokens == 1780     # 1500 + 280
+
+    def test_sync_wrapper_passes_usage(self):
+        """The sync generate_schema forwards usage to agenerate_schema."""
+        usage = TokenUsage()
+        mock_response = _make_llm_response(
+            json.dumps(VALID_SCHEMA), prompt_tokens=200, completion_tokens=80
+        )
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = JsonElementExtractionStrategy.generate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=False,
+                usage=usage,
+            )
+
+        assert result["name"] == "products"
+        assert usage.prompt_tokens == 200
+        assert usage.completion_tokens == 80
+        assert usage.total_tokens == 280
+
+    def test_sync_wrapper_usage_none_backward_compat(self):
+        """Sync wrapper with no usage arg (default) still works."""
+        mock_response = _make_llm_response(json.dumps(VALID_SCHEMA))
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = JsonElementExtractionStrategy.generate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=False,
+            )
+
+        assert isinstance(result, dict)
+        assert result["name"] == "products"
+
+    @pytest.mark.asyncio
+    async def test_max_refinements_zero_single_call(self):
+        """max_refinements=0 with validate=True means exactly 1 attempt, 1 usage entry."""
+        usage = TokenUsage()
+        mock_response = _make_llm_response(
+            json.dumps(BAD_SCHEMA), prompt_tokens=300, completion_tokens=100
+        )
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=True,
+                max_refinements=0,
+                usage=usage,
+                target_json_example='{"title": "x", "price": "y"}',
+            )
+
+        # Even though validation fails, only 1 attempt (0 refinements)
+        assert usage.prompt_tokens == 300
+        assert usage.completion_tokens == 100
+        assert usage.total_tokens == 400
+
+    @pytest.mark.asyncio
+    async def test_css_subclass_inherits_usage(self):
+        """JsonCssExtractionStrategy.agenerate_schema also supports usage."""
+        usage = TokenUsage()
+        mock_response = _make_llm_response(
+            json.dumps(VALID_SCHEMA), prompt_tokens=150, completion_tokens=60
+        )
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonCssExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=False,
+                usage=usage,
+            )
+
+        assert result["name"] == "products"
+        assert usage.total_tokens == 210
+
+    @pytest.mark.asyncio
+    async def test_infer_target_json_failure_still_tracks_nothing(self):
+        """If _infer_target_json raises (and catches), usage should not break.
+
+        When the inference LLM call itself throws an exception before we get
+        response.usage, no tokens should be added (graceful degradation).
+        """
+        usage = TokenUsage()
+
+        call_count = 0
+
+        async def mock_completion(*args, **kwargs):
+            nonlocal call_count
+            call_count += 1
+            if call_count == 1:
+                # _infer_target_json call — simulate exception
+                raise ConnectionError("LLM is down")
+            # Schema generation call
+            return _make_llm_response(
+                json.dumps(VALID_SCHEMA),
+                prompt_tokens=300,
+                completion_tokens=100,
+            )
+
+        with patch(
+            PATCH_TARGET,
+            side_effect=mock_completion,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                query="extract products",
+                llm_config=FakeLLMConfig(),
+                validate=True,
+                max_refinements=0,
+                usage=usage,
+            )
+
+        # Only the schema call counted; infer call failed before tracking
+        assert usage.prompt_tokens == 300
+        assert usage.completion_tokens == 100
+        assert usage.total_tokens == 400
+
+    @pytest.mark.asyncio
+    async def test_multiple_bad_retries_then_best_effort(self):
+        """All retries fail validation, usage still accumulates for every attempt."""
+        usage = TokenUsage()
+
+        # Every call returns bad schema — validation will always fail
+        mock_response = _make_llm_response(
+            json.dumps(BAD_SCHEMA), prompt_tokens=200, completion_tokens=80
+        )
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy.agenerate_schema(
+                html=SAMPLE_HTML,
+                llm_config=FakeLLMConfig(),
+                validate=True,
+                max_refinements=2,  # 1 initial + 2 retries = 3 calls
+                usage=usage,
+                target_json_example='{"title": "x", "price": "y"}',
+            )
+
+        # Returns best-effort (last schema), but all 3 calls tracked
+        assert usage.prompt_tokens == 600    # 200 * 3
+        assert usage.completion_tokens == 240  # 80 * 3
+        assert usage.total_tokens == 840     # 280 * 3
+
+
+class TestInferTargetJsonUsage:
+    """Isolated tests for _infer_target_json usage tracking."""
+
+    @pytest.mark.asyncio
+    async def test_infer_tracks_usage(self):
+        """Direct call to _infer_target_json with usage accumulator."""
+        usage = TokenUsage()
+        mock_response = _make_llm_response(
+            '{"name": "test", "value": "123"}',
+            prompt_tokens=80,
+            completion_tokens=25,
+        )
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy._infer_target_json(
+                query="extract names and values",
+                html_snippet="
      test
      ",
+                llm_config=FakeLLMConfig(),
+                usage=usage,
+            )
+
+        assert result == {"name": "test", "value": "123"}
+        assert usage.prompt_tokens == 80
+        assert usage.completion_tokens == 25
+        assert usage.total_tokens == 105
+
+    @pytest.mark.asyncio
+    async def test_infer_usage_none_backward_compat(self):
+        """_infer_target_json with usage=None (default) still works."""
+        mock_response = _make_llm_response('{"name": "test"}')
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy._infer_target_json(
+                query="extract names",
+                html_snippet="
      test
      ",
+                llm_config=FakeLLMConfig(),
+            )
+
+        assert result == {"name": "test"}
+
+    @pytest.mark.asyncio
+    async def test_infer_exception_no_usage_side_effect(self):
+        """When _infer_target_json fails, usage is untouched (exception before tracking)."""
+        usage = TokenUsage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            side_effect=RuntimeError("API down"),
+        ):
+            result = await JsonElementExtractionStrategy._infer_target_json(
+                query="extract names",
+                html_snippet="
      test
      ",
+                llm_config=FakeLLMConfig(),
+                usage=usage,
+            )
+
+        # Returns None on failure
+        assert result is None
+        # Usage unchanged — exception happened before tracking
+        assert usage.prompt_tokens == 100
+        assert usage.completion_tokens == 50
+        assert usage.total_tokens == 150
+
+    @pytest.mark.asyncio
+    async def test_infer_empty_response_still_tracks(self):
+        """When LLM returns empty content, usage is still tracked (response was received)."""
+        usage = TokenUsage()
+        mock_response = _make_llm_response("", prompt_tokens=80, completion_tokens=5)
+
+        with patch(
+            PATCH_TARGET,
+            new_callable=AsyncMock,
+            return_value=mock_response,
+        ):
+            result = await JsonElementExtractionStrategy._infer_target_json(
+                query="extract names",
+                html_snippet="
      test
      ",
+                llm_config=FakeLLMConfig(),
+                usage=usage,
+            )
+
+        # Returns None because content is empty
+        assert result is None
+        # But usage was tracked because we got a response
+        assert usage.prompt_tokens == 80
+        assert usage.completion_tokens == 5
+        assert usage.total_tokens == 85
diff --git a/tests/general/test_http_crawler_strategy.py b/tests/general/test_http_crawler_strategy.py
new file mode 100644
index 0000000..dc14141
--- /dev/null
+++ b/tests/general/test_http_crawler_strategy.py
@@ -0,0 +1,116 @@
+from tkinter import N
+from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
+from crawl4ai.async_logger import AsyncLogger
+from crawl4ai import CrawlerRunConfig, HTTPCrawlerConfig
+from crawl4ai.async_crawler_strategy import ConnectionTimeoutError
+import asyncio
+import os
+
+async def main():
+    """Test the AsyncHTTPCrawlerStrategy with various scenarios"""
+    logger = AsyncLogger(verbose=True)
+
+    # Initialize the strategy with default HTTPCrawlerConfig
+    crawler = AsyncHTTPCrawlerStrategy(
+        browser_config=HTTPCrawlerConfig(),
+        logger=logger
+    )
+    # Test 1: Basic HTTP GET
+    print("\n=== Test 1: Basic HTTP GET ===")
+    result = await crawler.crawl("https://example.com")
+    print(f"Status: {result.status_code}")
+    print(f"Content length: {len(result.html)}")
+    print(f"Headers: {dict(result.response_headers)}")
+
+    # Test 2: POST request with JSON
+    print("\n=== Test 2: POST with JSON ===")
+    crawler.browser_config = crawler.browser_config.clone(
+        method="POST",
+        json={"test": "data"},
+        headers={"Content-Type": "application/json"}
+    )
+    try:
+        result = await crawler.crawl(
+            "https://httpbin.org/post",
+        )
+        print(f"Status: {result.status_code}")
+        print(f"Response: {result.html[:200]}...")
+    except Exception as e:
+        print(f"Error: {e}")
+
+    # Test 3: File handling
+    crawler.browser_config = HTTPCrawlerConfig()
+    print("\n=== Test 3: Local file handling ===")
+    # Create a tmp file with test content
+    from tempfile import NamedTemporaryFile
+    with NamedTemporaryFile(delete=False) as f:
+        f.write(b"Test content")
+        f.close()
+        result = await crawler.crawl(f"file://{f.name}")
+        print(f"File content: {result.html}")
+
+    # Test 4: Raw content
+    print("\n=== Test 4: Raw content handling ===")
+    raw_html = "raw://Raw test content"
+    result = await crawler.crawl(raw_html)
+    print(f"Raw content: {result.html}")
+
+    # Test 5: Custom hooks
+    print("\n=== Test 5: Custom hooks ===")
+    async def before_request(url, kwargs):
+        print(f"Before request to {url}")
+        kwargs['headers']['X-Custom'] = 'test'
+
+    async def after_request(response):
+        print(f"After request, status: {response.status_code}")
+
+    crawler.set_hook('before_request', before_request)
+    crawler.set_hook('after_request', after_request)
+    result = await crawler.crawl("https://example.com")
+
+    # Test 6: Error handling
+    print("\n=== Test 6: Error handling ===")
+    try:
+        await crawler.crawl("https://nonexistent.domain.test")
+    except Exception as e:
+        print(f"Expected error: {e}")
+
+    # Test 7: Redirects
+    print("\n=== Test 7: Redirect handling ===")
+    crawler.browser_config = HTTPCrawlerConfig(follow_redirects=True)
+    result = await crawler.crawl("http://httpbin.org/redirect/1")
+    print(f"Final URL: {result.redirected_url}")
+
+    # Test 8: Custom timeout
+    print("\n=== Test 8: Custom timeout ===")
+    try:
+        await crawler.crawl(
+            "https://httpbin.org/delay/5",
+            config=CrawlerRunConfig(page_timeout=2)
+        )
+    except ConnectionTimeoutError as e:
+        print(f"Expected timeout: {e}")
+
+    # Test 9: SSL verification
+    print("\n=== Test 9: SSL verification ===")
+    crawler.browser_config = HTTPCrawlerConfig(verify_ssl=False)
+    try:
+        await crawler.crawl("https://expired.badssl.com/")
+        print("Connected to invalid SSL site with verification disabled")
+    except Exception as e:
+        print(f"SSL error: {e}")
+
+    # Test 10: Large file streaming
+    print("\n=== Test 10: Large file streaming ===")
+    from tempfile import NamedTemporaryFile
+    with NamedTemporaryFile(delete=False) as f:
+        f.write(b"" + b"X" * 1024 * 1024 * 10 + b"")
+        f.close()
+        result = await crawler.crawl("file://" + f.name)
+        print(f"Large file content length: {len(result.html)}")
+        os.remove(f.name)
+
+    crawler.close()
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/general/test_llm_filter.py b/tests/general/test_llm_filter.py
new file mode 100644
index 0000000..6211c42
--- /dev/null
+++ b/tests/general/test_llm_filter.py
@@ -0,0 +1,86 @@
+import os
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai import LLMConfig
+from crawl4ai.content_filter_strategy import LLMContentFilter
+
+async def test_llm_filter():
+    # Create an HTML source that needs intelligent filtering
+    url = "https://docs.python.org/3/tutorial/classes.html"
+    
+    browser_config = BrowserConfig(
+        headless=True,
+        verbose=True
+    )
+    
+    # run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    run_config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED)
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # First get the raw HTML
+        result = await crawler.arun(url, config=run_config)
+        html = result.cleaned_html
+
+        # Initialize LLM filter with focused instruction
+        filter = LLMContentFilter(
+            llm_config=LLMConfig(provider="openai/gpt-4o",api_token=os.getenv('OPENAI_API_KEY')),
+            instruction="""
+            Focus on extracting the core educational content about Python classes.
+            Include:
+            - Key concepts and their explanations
+            - Important code examples
+            - Essential technical details
+            Exclude:
+            - Navigation elements
+            - Sidebars
+            - Footer content
+            - Version information
+            - Any non-essential UI elements
+            
+            Format the output as clean markdown with proper code blocks and headers.
+            """,
+            verbose=True
+        )
+        
+        filter = LLMContentFilter(
+            llm_config = LLMConfig(provider="openai/gpt-4o",api_token=os.getenv('OPENAI_API_KEY')),
+            chunk_token_threshold=2 ** 12 * 2, # 2048 * 2
+            instruction="""
+            Extract the main educational content while preserving its original wording and substance completely. Your task is to:
+
+            1. Maintain the exact language and terminology used in the main content
+            2. Keep all technical explanations, examples, and educational content intact
+            3. Preserve the original flow and structure of the core content
+            4. Remove only clearly irrelevant elements like:
+            - Navigation menus
+            - Advertisement sections
+            - Cookie notices
+            - Footers with site information
+            - Sidebars with external links
+            - Any UI elements that don't contribute to learning
+
+            The goal is to create a clean markdown version that reads exactly like the original article, 
+            keeping all valuable content but free from distracting elements. Imagine you're creating 
+            a perfect reading experience where nothing valuable is lost, but all noise is removed.
+            """,
+            verbose=True
+        )        
+
+        # Apply filtering
+        filtered_content = filter.filter_content(html, ignore_cache = True)
+        
+        # Show results
+        print("\nFiltered Content Length:", len(filtered_content))
+        print("\nFirst 500 chars of filtered content:")
+        if filtered_content:
+            print(filtered_content[0][:500])
+        
+        # Save on disc the markdown version
+        with open("filtered_content.md", "w", encoding="utf-8") as f:
+            f.write("\n".join(filtered_content))
+        
+        # Show token usage
+        filter.show_usage()
+
+if __name__ == "__main__":
+    asyncio.run(test_llm_filter())
\ No newline at end of file
diff --git a/tests/general/test_max_scroll.py b/tests/general/test_max_scroll.py
new file mode 100644
index 0000000..1cf8908
--- /dev/null
+++ b/tests/general/test_max_scroll.py
@@ -0,0 +1,115 @@
+"""
+Sample script to test the max_scroll_steps parameter implementation
+"""
+import asyncio
+import os
+import sys
+
+# Get the grandparent directory
+grandparent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(grandparent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+
+
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def test_max_scroll_steps():
+    """
+    Test the max_scroll_steps parameter with different configurations
+    """
+    print("🚀 Testing max_scroll_steps parameter implementation")
+    print("=" * 60)
+    
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        
+        # Test 1: Without max_scroll_steps (unlimited scrolling)
+        print("\\n📋 Test 1: Unlimited scrolling (max_scroll_steps=None)")
+        config1 = CrawlerRunConfig(
+            scan_full_page=True,
+            scroll_delay=0.1,
+            max_scroll_steps=None,  # Default behavior
+            verbose=True
+        )
+        
+        print(f"Config: scan_full_page={config1.scan_full_page}, max_scroll_steps={config1.max_scroll_steps}")
+        
+        try:
+            result1 = await crawler.arun(
+                url="https://example.com",  # Simple page for testing
+                config=config1
+            )
+            print(f"✅ Test 1 Success: Crawled {len(result1.markdown)} characters")
+        except Exception as e:
+            print(f"❌ Test 1 Failed: {e}")
+        
+        # Test 2: With limited scroll steps
+        print("\\n📋 Test 2: Limited scrolling (max_scroll_steps=3)")
+        config2 = CrawlerRunConfig(
+            scan_full_page=True,
+            scroll_delay=0.1,
+            max_scroll_steps=3,  # Limit to 3 scroll steps
+            verbose=True
+        )
+        
+        print(f"Config: scan_full_page={config2.scan_full_page}, max_scroll_steps={config2.max_scroll_steps}")
+        
+        try:
+            result2 = await crawler.arun(
+                url="https://techcrunch.com/",  # Another test page
+                config=config2
+            )
+            print(f"✅ Test 2 Success: Crawled {len(result2.markdown)} characters")
+        except Exception as e:
+            print(f"❌ Test 2 Failed: {e}")
+        
+        # Test 3: Test serialization/deserialization
+        print("\\n📋 Test 3: Configuration serialization test")
+        config3 = CrawlerRunConfig(
+            scan_full_page=True,
+            max_scroll_steps=5,
+            scroll_delay=0.2
+        )
+        
+        # Test to_dict
+        config_dict = config3.to_dict()
+        print(f"Serialized max_scroll_steps: {config_dict.get('max_scroll_steps')}")
+        
+        # Test from_kwargs
+        config4 = CrawlerRunConfig.from_kwargs({
+            'scan_full_page': True,
+            'max_scroll_steps': 7,
+            'scroll_delay': 0.3
+        })
+        print(f"Deserialized max_scroll_steps: {config4.max_scroll_steps}")
+        print("✅ Test 3 Success: Serialization works correctly")
+        
+        # Test 4: Edge case - max_scroll_steps = 0
+        print("\\n📋 Test 4: Edge case (max_scroll_steps=0)")
+        config5 = CrawlerRunConfig(
+            scan_full_page=True,
+            max_scroll_steps=0,  # Should not scroll at all
+            verbose=True
+        )
+        
+        try:
+            result5 = await crawler.arun(
+                url="https://techcrunch.com/",
+                config=config5
+            )
+            print(f"✅ Test 4 Success: No scrolling performed, crawled {len(result5.markdown)} characters")
+        except Exception as e:
+            print(f"❌ Test 4 Failed: {e}")
+    
+    print("\\n" + "=" * 60)
+    print("🎉 All tests completed!")
+    print("\\nThe max_scroll_steps parameter is working correctly:")
+    print("- None: Unlimited scrolling (default behavior)")
+    print("- Positive integer: Limits scroll steps to that number")
+    print("- 0: No scrolling performed")
+    print("- Properly serializes/deserializes in config")
+
+if __name__ == "__main__":
+    print("Starting max_scroll_steps test...")
+    asyncio.run(test_max_scroll_steps())
\ No newline at end of file
diff --git a/tests/general/test_mhtml.py b/tests/general/test_mhtml.py
new file mode 100644
index 0000000..06e0e29
--- /dev/null
+++ b/tests/general/test_mhtml.py
@@ -0,0 +1,213 @@
+# test_mhtml_capture.py
+
+import pytest
+import asyncio
+import re  # For more robust MHTML checks
+
+# Assuming these can be imported directly from the crawl4ai library
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CrawlResult
+
+# A reliable, simple static HTML page for testing
+# Using httpbin as it's designed for testing clients
+TEST_URL_SIMPLE = "https://httpbin.org/html"
+EXPECTED_CONTENT_SIMPLE = "Herman Melville - Moby-Dick"
+
+# A slightly more complex page that might involve JS (good secondary test)
+TEST_URL_JS = "https://quotes.toscrape.com/js/"
+EXPECTED_CONTENT_JS = "Quotes to Scrape" # Title of the page, which should be present in MHTML
+
+# Removed the custom event_loop fixture as pytest-asyncio provides a default one.
+
+@pytest.mark.asyncio
+async def test_mhtml_capture_when_enabled():
+    """
+    Verify that when CrawlerRunConfig has capture_mhtml=True,
+    the CrawlResult contains valid MHTML content.
+    """
+    # Create a fresh browser config and crawler instance for this test
+    browser_config = BrowserConfig(headless=True) # Use headless for testing CI/CD
+    # --- Key: Enable MHTML capture in the run config ---
+    run_config = CrawlerRunConfig(capture_mhtml=True)
+
+    # Create a fresh crawler instance
+    crawler = AsyncWebCrawler(config=browser_config)
+
+    try:
+        # Start the browser
+        await crawler.start()
+        
+        # Perform the crawl with the MHTML-enabled config
+        result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
+
+        # --- Assertions ---
+        assert result is not None, "Crawler should return a result object"
+        assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
+
+        # 1. Check if the mhtml attribute exists (will fail if CrawlResult not updated)
+        assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
+
+        # 2. Check if mhtml is populated
+        assert result.mhtml is not None, "MHTML content should be captured when enabled"
+        assert isinstance(result.mhtml, str), "MHTML content should be a string"
+        assert len(result.mhtml) > 500, "MHTML content seems too short, likely invalid" # Basic sanity check
+
+        # 3. Check for MHTML structure indicators (more robust than simple string contains)
+        # MHTML files are multipart MIME messages
+        assert re.search(r"Content-Type: multipart/related;", result.mhtml, re.IGNORECASE), \
+            "MHTML should contain 'Content-Type: multipart/related;'"
+        # Should contain a boundary definition
+        assert re.search(r"boundary=\"----MultipartBoundary", result.mhtml), \
+            "MHTML should contain a multipart boundary"
+        # Should contain the main HTML part
+        assert re.search(r"Content-Type: text/html", result.mhtml, re.IGNORECASE), \
+            "MHTML should contain a 'Content-Type: text/html' part"
+
+        # 4. Check if the *actual page content* is within the MHTML string
+        # This confirms the snapshot captured the rendered page
+        assert EXPECTED_CONTENT_SIMPLE in result.mhtml, \
+            f"Expected content '{EXPECTED_CONTENT_SIMPLE}' not found within the captured MHTML"
+
+        # 5. Ensure standard HTML is still present and correct
+        assert result.html is not None, "Standard HTML should still be present"
+        assert isinstance(result.html, str), "Standard HTML should be a string"
+        assert EXPECTED_CONTENT_SIMPLE in result.html, \
+            f"Expected content '{EXPECTED_CONTENT_SIMPLE}' not found within the standard HTML"
+
+    finally:
+        # Important: Ensure browser is completely closed even if assertions fail
+        await crawler.close()
+        # Help the garbage collector clean up
+        crawler = None
+
+
+@pytest.mark.asyncio
+async def test_mhtml_capture_when_disabled_explicitly():
+    """
+    Verify that when CrawlerRunConfig explicitly has capture_mhtml=False,
+    the CrawlResult.mhtml attribute is None.
+    """
+    # Create a fresh browser config and crawler instance for this test
+    browser_config = BrowserConfig(headless=True)
+    # --- Key: Explicitly disable MHTML capture ---
+    run_config = CrawlerRunConfig(capture_mhtml=False)
+
+    # Create a fresh crawler instance
+    crawler = AsyncWebCrawler(config=browser_config)
+
+    try:
+        # Start the browser
+        await crawler.start()
+        result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
+
+        assert result is not None
+        assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
+
+        # 1. Check attribute existence (important for TDD start)
+        assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
+
+        # 2. Check mhtml is None
+        assert result.mhtml is None, "MHTML content should be None when explicitly disabled"
+
+        # 3. Ensure standard HTML is still present
+        assert result.html is not None
+        assert EXPECTED_CONTENT_SIMPLE in result.html
+
+    finally:
+        # Important: Ensure browser is completely closed even if assertions fail
+        await crawler.close()
+        # Help the garbage collector clean up
+        crawler = None
+
+
+@pytest.mark.asyncio
+async def test_mhtml_capture_when_disabled_by_default():
+    """
+    Verify that if capture_mhtml is not specified (using its default),
+    the CrawlResult.mhtml attribute is None.
+    (This assumes the default value for capture_mhtml in CrawlerRunConfig is False)
+    """
+    # Create a fresh browser config and crawler instance for this test
+    browser_config = BrowserConfig(headless=True)
+    # --- Key: Use default run config ---
+    run_config = CrawlerRunConfig() # Do not specify capture_mhtml
+
+    # Create a fresh crawler instance
+    crawler = AsyncWebCrawler(config=browser_config)
+
+    try:
+        # Start the browser
+        await crawler.start()
+        result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
+
+        assert result is not None
+        assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
+
+        # 1. Check attribute existence
+        assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
+
+        # 2. Check mhtml is None (assuming default is False)
+        assert result.mhtml is None, "MHTML content should be None when using default config (assuming default=False)"
+
+        # 3. Ensure standard HTML is still present
+        assert result.html is not None
+        assert EXPECTED_CONTENT_SIMPLE in result.html
+
+    finally:
+        # Important: Ensure browser is completely closed even if assertions fail
+        await crawler.close()
+        # Help the garbage collector clean up
+        crawler = None
+
+# Optional: Add a test for a JS-heavy page if needed
+@pytest.mark.asyncio
+async def test_mhtml_capture_on_js_page_when_enabled():
+    """
+    Verify MHTML capture works on a page requiring JavaScript execution.
+    """
+    # Create a fresh browser config and crawler instance for this test
+    browser_config = BrowserConfig(headless=True)
+    run_config = CrawlerRunConfig(
+        capture_mhtml=True,
+        # Add a small wait or JS execution if needed for the JS page to fully render
+        # For quotes.toscrape.com/js/, it renders quickly, but a wait might be safer
+        # wait_for_timeout=2000 # Example: wait up to 2 seconds
+        js_code="await new Promise(r => setTimeout(r, 500));" # Small delay after potential load
+    )
+
+    # Create a fresh crawler instance
+    crawler = AsyncWebCrawler(config=browser_config)
+
+    try:
+        # Start the browser
+        await crawler.start()
+        result: CrawlResult = await crawler.arun(TEST_URL_JS, config=run_config)
+
+        assert result is not None
+        assert result.success is True, f"Crawling {TEST_URL_JS} should succeed. Error: {result.error_message}"
+        assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
+        assert result.mhtml is not None, "MHTML content should be captured on JS page when enabled"
+        assert isinstance(result.mhtml, str), "MHTML content should be a string"
+        assert len(result.mhtml) > 500, "MHTML content from JS page seems too short"
+
+        # Check for MHTML structure
+        assert re.search(r"Content-Type: multipart/related;", result.mhtml, re.IGNORECASE)
+        assert re.search(r"Content-Type: text/html", result.mhtml, re.IGNORECASE)
+
+        # Check for content rendered by JS within the MHTML
+        assert EXPECTED_CONTENT_JS in result.mhtml, \
+            f"Expected JS-rendered content '{EXPECTED_CONTENT_JS}' not found within the captured MHTML"
+
+        # Check standard HTML too
+        assert result.html is not None
+        assert EXPECTED_CONTENT_JS in result.html, \
+             f"Expected JS-rendered content '{EXPECTED_CONTENT_JS}' not found within the standard HTML"
+
+    finally:
+        # Important: Ensure browser is completely closed even if assertions fail
+        await crawler.close()
+        # Help the garbage collector clean up
+        crawler = None
+
+if __name__ == "__main__":
+    # Use pytest for async tests
+    pytest.main(["-xvs", __file__])
diff --git a/tests/general/test_network_console_capture.py b/tests/general/test_network_console_capture.py
new file mode 100644
index 0000000..da41ece
--- /dev/null
+++ b/tests/general/test_network_console_capture.py
@@ -0,0 +1,185 @@
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig
+import asyncio
+import aiohttp
+from aiohttp import web
+import tempfile
+import shutil
+import os, sys, time, json
+
+
+async def start_test_server():
+    app = web.Application()
+    
+    async def basic_page(request):
+        return web.Response(text="""
+        
+        
+        
+            Network Request Test
+        
+        
+            
      Test Page for Network Capture
      
+            
      This page performs network requests and console logging.
      
+            Test Image
+            
+        
+        
+        """, content_type="text/html")
+    
+    async def image(request):
+        # Return a small 1x1 transparent PNG
+        return web.Response(body=bytes.fromhex('89504E470D0A1A0A0000000D49484452000000010000000108060000001F15C4890000000D4944415478DA63FAFFFF3F030079DB00018D959DE70000000049454E44AE426082'), content_type="image/png")
+    
+    async def api_data(request):
+        return web.Response(text="sample data")
+    
+    async def api_json(request):
+        return web.json_response({"status": "success", "message": "JSON data"})
+    
+    # Register routes
+    app.router.add_get('/', basic_page)
+    app.router.add_get('/image.png', image)
+    app.router.add_get('/api/data', api_data)
+    app.router.add_get('/api/json', api_json)
+    
+    runner = web.AppRunner(app)
+    await runner.setup()
+    site = web.TCPSite(runner, 'localhost', 8080)
+    await site.start()
+    
+    return runner
+
+
+async def test_network_console_capture():
+    print("\n=== Testing Network and Console Capture ===\n")
+    
+    # Start test server
+    runner = await start_test_server()
+    try:
+        browser_config = BrowserConfig(headless=True)
+        
+        # Test with capture disabled (default)
+        print("\n1. Testing with capture disabled (default)...")
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            config = CrawlerRunConfig(
+                wait_until="networkidle",  # Wait for network to be idle
+            )
+            result = await crawler.arun(url="http://localhost:8080/", config=config)
+            
+            assert result.network_requests is None, "Network requests should be None when capture is disabled"
+            assert result.console_messages is None, "Console messages should be None when capture is disabled"
+            print("✓ Default config correctly returns None for network_requests and console_messages")
+        
+        # Test with network capture enabled
+        print("\n2. Testing with network capture enabled...")
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            config = CrawlerRunConfig(
+                wait_until="networkidle",  # Wait for network to be idle
+                capture_network_requests=True
+            )
+            result = await crawler.arun(url="http://localhost:8080/", config=config)
+            
+            assert result.network_requests is not None, "Network requests should be captured"
+            print(f"✓ Captured {len(result.network_requests)} network requests")
+            
+            # Check if we have both requests and responses
+            request_count = len([r for r in result.network_requests if r.get("event_type") == "request"])
+            response_count = len([r for r in result.network_requests if r.get("event_type") == "response"])
+            print(f"  - {request_count} requests, {response_count} responses")
+            
+            # Check if we captured specific resources
+            urls = [r.get("url") for r in result.network_requests]
+            has_image = any("/image.png" in url for url in urls)
+            has_api_data = any("/api/data" in url for url in urls)
+            has_api_json = any("/api/json" in url for url in urls)
+            
+            assert has_image, "Should have captured image request"
+            assert has_api_data, "Should have captured API data request"
+            assert has_api_json, "Should have captured API JSON request"
+            print("✓ Captured expected network requests (image, API endpoints)")
+        
+        # Test with console capture enabled
+        print("\n3. Testing with console capture enabled...")
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            config = CrawlerRunConfig(
+                wait_until="networkidle",  # Wait for network to be idle
+                capture_console_messages=True
+            )
+            result = await crawler.arun(url="http://localhost:8080/", config=config)
+            
+            assert result.console_messages is not None, "Console messages should be captured"
+            print(f"✓ Captured {len(result.console_messages)} console messages")
+            
+            # Check if we have different types of console messages
+            message_types = set(msg.get("type") for msg in result.console_messages if "type" in msg)
+            print(f"  - Message types: {', '.join(message_types)}")
+            
+            # Print all captured messages for debugging
+            print("  - Captured messages:")
+            for msg in result.console_messages:
+                print(f"    * Type: {msg.get('type', 'N/A')}, Text: {msg.get('text', 'N/A')}")
+            
+            # Look for specific messages
+            messages = [msg.get("text") for msg in result.console_messages if "text" in msg]
+            has_basic_log = any("Basic console log" in msg for msg in messages)
+            has_error_msg = any("Error message" in msg for msg in messages)
+            has_warning_msg = any("Warning message" in msg for msg in messages)
+            
+            assert has_basic_log, "Should have captured basic console.log message"
+            assert has_error_msg, "Should have captured console.error message"
+            assert has_warning_msg, "Should have captured console.warn message"
+            print("✓ Captured expected console messages (log, error, warning)")
+        
+        # Test with both captures enabled
+        print("\n4. Testing with both network and console capture enabled...")
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            config = CrawlerRunConfig(
+                wait_until="networkidle",  # Wait for network to be idle
+                capture_network_requests=True,
+                capture_console_messages=True
+            )
+            result = await crawler.arun(url="http://localhost:8080/", config=config)
+            
+            assert result.network_requests is not None, "Network requests should be captured"
+            assert result.console_messages is not None, "Console messages should be captured"
+            print(f"✓ Successfully captured both {len(result.network_requests)} network requests and {len(result.console_messages)} console messages")
+        
+    finally:
+        await runner.cleanup()
+        print("\nTest server shutdown")
+
+
+async def main():
+    try:
+        await test_network_console_capture()
+        print("\n✅ All tests passed successfully!")
+    except Exception as e:
+        print(f"\n❌ Test failed: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/general/test_persistent_context.py b/tests/general/test_persistent_context.py
new file mode 100644
index 0000000..48c01bf
--- /dev/null
+++ b/tests/general/test_persistent_context.py
@@ -0,0 +1,43 @@
+import asyncio
+import os
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, CacheMode
+
+# Simple concurrency test for persistent context page creation
+# Usage: python scripts/test_persistent_context.py
+
+URLS = [
+    # "https://example.com",
+    "https://httpbin.org/html",
+    "https://www.python.org/",
+    "https://www.rust-lang.org/",
+]
+
+async def main():
+    profile_dir = os.path.join(os.path.expanduser("~"), ".crawl4ai", "profiles", "test-persistent-profile")
+    os.makedirs(profile_dir, exist_ok=True)
+
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        use_persistent_context=True,
+        user_data_dir=profile_dir,
+        use_managed_browser=True,
+        verbose=True,
+    )
+
+    run_cfg = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        stream=False,
+        verbose=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        results = await crawler.arun_many(URLS, config=run_cfg)
+        for r in results:
+            print(r.url, r.success, len(r.markdown.raw_markdown) if r.markdown else 0)
+        # r = await crawler.arun(url=URLS[0], config=run_cfg)
+        # print(r.url, r.success, len(r.markdown.raw_markdown) if r.markdown else 0)
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/general/test_robot_parser.py b/tests/general/test_robot_parser.py
new file mode 100644
index 0000000..a2fc30f
--- /dev/null
+++ b/tests/general/test_robot_parser.py
@@ -0,0 +1,159 @@
+from crawl4ai.utils import RobotsParser
+            
+import asyncio
+import aiohttp
+from aiohttp import web
+import tempfile
+import shutil
+import os, sys, time, json
+
+
+async def test_robots_parser():
+    print("\n=== Testing RobotsParser ===\n")
+    
+    # Setup temporary directory for testing
+    temp_dir = tempfile.mkdtemp()
+    try:
+        # 1. Basic setup test
+        print("1. Testing basic initialization...")
+        parser = RobotsParser(cache_dir=temp_dir)
+        assert os.path.exists(parser.db_path), "Database file not created"
+        print("✓ Basic initialization passed")
+
+        # 2. Test common cases
+        print("\n2. Testing common cases...")
+        allowed = await parser.can_fetch("https://www.example.com", "MyBot/1.0")
+        print(f"✓ Regular website fetch: {'allowed' if allowed else 'denied'}")
+        
+        # Test caching
+        print("Testing cache...")
+        start = time.time()
+        await parser.can_fetch("https://www.example.com", "MyBot/1.0")
+        duration = time.time() - start
+        print(f"✓ Cached lookup took: {duration*1000:.2f}ms")
+        assert duration < 0.03, "Cache lookup too slow"
+
+        # 3. Edge cases
+        print("\n3. Testing edge cases...")
+        
+        # Empty URL
+        result = await parser.can_fetch("", "MyBot/1.0")
+        print(f"✓ Empty URL handled: {'allowed' if result else 'denied'}")
+        
+        # Invalid URL
+        result = await parser.can_fetch("not_a_url", "MyBot/1.0")
+        print(f"✓ Invalid URL handled: {'allowed' if result else 'denied'}")
+        
+        # URL without scheme
+        result = await parser.can_fetch("example.com/page", "MyBot/1.0")
+        print(f"✓ URL without scheme handled: {'allowed' if result else 'denied'}")
+
+        # 4. Test with local server
+        async def start_test_server():
+            app = web.Application()
+            
+            async def robots_txt(request):
+                return web.Response(text="""User-agent: *
+Disallow: /private/
+Allow: /public/
+""")
+            
+            async def malformed_robots(request):
+                return web.Response(text="<<>>")
+            
+            async def timeout_robots(request):
+                await asyncio.sleep(5)
+                return web.Response(text="Should timeout")
+            
+            async def empty_robots(request):
+                return web.Response(text="")
+            
+            async def giant_robots(request):
+                return web.Response(text="User-agent: *\nDisallow: /\n" * 10000)
+
+            # Mount all handlers at root level
+            app.router.add_get('/robots.txt', robots_txt)
+            app.router.add_get('/malformed/robots.txt', malformed_robots)
+            app.router.add_get('/timeout/robots.txt', timeout_robots)
+            app.router.add_get('/empty/robots.txt', empty_robots)
+            app.router.add_get('/giant/robots.txt', giant_robots)
+            
+            runner = web.AppRunner(app)
+            await runner.setup()
+            site = web.TCPSite(runner, 'localhost', 8080)
+            await site.start()
+            return runner
+
+        runner = await start_test_server()
+        try:
+            print("\n4. Testing robots.txt rules...")
+            base_url = "http://localhost:8080"
+            
+            # Test public access
+            result = await parser.can_fetch(f"{base_url}/public/page", "bot")
+            print(f"Public access (/public/page): {'allowed' if result else 'denied'}")
+            assert result, "Public path should be allowed"
+            
+            # Test private access
+            result = await parser.can_fetch(f"{base_url}/private/secret", "bot")
+            print(f"Private access (/private/secret): {'allowed' if result else 'denied'}")
+            assert not result, "Private path should be denied"
+            
+            # Test malformed
+            result = await parser.can_fetch("http://localhost:8080/malformed/page", "bot")
+            print(f"✓ Malformed robots.txt handled: {'allowed' if result else 'denied'}")
+            
+            # Test timeout
+            start = time.time()
+            result = await parser.can_fetch("http://localhost:8080/timeout/page", "bot")
+            duration = time.time() - start
+            print(f"✓ Timeout handled (took {duration:.2f}s): {'allowed' if result else 'denied'}")
+            assert duration < 3, "Timeout not working"
+            
+            # Test empty
+            result = await parser.can_fetch("http://localhost:8080/empty/page", "bot")
+            print(f"✓ Empty robots.txt handled: {'allowed' if result else 'denied'}")
+            
+            # Test giant file
+            start = time.time()
+            result = await parser.can_fetch("http://localhost:8080/giant/page", "bot")
+            duration = time.time() - start
+            print(f"✓ Giant robots.txt handled (took {duration:.2f}s): {'allowed' if result else 'denied'}")
+
+        finally:
+            await runner.cleanup()
+
+        # 5. Cache manipulation
+        print("\n5. Testing cache manipulation...")
+        
+        # Clear expired
+        parser.clear_expired()
+        print("✓ Clear expired entries completed")
+        
+        # Clear all
+        parser.clear_cache()
+        print("✓ Clear all cache completed")
+        
+        # Test with custom TTL
+        custom_parser = RobotsParser(cache_dir=temp_dir, cache_ttl=1)  # 1 second TTL
+        await custom_parser.can_fetch("https://www.example.com", "bot")
+        print("✓ Custom TTL fetch completed")
+        await asyncio.sleep(1.1)
+        start = time.time()
+        await custom_parser.can_fetch("https://www.example.com", "bot")
+        print(f"✓ TTL expiry working (refetched after {time.time() - start:.2f}s)")
+
+    finally:
+        # Cleanup
+        shutil.rmtree(temp_dir)
+        print("\nTest cleanup completed")
+
+async def main():
+    try:
+        await test_robots_parser()
+    except Exception as e:
+        print(f"Test failed: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/general/test_schema_builder.py b/tests/general/test_schema_builder.py
new file mode 100644
index 0000000..9296322
--- /dev/null
+++ b/tests/general/test_schema_builder.py
@@ -0,0 +1,112 @@
+# https://claude.ai/chat/c4bbe93d-fb54-44ce-92af-76b4c8086c6b
+# https://claude.ai/chat/c24a768c-d8b2-478a-acc7-d76d42a308da
+import os, sys
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai import JsonCssExtractionStrategy, JsonXPathExtractionStrategy
+from crawl4ai.utils import preprocess_html_for_schema, JsonXPathExtractionStrategy
+import json
+
+# Test HTML - A complex job board with companies, departments, and positions
+test_html = """
+
      
+    
      
+        
      
+            
+            
      Google
      
+            
      
+                10,000+ employees
+                Technology
+                Careers Page
+            
      
+        
      
+        
+        
      
+            
      
+                
      Engineering
      
+                
      
+                    
      
+                        
      Senior Software Engineer
      
+                        $150,000 - $250,000
+                        
      
+                            Mountain View, CA
+                            Full-time
+                            5+ years
+                        
      
+                        
      
+                            Python
+                            Kubernetes
+                            Machine Learning
+                        
      
+                        
      Join our core engineering team...
      
+                        
      
+                            Posted: 2024-03-15
+                            
+                        
      
+                    
      
+                    
+                
      
+            
      
+            
+            
      
+                
      Marketing
      
+                
      
+                    
      
+                        
      Growth Marketing Manager
      
+                        $120,000 - $180,000
+                        
      
+                            New York, NY
+                            Full-time
+                            3+ years
+                        
      
+                        
      
+                            SEO
+                            Analytics
+                            Content Strategy
+                        
      
+                        
      Drive our growth initiatives...
      
+                        
      
+                            Posted: 2024-03-14
+                            
+                        
      
+                    
      
+                
      
+            
      
+        
      
+    
      
+
      
+"""
+
+# Test cases
+def test_schema_generation():
+    # Test 1: No query (should extract everything)
+    print("\nTest 1: No Query (Full Schema)")
+    schema1 = JsonCssExtractionStrategy.generate_schema(test_html)
+    print(json.dumps(schema1, indent=2))
+    
+    # Test 2: Query for just basic job info
+    print("\nTest 2: Basic Job Info Query")
+    query2 = "I only need job titles, salaries, and locations"
+    schema2 = JsonCssExtractionStrategy.generate_schema(test_html, query2)
+    print(json.dumps(schema2, indent=2))
+    
+    # Test 3: Query for company and department structure
+    print("\nTest 3: Organizational Structure Query")
+    query3 = "Extract company details and department names, without position details"
+    schema3 = JsonCssExtractionStrategy.generate_schema(test_html, query3)
+    print(json.dumps(schema3, indent=2))
+    
+    # Test 4: Query for specific skills tracking
+    print("\nTest 4: Skills Analysis Query")
+    query4 = "I want to analyze required skills across all positions"
+    schema4 = JsonCssExtractionStrategy.generate_schema(test_html, query4)
+    print(json.dumps(schema4, indent=2))
+
+if __name__ == "__main__":
+    test_schema_generation()
\ No newline at end of file
diff --git a/tests/general/test_stream.py b/tests/general/test_stream.py
new file mode 100644
index 0000000..5614eb7
--- /dev/null
+++ b/tests/general/test_stream.py
@@ -0,0 +1,50 @@
+import os, sys
+# append 2 parent directories to sys.path to import crawl4ai
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+parent_parent_dir = os.path.dirname(parent_dir)
+sys.path.append(parent_parent_dir)
+
+import asyncio
+from crawl4ai import *
+
+async def test_crawler():
+    # Setup configurations
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(
+                threshold=0.48, 
+                threshold_type="fixed", 
+                min_word_threshold=0
+            )
+        ),
+    )
+
+    # Test URLs - mix of different sites
+    urls = [
+        "http://example.com",
+        "http://example.org",
+        "http://example.net",
+    ] * 10  # 15 total URLs
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        print("\n=== Testing Streaming Mode ===")
+        async for result in await crawler.arun_many(
+            urls=urls,
+            config=crawler_config.clone(stream=True),
+        ):
+            print(f"Received result for: {result.url} - Success: {result.success}")
+            
+        print("\n=== Testing Batch Mode ===")
+        results = await crawler.arun_many(
+            urls=urls,
+            config=crawler_config,
+        )
+        print(f"Received all {len(results)} results at once")
+        for result in results:
+            print(f"Batch result for: {result.url} - Success: {result.success}")
+
+if __name__ == "__main__":
+    asyncio.run(test_crawler())
\ No newline at end of file
diff --git a/tests/general/test_stream_dispatch.py b/tests/general/test_stream_dispatch.py
new file mode 100644
index 0000000..0b5d004
--- /dev/null
+++ b/tests/general/test_stream_dispatch.py
@@ -0,0 +1,39 @@
+import os, sys
+# append 2 parent directories to sys.path to import crawl4ai
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+parent_parent_dir = os.path.dirname(parent_dir)
+sys.path.append(parent_parent_dir)
+
+
+import asyncio
+from typing import List
+from crawl4ai import *
+from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher
+
+async def test_streaming():
+    browser_config = BrowserConfig(headless=True, verbose=True)
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(
+            # content_filter=PruningContentFilter(
+            #     threshold=0.48, 
+            #     threshold_type="fixed", 
+            #     min_word_threshold=0
+            # )
+        ),
+    )
+
+    urls = ["http://example.com"] * 10
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        dispatcher = MemoryAdaptiveDispatcher(
+            max_session_permit=5,
+            check_interval=0.5
+        )
+        
+        async for result in dispatcher.run_urls_stream(urls, crawler, crawler_config):
+            print(f"Got result for {result.url} - Success: {result.result.success}")
+
+if __name__ == "__main__":
+    asyncio.run(test_streaming())
\ No newline at end of file
diff --git a/tests/general/test_strip_markdown_fences.py b/tests/general/test_strip_markdown_fences.py
new file mode 100644
index 0000000..57a8c14
--- /dev/null
+++ b/tests/general/test_strip_markdown_fences.py
@@ -0,0 +1,321 @@
+"""
+Tests for _strip_markdown_fences helper and agenerate_schema() JSON parsing fix.
+
+Covers:
+- Unit tests for _strip_markdown_fences (pure logic, no API calls)
+- Real integration tests calling Anthropic/OpenAI/Groq against quotes.toscrape.com
+- Regression tests ensuring clean JSON is never corrupted
+"""
+
+import json
+import os
+import pytest
+
+from crawl4ai.extraction_strategy import (
+    _strip_markdown_fences,
+    JsonCssExtractionStrategy,
+    JsonXPathExtractionStrategy,
+)
+from crawl4ai.async_configs import LLMConfig
+
+
+# ---------------------------------------------------------------------------
+# Sample schemas for unit tests
+# ---------------------------------------------------------------------------
+
+SIMPLE_SCHEMA = {
+    "name": "Quotes",
+    "baseSelector": ".quote",
+    "fields": [
+        {"name": "text", "selector": ".text", "type": "text"},
+        {"name": "author", "selector": ".author", "type": "text"},
+    ],
+}
+
+NESTED_SCHEMA = {
+    "name": "Products",
+    "baseSelector": ".product-card",
+    "baseFields": [{"name": "id", "selector": "", "type": "attribute", "attribute": "data-id"}],
+    "fields": [
+        {"name": "title", "selector": "h2.title", "type": "text"},
+        {"name": "price", "selector": ".price", "type": "text"},
+        {"name": "description", "selector": ".desc", "type": "text"},
+        {"name": "image", "selector": "img.product-img", "type": "attribute", "attribute": "src"},
+    ],
+}
+
+TEST_URL = "https://quotes.toscrape.com/"
+
+
+# ===========================================================================
+# Unit tests for _strip_markdown_fences
+# ===========================================================================
+
+
+class TestStripMarkdownFences:
+    """Direct unit tests for the _strip_markdown_fences helper."""
+
+    def test_clean_json_passthrough(self):
+        """Clean JSON (no fences) must pass through unchanged."""
+        raw = json.dumps(SIMPLE_SCHEMA)
+        assert _strip_markdown_fences(raw) == raw
+
+    def test_json_fence(self):
+        """```json ... ``` wrapping is stripped correctly."""
+        raw = '```json\n{"key": "value"}\n```'
+        assert json.loads(_strip_markdown_fences(raw)) == {"key": "value"}
+
+    def test_bare_fence(self):
+        """``` ... ``` (no language tag) is stripped correctly."""
+        raw = '```\n{"key": "value"}\n```'
+        assert json.loads(_strip_markdown_fences(raw)) == {"key": "value"}
+
+    def test_fence_with_language_variants(self):
+        """Various language tags after ``` are stripped."""
+        for lang in ["json", "JSON", "javascript", "js", "text", "jsonc"]:
+            raw = f"```{lang}\n{{\"a\": 1}}\n```"
+            result = _strip_markdown_fences(raw)
+            assert json.loads(result) == {"a": 1}, f"Failed for language tag: {lang}"
+
+    def test_leading_trailing_whitespace(self):
+        """Whitespace around fenced content is stripped."""
+        raw = '  \n  ```json\n{"key": "value"}\n```  \n  '
+        assert json.loads(_strip_markdown_fences(raw)) == {"key": "value"}
+
+    def test_no_fences_with_whitespace(self):
+        """Plain JSON with surrounding whitespace is handled."""
+        raw = '  \n  {"key": "value"}  \n  '
+        assert json.loads(_strip_markdown_fences(raw)) == {"key": "value"}
+
+    def test_nested_code_block_in_value(self):
+        """JSON with a string value containing ``` is not corrupted."""
+        inner = {"code": "Use ```python\\nprint()\\n``` for code blocks"}
+        raw = f'```json\n{json.dumps(inner)}\n```'
+        result = _strip_markdown_fences(raw)
+        parsed = json.loads(result)
+        assert "```python" in parsed["code"]
+
+    def test_complex_schema(self):
+        """A real-world multi-field schema wrapped in fences parses correctly."""
+        raw = f"```json\n{json.dumps(NESTED_SCHEMA, indent=2)}\n```"
+        result = _strip_markdown_fences(raw)
+        assert json.loads(result) == NESTED_SCHEMA
+
+    def test_empty_string(self):
+        """Empty string returns empty string."""
+        assert _strip_markdown_fences("") == ""
+
+    def test_only_whitespace(self):
+        """Whitespace-only string returns empty string."""
+        assert _strip_markdown_fences("   \n\n  ") == ""
+
+    def test_only_fences(self):
+        """Bare fences with nothing inside return empty string."""
+        assert _strip_markdown_fences("```json\n```") == ""
+
+    def test_multiline_json(self):
+        """Multiline pretty-printed JSON inside fences."""
+        pretty = json.dumps(SIMPLE_SCHEMA, indent=4)
+        raw = f"```json\n{pretty}\n```"
+        assert json.loads(_strip_markdown_fences(raw)) == SIMPLE_SCHEMA
+
+    def test_already_clean_does_not_mutate(self):
+        """Passing already-clean JSON multiple times is idempotent."""
+        raw = json.dumps(SIMPLE_SCHEMA)
+        once = _strip_markdown_fences(raw)
+        twice = _strip_markdown_fences(once)
+        assert once == twice == raw
+
+
+# ===========================================================================
+# Real integration tests — actual LLM API calls against quotes.toscrape.com
+# ===========================================================================
+
+
+def _validate_schema(schema: dict):
+    """Validate that a generated schema has the expected structure."""
+    assert isinstance(schema, dict), f"Schema must be a dict, got {type(schema)}"
+    assert "name" in schema, "Schema must have a 'name' field"
+    assert "baseSelector" in schema, "Schema must have a 'baseSelector' field"
+    assert "fields" in schema, "Schema must have a 'fields' field"
+    assert isinstance(schema["fields"], list), "'fields' must be a list"
+    assert len(schema["fields"]) > 0, "'fields' must not be empty"
+    for field in schema["fields"]:
+        assert "name" in field, f"Each field must have a 'name': {field}"
+        assert "selector" in field, f"Each field must have a 'selector': {field}"
+        assert "type" in field, f"Each field must have a 'type': {field}"
+
+
+class TestRealAnthropicSchemaGeneration:
+    """Real API calls to Anthropic models — the exact scenario from the bug report."""
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
+        reason="CRAWL4AI_ANTHROPIC_KEY not set",
+    )
+    async def test_anthropic_haiku_css_schema(self):
+        """Reproduce the original bug: anthropic/claude-haiku-4-5 + CSS schema."""
+        schema = await JsonCssExtractionStrategy.agenerate_schema(
+            url=TEST_URL,
+            schema_type="CSS",
+            query="Extract all quotes with their text, author, and tags",
+            llm_config=LLMConfig(
+                provider="anthropic/claude-haiku-4-5",
+                api_token=os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
+            ),
+        )
+        _validate_schema(schema)
+        print(f"\n[Anthropic Haiku CSS] Generated schema: {json.dumps(schema, indent=2)}")
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
+        reason="CRAWL4AI_ANTHROPIC_KEY not set",
+    )
+    async def test_anthropic_haiku_xpath_schema(self):
+        """Anthropic haiku with XPath schema type."""
+        schema = await JsonXPathExtractionStrategy.agenerate_schema(
+            url=TEST_URL,
+            schema_type="XPATH",
+            query="Extract all quotes with their text, author, and tags",
+            llm_config=LLMConfig(
+                provider="anthropic/claude-haiku-4-5",
+                api_token=os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
+            ),
+        )
+        _validate_schema(schema)
+        print(f"\n[Anthropic Haiku XPath] Generated schema: {json.dumps(schema, indent=2)}")
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
+        reason="CRAWL4AI_ANTHROPIC_KEY not set",
+    )
+    async def test_anthropic_no_query(self):
+        """Anthropic with no query — should auto-detect schema from page structure."""
+        schema = await JsonCssExtractionStrategy.agenerate_schema(
+            url=TEST_URL,
+            schema_type="CSS",
+            llm_config=LLMConfig(
+                provider="anthropic/claude-haiku-4-5",
+                api_token=os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
+            ),
+        )
+        _validate_schema(schema)
+        print(f"\n[Anthropic Haiku no-query] Generated schema: {json.dumps(schema, indent=2)}")
+
+
+class TestRealOpenAISchemaGeneration:
+    """OpenAI models — should still work as before (regression check)."""
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.getenv("CRAWL4AI_OPENAI_KEY"),
+        reason="CRAWL4AI_OPENAI_KEY not set",
+    )
+    async def test_openai_gpt4o_mini_css_schema(self):
+        """OpenAI gpt-4o-mini with CSS — this already worked, must not regress."""
+        schema = await JsonCssExtractionStrategy.agenerate_schema(
+            url=TEST_URL,
+            schema_type="CSS",
+            query="Extract all quotes with their text, author, and tags",
+            llm_config=LLMConfig(
+                provider="openai/gpt-4o-mini",
+                api_token=os.getenv("CRAWL4AI_OPENAI_KEY"),
+            ),
+        )
+        _validate_schema(schema)
+        print(f"\n[OpenAI gpt-4o-mini CSS] Generated schema: {json.dumps(schema, indent=2)}")
+
+
+class TestRealGroqSchemaGeneration:
+    """Groq with the updated model name."""
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.getenv("CRAWL4AI_GROQ_KEY") and not os.getenv("GROQ_API_KEY"),
+        reason="No Groq API key set",
+    )
+    async def test_groq_llama33_css_schema(self):
+        """Groq with llama-3.3-70b-versatile (replacement for decommissioned 3.1)."""
+        api_key = os.getenv("CRAWL4AI_GROQ_KEY") or os.getenv("GROQ_API_KEY")
+        schema = await JsonCssExtractionStrategy.agenerate_schema(
+            url=TEST_URL,
+            schema_type="CSS",
+            query="Extract all quotes with their text, author, and tags",
+            llm_config=LLMConfig(
+                provider="groq/llama-3.3-70b-versatile",
+                api_token=api_key,
+            ),
+        )
+        _validate_schema(schema)
+        print(f"\n[Groq llama-3.3] Generated schema: {json.dumps(schema, indent=2)}")
+
+
+# ===========================================================================
+# Regression: ensure _strip_markdown_fences doesn't break valid JSON
+# ===========================================================================
+
+
+class TestRegressionNoBreakage:
+    """Ensure the fix doesn't break any currently-working JSON formats."""
+
+    @pytest.mark.parametrize(
+        "raw_json",
+        [
+            '{"simple": true}',
+            '[]',
+            '[{"a": 1}, {"a": 2}]',
+            '{"nested": {"deep": {"value": 42}}}',
+            '{"unicode": "\u3053\u3093\u306b\u3061\u306f\u4e16\u754c"}',
+            '{"special": "line1\\nline2\\ttab"}',
+            '{"url": "https://example.com/path?q=1&b=2"}',
+            json.dumps(SIMPLE_SCHEMA),
+            json.dumps(NESTED_SCHEMA),
+            json.dumps(NESTED_SCHEMA, indent=2),
+            json.dumps(NESTED_SCHEMA, indent=4),
+        ],
+        ids=[
+            "simple_object",
+            "empty_array",
+            "array_of_objects",
+            "deeply_nested",
+            "unicode_content",
+            "escape_sequences",
+            "url_in_value",
+            "simple_schema_compact",
+            "nested_schema_compact",
+            "nested_schema_indent2",
+            "nested_schema_indent4",
+        ],
+    )
+    def test_clean_json_unchanged(self, raw_json):
+        """Already-clean JSON must parse identically after stripping."""
+        original = json.loads(raw_json)
+        after_strip = json.loads(_strip_markdown_fences(raw_json))
+        assert after_strip == original
+
+    @pytest.mark.parametrize(
+        "raw_json",
+        [
+            '{"simple": true}',
+            '[]',
+            '[{"a": 1}, {"a": 2}]',
+            json.dumps(SIMPLE_SCHEMA),
+            json.dumps(NESTED_SCHEMA, indent=2),
+        ],
+        ids=[
+            "simple_object",
+            "empty_array",
+            "array_of_objects",
+            "simple_schema",
+            "nested_schema",
+        ],
+    )
+    def test_fenced_json_matches_clean(self, raw_json):
+        """Fenced version of any JSON must parse to the same value as clean."""
+        original = json.loads(raw_json)
+        fenced = f"```json\n{raw_json}\n```"
+        after_strip = json.loads(_strip_markdown_fences(fenced))
+        assert after_strip == original
diff --git a/tests/general/test_url_pattern.py b/tests/general/test_url_pattern.py
new file mode 100644
index 0000000..3aea14d
--- /dev/null
+++ b/tests/general/test_url_pattern.py
@@ -0,0 +1,85 @@
+import sys
+import os
+
+# Get the grandparent directory
+grandparent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(grandparent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+import asyncio
+from crawl4ai.deep_crawling.filters import URLPatternFilter
+
+
+def test_prefix_boundary_matching():
+    """Test that prefix patterns respect path boundaries"""
+    print("=== Testing URLPatternFilter Prefix Boundary Fix ===")
+    
+    filter_obj = URLPatternFilter(patterns=['https://langchain-ai.github.io/langgraph/*'])
+    
+    test_cases = [
+        ('https://langchain-ai.github.io/langgraph/', True),
+        ('https://langchain-ai.github.io/langgraph/concepts/', True),
+        ('https://langchain-ai.github.io/langgraph/tutorials/', True),
+        ('https://langchain-ai.github.io/langgraph?param=1', True),
+        ('https://langchain-ai.github.io/langgraph#section', True),
+        ('https://langchain-ai.github.io/langgraphjs/', False),
+        ('https://langchain-ai.github.io/langgraphjs/concepts/', False),
+        ('https://other-site.com/langgraph/', False),
+    ]
+    
+    all_passed = True
+    for url, expected in test_cases:
+        result = filter_obj.apply(url)
+        status = "PASS" if result == expected else "FAIL"
+        if result != expected:
+            all_passed = False
+        print(f"{status:4} | Expected: {expected:5} | Got: {result:5} | {url}")
+    
+    return all_passed
+
+
+def test_edge_cases():
+    """Test edge cases for path boundary matching"""
+    print("\n=== Testing Edge Cases ===")
+    
+    test_patterns = [
+        ('/api/*', [
+            ('/api/', True),
+            ('/api/v1', True),
+            ('/api?param=1', True),
+            ('/apiv2/', False),
+            ('/api_old/', False),
+        ]),
+        
+        ('*/docs/*', [
+            ('example.com/docs/', True),
+            ('example.com/docs/guide', True),
+            ('example.com/documentation/', False),
+            ('example.com/docs_old/', False),
+        ]),
+    ]
+    
+    all_passed = True
+    for pattern, test_cases in test_patterns:
+        print(f"\nPattern: {pattern}")
+        filter_obj = URLPatternFilter(patterns=[pattern])
+        
+        for url, expected in test_cases:
+            result = filter_obj.apply(url)
+            status = "PASS" if result == expected else "FAIL"
+            if result != expected:
+                all_passed = False
+            print(f"  {status:4} | Expected: {expected:5} | Got: {result:5} | {url}")
+    
+    return all_passed
+
+if __name__ == "__main__":
+    test1_passed = test_prefix_boundary_matching()
+    test2_passed = test_edge_cases()
+    
+    if test1_passed and test2_passed:
+        print("\n✅ All tests passed!")
+        sys.exit(0)
+    else:
+        print("\n❌ Some tests failed!")
+        sys.exit(1)
diff --git a/tests/general/test_url_seeder_for_only_sitemap.py b/tests/general/test_url_seeder_for_only_sitemap.py
new file mode 100644
index 0000000..63bb52d
--- /dev/null
+++ b/tests/general/test_url_seeder_for_only_sitemap.py
@@ -0,0 +1,32 @@
+import asyncio
+import pytest
+from crawl4ai import AsyncLogger, AsyncUrlSeeder, SeedingConfig
+from pathlib import Path
+import httpx
+
+
+@pytest.mark.asyncio
+async def test_sitemap_source_does_not_hit_commoncrawl():
+    config = SeedingConfig(
+        source="sitemap",
+        live_check=False,
+        extract_head=False,
+        max_urls=50,
+        verbose=True,
+        force=False
+    )
+
+    async with AsyncUrlSeeder(logger=AsyncLogger(verbose=True)) as seeder:
+        async def boom(*args, **kwargs):
+            print("DEBUG: _latest_index called")
+            raise httpx.ConnectTimeout("Simulated CommonCrawl outage")
+
+        seeder._latest_index = boom
+        try:
+            await seeder.urls("https://docs.crawl4ai.com/", config)
+            print("PASS: _latest_index was NOT called (expected after fix).")
+        except httpx.ConnectTimeout:
+            print("FAIL: _latest_index WAS called even though source='sitemap'.")
+
+if __name__ == "__main__":
+    asyncio.run(test_sitemap_source_does_not_hit_commoncrawl())
diff --git a/tests/general/tets_robot.py b/tests/general/tets_robot.py
new file mode 100644
index 0000000..9bb30bb
--- /dev/null
+++ b/tests/general/tets_robot.py
@@ -0,0 +1,62 @@
+import asyncio
+from crawl4ai import *
+
+async def test_real_websites():
+    print("\n=== Testing Real Website Robots.txt Compliance ===\n")
+    
+    browser_config = BrowserConfig(headless=True, verbose=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        
+        # Test cases with URLs
+        test_cases = [
+            # Public sites that should be allowed
+            ("https://example.com", True),  # Simple public site
+            ("https://httpbin.org/get", True),  # API endpoint
+            
+            # Sites with known strict robots.txt
+            ("https://www.facebook.com/robots.txt", False),  # Social media
+            ("https://www.google.com/search", False),  # Search pages
+            
+            # Edge cases
+            ("https://api.github.com", True),  # API service
+            ("https://raw.githubusercontent.com", True),  # Content delivery
+            
+            # Non-existent/error cases
+            ("https://thisisnotarealwebsite.com", True),  # Non-existent domain
+            ("https://localhost:12345", True),  # Invalid port
+        ]
+
+        for url, expected in test_cases:
+            print(f"\nTesting: {url}")
+            try:
+                config = CrawlerRunConfig(
+                    cache_mode=CacheMode.BYPASS,
+                    check_robots_txt=True,  # Enable robots.txt checking
+                    verbose=True
+                )
+                
+                result = await crawler.arun(url=url, config=config)
+                allowed = result.success and not result.error_message
+                
+                print(f"Expected: {'allowed' if expected else 'denied'}")
+                print(f"Actual: {'allowed' if allowed else 'denied'}")
+                print(f"Status Code: {result.status_code}")
+                if result.error_message:
+                    print(f"Error: {result.error_message}")
+                
+                # Optional: Print robots.txt content if available
+                if result.metadata and 'robots_txt' in result.metadata:
+                    print(f"Robots.txt rules:\n{result.metadata['robots_txt']}")
+                
+            except Exception as e:
+                print(f"Test failed with error: {str(e)}")
+
+async def main():
+    try:
+        await test_real_websites()
+    except Exception as e:
+        print(f"Test suite failed: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    asyncio.run(main())
\ No newline at end of file
diff --git a/tests/hub/test_simple.py b/tests/hub/test_simple.py
new file mode 100644
index 0000000..a970d68
--- /dev/null
+++ b/tests/hub/test_simple.py
@@ -0,0 +1,34 @@
+# test.py
+from crawl4ai import CrawlerHub
+import json
+
+async def amazon_example():
+    if (crawler_cls := CrawlerHub.get("amazon_product")) :
+        crawler = crawler_cls()
+        print(f"Crawler version: {crawler_cls.meta['version']}")
+        print(f"Rate limits: {crawler_cls.meta.get('rate_limit', 'Unlimited')}")
+        print(await crawler.run("https://amazon.com/test"))
+    else:
+        print("Crawler not found!")
+
+async def google_example():
+    # Get crawler dynamically
+    crawler_cls = CrawlerHub.get("google_search")
+    crawler = crawler_cls()
+
+    # Text search
+    text_results = await crawler.run(
+        query="apple inc", 
+        search_type="text",  
+        schema_cache_path="/Users/unclecode/.crawl4ai"
+    )
+    print(json.dumps(json.loads(text_results), indent=4))
+
+    # Image search
+    # image_results = await crawler.run(query="apple inc", search_type="image")
+    # print(image_results)
+
+if __name__ == "__main__":
+    import asyncio
+    # asyncio.run(amazon_example())
+    asyncio.run(google_example())
\ No newline at end of file
diff --git a/tests/integration/test_domain_mapper_e2e.py b/tests/integration/test_domain_mapper_e2e.py
new file mode 100644
index 0000000..9b248cf
--- /dev/null
+++ b/tests/integration/test_domain_mapper_e2e.py
@@ -0,0 +1,138 @@
+"""Integration tests for DomainMapper — hits real endpoints."""
+import asyncio
+import pytest
+import pytest_asyncio
+from crawl4ai import DomainMapper, DomainMapperConfig
+
+
+pytestmark = pytest.mark.network
+
+
+@pytest_asyncio.fixture
+async def mapper():
+    async with DomainMapper() as m:
+        yield m
+
+
+class TestDomainMapperE2E:
+
+    @pytest.mark.asyncio
+    async def test_scan_superdesign_dev(self, mapper):
+        """Full scan of superdesign.dev should find >=30 URLs across >=5 hosts."""
+        config = DomainMapperConfig(
+            source="sitemap+cc+crt+probe+robots+homepage",
+            extract_head=False,
+            force=True,
+            verbose=False,
+        )
+        results = await mapper.scan("superdesign.dev", config)
+        hosts = {r["host"] for r in results}
+
+        assert len(results) >= 20, f"Expected >=20 URLs, got {len(results)}"
+        assert len(hosts) >= 4, f"Expected >=4 hosts, got {len(hosts)}: {hosts}"
+        assert any("docs.superdesign.dev" == r["host"] for r in results), \
+            "docs.superdesign.dev should be discovered"
+
+    @pytest.mark.asyncio
+    async def test_scan_docs_crawl4ai(self, mapper):
+        """docs.crawl4ai.com has a known good sitemap."""
+        config = DomainMapperConfig(
+            source="sitemap",
+            extract_head=False,
+            force=True,
+            verbose=False,
+        )
+        results = await mapper.scan("docs.crawl4ai.com", config)
+        assert len(results) >= 5, f"Expected >=5 URLs from sitemap, got {len(results)}"
+        assert all(r["source"] == "sitemap" for r in results)
+
+    @pytest.mark.asyncio
+    async def test_sitemap_only_source(self, mapper):
+        """source='sitemap' should not hit CC, crt, or wayback."""
+        config = DomainMapperConfig(
+            source="sitemap",
+            extract_head=False,
+            force=True,
+            verbose=False,
+        )
+        results = await mapper.scan("superdesign.dev", config)
+        sources = {r["source"] for r in results}
+        # Should only have sitemap source
+        for s in sources:
+            for part in s.split("+"):
+                assert part == "sitemap", f"Unexpected source: {part}"
+
+    @pytest.mark.asyncio
+    async def test_crt_discovers_subdomains(self, mapper):
+        """crt source should discover subdomains for superdesign.dev."""
+        config = DomainMapperConfig(
+            source="crt+probe",
+            extract_head=False,
+            force=True,
+            verbose=False,
+        )
+        results = await mapper.scan("superdesign.dev", config)
+        hosts = {r["host"] for r in results}
+        # crt should find at least docs, app, cloud subdomains
+        assert len(hosts) >= 3, f"Expected >=3 hosts, got {len(hosts)}: {hosts}"
+
+    @pytest.mark.asyncio
+    async def test_max_urls_limit(self, mapper):
+        """max_urls should cap results."""
+        config = DomainMapperConfig(
+            source="sitemap+crt+probe",
+            extract_head=False,
+            max_urls=10,
+            force=True,
+            verbose=False,
+        )
+        results = await mapper.scan("superdesign.dev", config)
+        assert len(results) <= 10, f"Expected <=10 URLs, got {len(results)}"
+
+    @pytest.mark.asyncio
+    async def test_source_attribution(self, mapper):
+        """Each result should have a source field."""
+        config = DomainMapperConfig(
+            source="sitemap+probe",
+            extract_head=False,
+            force=True,
+            verbose=False,
+        )
+        results = await mapper.scan("docs.crawl4ai.com", config)
+        for r in results:
+            assert "source" in r
+            assert r["source"], "Source should not be empty"
+            assert "host" in r
+            assert "url" in r
+
+    @pytest.mark.asyncio
+    async def test_head_extraction(self, mapper):
+        """extract_head=True should populate head_data with titles."""
+        config = DomainMapperConfig(
+            source="sitemap",
+            extract_head=True,
+            max_urls=5,
+            force=True,
+            verbose=False,
+        )
+        results = await mapper.scan("docs.crawl4ai.com", config)
+        has_title = any(r.get("head_data", {}).get("title") for r in results)
+        assert has_title, "At least one result should have a title in head_data"
+
+    @pytest.mark.asyncio
+    async def test_crawler_integration(self):
+        """Test amap_domain() on AsyncWebCrawler works."""
+        from crawl4ai import AsyncWebCrawler
+        async with AsyncWebCrawler() as crawler:
+            results = await crawler.amap_domain(
+                "docs.crawl4ai.com",
+                DomainMapperConfig(
+                    source="sitemap",
+                    extract_head=False,
+                    force=True,
+                    verbose=False,
+                    max_urls=5,
+                ),
+            )
+            assert len(results) >= 1
+            assert all("url" in r for r in results)
diff --git a/tests/loggers/test_logger.py b/tests/loggers/test_logger.py
new file mode 100644
index 0000000..6c3a811
--- /dev/null
+++ b/tests/loggers/test_logger.py
@@ -0,0 +1,80 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, AsyncLoggerBase
+import os
+from datetime import datetime
+
+class AsyncFileLogger(AsyncLoggerBase):
+    """
+    File-only asynchronous logger that writes logs to a specified file.
+    """
+
+    def __init__(self, log_file: str):
+        """
+        Initialize the file logger.
+
+        Args:
+            log_file: File path for logging
+        """
+        self.log_file = log_file
+        os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
+
+    def _write_to_file(self, level: str, message: str, tag: str):
+        """Write a message to the log file."""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+        with open(self.log_file, "a", encoding="utf-8") as f:
+            f.write(f"[{timestamp}] [{level}] [{tag}] {message}\n")
+
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        """Log a debug message to file."""
+        self._write_to_file("DEBUG", message, tag)
+
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        """Log an info message to file."""
+        self._write_to_file("INFO", message, tag)
+
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        """Log a success message to file."""
+        self._write_to_file("SUCCESS", message, tag)
+
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        """Log a warning message to file."""
+        self._write_to_file("WARNING", message, tag)
+
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        """Log an error message to file."""
+        self._write_to_file("ERROR", message, tag)
+
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+        """Log URL fetch status to file."""
+        status = "SUCCESS" if success else "FAILED"
+        message = f"{url[:url_length]}... | Status: {status} | Time: {timing:.2f}s"
+        self._write_to_file("URL_STATUS", message, tag)
+
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+        """Log error status to file."""
+        message = f"{url[:url_length]}... | Error: {error}"
+        self._write_to_file("ERROR", message, tag)
+
+async def main():
+    browser_config = BrowserConfig(headless=True, verbose=True)
+    crawler = AsyncWebCrawler(config=browser_config, logger=AsyncFileLogger("/Users/unclecode/devs/crawl4ai/.private/tmp/crawl.log"))
+    await crawler.start()
+    
+    try:
+        crawl_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+        )
+        # Use the crawler multiple times
+        result = await crawler.arun(
+            url='https://kidocode.com/',
+            config=crawl_config
+        )
+        if result.success:
+            print("First crawl - Raw Markdown Length:", len(result.markdown.raw_markdown))
+            
+    finally:
+        # Always ensure we close the crawler
+        await crawler.close()
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/mcp/test_mcp_socket.py b/tests/mcp/test_mcp_socket.py
new file mode 100644
index 0000000..32456b3
--- /dev/null
+++ b/tests/mcp/test_mcp_socket.py
@@ -0,0 +1,119 @@
+# pip install "mcp-sdk[ws]" anyio
+import anyio, json
+from mcp.client.websocket import websocket_client
+from mcp.client.session import ClientSession
+
+async def test_list():
+    async with websocket_client("ws://localhost:8020/mcp/ws") as (r, w):
+        async with ClientSession(r, w) as s:
+            await s.initialize()
+
+            print("tools      :", [t.name for t in (await s.list_tools()).tools])
+            print("resources  :", [r.name for r in (await s.list_resources()).resources])
+            print("templates  :", [t.name for t in (await s.list_resource_templates()).resource_templates])
+
+
+async def test_crawl(s: ClientSession) -> None:
+    """Hit the @mcp_tool('crawl') endpoint."""
+    res = await s.call_tool(
+        "crawl",
+        {
+            "urls": ["https://example.com"],
+            "browser_config": {},
+            "crawler_config": {},
+        },
+    )
+    print("crawl →", json.loads(res.content[0].text))
+
+
+async def test_md(s: ClientSession) -> None:
+    """Hit the @mcp_tool('md') endpoint."""
+    res = await s.call_tool(
+        "md",
+        {
+            "url": "https://example.com",
+            "f": "fit",   # or RAW, BM25, LLM
+            "q": None,
+            "c": "0",
+        },
+    )
+    result = json.loads(res.content[0].text)
+    print("md →", result['markdown'][:100], "...")
+
+async def test_screenshot(s: ClientSession):
+    res = await s.call_tool(
+        "screenshot",
+        {
+            "url": "https://example.com",
+            "screenshot_wait_for": 1.0,
+        },
+    )
+    png_b64 = json.loads(res.content[0].text)["screenshot"]
+    print("screenshot →", png_b64[:60], "… (base64)")
+
+
+async def test_pdf(s: ClientSession):
+    res = await s.call_tool(
+        "pdf",
+        {
+            "url": "https://example.com",
+        },
+    )
+    pdf_b64 = json.loads(res.content[0].text)["pdf"]
+    print("pdf →", pdf_b64[:60], "… (base64)")
+
+async def test_execute_js(s: ClientSession):
+    # click the “More” link on Hacker News front page and wait 1 s
+    res = await s.call_tool(
+        "execute_js",
+        {
+            "url": "https://news.ycombinator.com/news",
+            "js_code": [
+                "await page.click('a.morelink')",
+                "await page.waitForTimeout(1000)",
+            ],
+        },
+    )
+    crawl_result = json.loads(res.content[0].text)
+    print("execute_js → status", crawl_result["success"], "| html len:", len(crawl_result["html"]))
+    
+async def test_html(s: ClientSession):
+    # click the “More” link on Hacker News front page and wait 1 s
+    res = await s.call_tool(
+        "html",
+        {
+            "url": "https://news.ycombinator.com/news",
+        },
+    )
+    crawl_result = json.loads(res.content[0].text)
+    print("execute_js → status", crawl_result["success"], "| html len:", len(crawl_result["html"]))    
+    
+async def test_context(s: ClientSession):
+    # click the “More” link on Hacker News front page and wait 1 s
+    res = await s.call_tool(
+        "ask",
+        {
+            "query": "I hv a question about Crawl4ai library, how to extract internal links when crawling a page?"
+        },
+    )
+    crawl_result = json.loads(res.content[0].text)
+    print("execute_js → status", crawl_result["success"], "| html len:", len(crawl_result["html"]))    
+
+
+async def main() -> None:
+    async with websocket_client("ws://localhost:11235/mcp/ws") as (r, w):
+        async with ClientSession(r, w) as s:
+            await s.initialize()                       # handshake
+            tools = (await s.list_tools()).tools
+            print("tools:", [t.name for t in tools])
+
+            # await test_list()
+            await test_crawl(s)
+            await test_md(s)
+            await test_screenshot(s)
+            await test_pdf(s)
+            await test_execute_js(s)
+            await test_html(s)
+            await test_context(s)
+
+anyio.run(main)
diff --git a/tests/mcp/test_mcp_sse.py b/tests/mcp/test_mcp_sse.py
new file mode 100644
index 0000000..d9eee55
--- /dev/null
+++ b/tests/mcp/test_mcp_sse.py
@@ -0,0 +1,11 @@
+from mcp.client.sse import sse_client
+from mcp.client.session import ClientSession
+
+async def main():
+    async with sse_client("http://127.0.0.1:8020/mcp") as (r, w):
+        async with ClientSession(r, w) as sess:
+            print(await sess.list_tools())      # now works
+            
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
diff --git a/tests/memory/README.md b/tests/memory/README.md
new file mode 100644
index 0000000..164ef09
--- /dev/null
+++ b/tests/memory/README.md
@@ -0,0 +1,315 @@
+# Crawl4AI Stress Testing and Benchmarking
+
+This directory contains tools for stress testing Crawl4AI's `arun_many` method and dispatcher system with high volumes of URLs to evaluate performance, concurrency handling, and potentially detect memory issues. It also includes a benchmarking system to track performance over time.
+
+## Quick Start
+
+```bash
+# Run a default stress test (small config) and generate a report
+# (Assumes run_all.sh is updated to call run_benchmark.py)
+./run_all.sh
+```
+*Note: `run_all.sh` might need to be updated if it directly called the old script.*
+
+## Overview
+
+The stress testing system works by:
+
+1.  Generating a local test site with heavy HTML pages (regenerated by default for each test).
+2.  Starting a local HTTP server to serve these pages.
+3.  Running Crawl4AI's `arun_many` method against this local site using the `MemoryAdaptiveDispatcher` with configurable concurrency (`max_sessions`).
+4.  Monitoring performance metrics via the `CrawlerMonitor` and optionally logging memory usage.
+5.  Optionally generating detailed benchmark reports with visualizations using `benchmark_report.py`.
+
+## Available Tools
+
+-   `test_stress_sdk.py` - Main stress testing script utilizing `arun_many` and dispatchers.
+-   `benchmark_report.py` - Report generator for comparing test results (assumes compatibility with `test_stress_sdk.py` outputs).
+-   `run_benchmark.py` - Python script with predefined test configurations that orchestrates tests using `test_stress_sdk.py`.
+-   `run_all.sh` - Simple wrapper script (may need updating).
+
+## Usage Guide
+
+### Using Predefined Configurations (Recommended)
+
+The `run_benchmark.py` script offers the easiest way to run standardized tests:
+
+```bash
+# Quick test (50 URLs, 4 max sessions)
+python run_benchmark.py quick
+
+# Medium test (500 URLs, 16 max sessions)
+python run_benchmark.py medium
+
+# Large test (1000 URLs, 32 max sessions)
+python run_benchmark.py large
+
+# Extreme test (2000 URLs, 64 max sessions)
+python run_benchmark.py extreme
+
+# Custom configuration
+python run_benchmark.py custom --urls 300 --max-sessions 24 --chunk-size 50
+
+# Run 'small' test in streaming mode
+python run_benchmark.py small --stream
+
+# Override max_sessions for the 'medium' config
+python run_benchmark.py medium --max-sessions 20
+
+# Skip benchmark report generation after the test
+python run_benchmark.py small --no-report
+
+# Clean up reports and site files before running
+python run_benchmark.py medium --clean
+```
+
+#### `run_benchmark.py` Parameters
+
+| Parameter            | Default         | Description                                                                 |
+| -------------------- | --------------- | --------------------------------------------------------------------------- |
+| `config`             | *required*      | Test configuration: `quick`, `small`, `medium`, `large`, `extreme`, `custom`|
+| `--urls`             | config-specific | Number of URLs (required for `custom`)                                      |
+| `--max-sessions`     | config-specific | Max concurrent sessions managed by dispatcher (required for `custom`)         |
+| `--chunk-size`       | config-specific | URLs per batch for non-stream logging (required for `custom`)               |
+| `--stream`           | False           | Enable streaming results (disables batch logging)                           |
+| `--monitor-mode`     | DETAILED        | `DETAILED` or `AGGREGATED` display for the live monitor                     |
+| `--use-rate-limiter` | False           | Enable basic rate limiter in the dispatcher                                 |
+| `--port`             | 8000            | HTTP server port                                                            |
+| `--no-report`        | False           | Skip generating comparison report via `benchmark_report.py`                 |
+| `--clean`            | False           | Clean up reports and site files before running                              |
+| `--keep-server-alive`| False           | Keep local HTTP server running after test                                   |
+| `--use-existing-site`| False           | Use existing site on specified port (no local server start/site gen)        |
+| `--skip-generation`  | False           | Use existing site files but start local server                              |
+| `--keep-site`        | False           | Keep generated site files after test                                        |
+
+#### Predefined Configurations
+
+| Configuration | URLs   | Max Sessions | Chunk Size | Description                      |
+| ------------- | ------ | ------------ | ---------- | -------------------------------- |
+| `quick`       | 50     | 4            | 10         | Quick test for basic validation  |
+| `small`       | 100    | 8            | 20         | Small test for routine checks    |
+| `medium`      | 500    | 16           | 50         | Medium test for thorough checks  |
+| `large`       | 1000   | 32           | 100        | Large test for stress testing    |
+| `extreme`     | 2000   | 64           | 200        | Extreme test for limit testing   |
+
+### Direct Usage of `test_stress_sdk.py`
+
+For fine-grained control or debugging, you can run the stress test script directly:
+
+```bash
+# Test with 200 URLs and 32 max concurrent sessions
+python test_stress_sdk.py --urls 200 --max-sessions 32 --chunk-size 40
+
+# Clean up previous test data first
+python test_stress_sdk.py --clean-reports --clean-site --urls 100 --max-sessions 16 --chunk-size 20
+
+# Change the HTTP server port and use aggregated monitor
+python test_stress_sdk.py --port 8088 --urls 100 --max-sessions 16 --monitor-mode AGGREGATED
+
+# Enable streaming mode and use rate limiting
+python test_stress_sdk.py --urls 50 --max-sessions 8 --stream --use-rate-limiter
+
+# Change report output location
+python test_stress_sdk.py --report-path custom_reports --urls 100 --max-sessions 16
+```
+
+#### `test_stress_sdk.py` Parameters
+
+| Parameter            | Default    | Description                                                          |
+| -------------------- | ---------- | -------------------------------------------------------------------- |
+| `--urls`             | 100        | Number of URLs to test                                               |
+| `--max-sessions`     | 16         | Maximum concurrent crawling sessions managed by the dispatcher       |
+| `--chunk-size`       | 10         | Number of URLs per batch (relevant for non-stream logging)           |
+| `--stream`           | False      | Enable streaming results (disables batch logging)                    |
+| `--monitor-mode`     | DETAILED   | `DETAILED` or `AGGREGATED` display for the live `CrawlerMonitor`     |
+| `--use-rate-limiter` | False      | Enable a basic `RateLimiter` within the dispatcher                   |
+| `--site-path`        | "test_site"| Path to store/use the generated test site                            |
+| `--port`             | 8000       | Port for the local HTTP server                                       |
+| `--report-path`      | "reports"  | Path to save test result summary (JSON) and memory samples (CSV)   |
+| `--skip-generation`  | False      | Use existing test site files but still start local server            |
+| `--use-existing-site`| False      | Use existing site on specified port (no local server/site gen)     |
+| `--keep-server-alive`| False      | Keep local HTTP server running after test completion                 |
+| `--keep-site`        | False      | Keep the generated test site files after test completion             |
+| `--clean-reports`    | False      | Clean up report directory before running                             |
+| `--clean-site`       | False      | Clean up site directory before/after running (see script logic)    |
+
+### Generating Reports Only
+
+If you only want to generate a benchmark report from existing test results (assuming `benchmark_report.py` is compatible):
+
+```bash
+# Generate a report from existing test results in ./reports/
+python benchmark_report.py
+
+# Limit to the most recent 5 test results
+python benchmark_report.py --limit 5
+
+# Specify a custom source directory for test results
+python benchmark_report.py --reports-dir alternate_results
+```
+
+#### `benchmark_report.py` Parameters (Assumed)
+
+| Parameter       | Default              | Description                                                 |
+| --------------- | -------------------- | ----------------------------------------------------------- |
+| `--reports-dir` | "reports"            | Directory containing `test_stress_sdk.py` result files      |
+| `--output-dir`  | "benchmark_reports"  | Directory to save generated HTML reports and charts         |
+| `--limit`       | None (all results)   | Limit comparison to N most recent test results              |
+| `--output-file` | Auto-generated       | Custom output filename for the HTML report                  |
+
+## Understanding the Test Output
+
+### Real-time Progress Display (`CrawlerMonitor`)
+
+When running `test_stress_sdk.py`, the `CrawlerMonitor` provides a live view of the crawling process managed by the dispatcher.
+
+-   **DETAILED Mode (Default):** Shows individual task status (Queued, Active, Completed, Failed), timings, memory usage per task (if `psutil` is available), overall queue statistics, and memory pressure status (if `psutil` available).
+-   **AGGREGATED Mode:** Shows summary counts (Queued, Active, Completed, Failed), overall progress percentage, estimated time remaining, average URLs/sec, and memory pressure status.
+
+### Batch Log Output (Non-Streaming Mode Only)
+
+If running `test_stress_sdk.py` **without** the `--stream` flag, you will *also* see per-batch summary lines printed to the console *after* the monitor display, once each chunk of URLs finishes processing:
+
+```
+ Batch | Progress | Start Mem | End Mem   | URLs/sec | Success/Fail | Time (s) | Status
+───────────────────────────────────────────────────────────────────────────────────────────
+ 1     |  10.0%   |  50.1 MB  |  55.3 MB  |    23.8    |    10/0      |     0.42   | Success
+ 2     |  20.0%   |  55.3 MB  |  60.1 MB  |    24.1    |    10/0      |     0.41   | Success
+ ...
+```
+
+This display provides chunk-specific metrics:
+-   **Batch**: The batch number being reported.
+-   **Progress**: Overall percentage of total URLs processed *after* this batch.
+-   **Start Mem / End Mem**: Memory usage before and after processing this batch (if tracked).
+-   **URLs/sec**: Processing speed *for this specific batch*.
+-   **Success/Fail**: Number of successful and failed URLs *in this batch*.
+-   **Time (s)**: Wall-clock time taken to process *this batch*.
+-   **Status**: Color-coded status for the batch outcome.
+
+### Summary Output
+
+After test completion, a final summary is displayed:
+
+```
+================================================================================
+Test Completed
+================================================================================
+Test ID: 20250418_103015
+Configuration: 100 URLs, 16 max sessions, Chunk: 10, Stream: False, Monitor: DETAILED
+Results: 100 successful, 0 failed (100 processed, 100.0% success)
+Performance: 5.85 seconds total, 17.09 URLs/second avg
+Memory Usage: Start: 50.1 MB, End: 75.3 MB, Max: 78.1 MB, Growth: 25.2 MB
+Results summary saved to reports/test_summary_20250418_103015.json
+```
+
+### HTML Report Structure (Generated by `benchmark_report.py`)
+
+(This section remains the same, assuming `benchmark_report.py` generates these)
+The benchmark report contains several sections:
+1.  **Summary**: Overview of the latest test results and trends
+2.  **Performance Comparison**: Charts showing throughput across tests
+3.  **Memory Usage**: Detailed memory usage graphs for each test
+4.  **Detailed Results**: Tabular data of all test metrics
+5.  **Conclusion**: Automated analysis of performance and memory patterns
+
+### Memory Metrics
+
+(This section remains conceptually the same)
+Memory growth is the key metric for detecting leaks...
+
+### Performance Metrics
+
+(This section remains conceptually the same, though "URLs per Worker" is less relevant - focus on overall URLs/sec)
+Key performance indicators include:
+-   **URLs per Second**: Higher is better (throughput)
+-   **Success Rate**: Should be 100% in normal conditions
+-   **Total Processing Time**: Lower is better
+-   **Dispatcher Efficiency**: Observe queue lengths and wait times in the monitor (Detailed mode)
+
+### Raw Data Files
+
+Raw data is saved in the `--report-path` directory (default `./reports/`):
+
+-   **JSON files** (`test_summary_*.json`): Contains the final summary for each test run.
+-   **CSV files** (`memory_samples_*.csv`): Contains time-series memory samples taken during the test run.
+
+Example of reading raw data:
+```python
+import json
+import pandas as pd
+
+# Load test summary
+test_id = "20250418_103015" # Example ID
+with open(f'reports/test_summary_{test_id}.json', 'r') as f:
+    results = json.load(f)
+
+# Load memory samples
+memory_df = pd.read_csv(f'reports/memory_samples_{test_id}.csv')
+
+# Analyze memory_df (e.g., calculate growth, plot)
+if not memory_df['memory_info_mb'].isnull().all():
+    growth = memory_df['memory_info_mb'].iloc[-1] - memory_df['memory_info_mb'].iloc[0]
+    print(f"Total Memory Growth: {growth:.1f} MB")
+else:
+    print("No valid memory samples found.")
+
+print(f"Avg URLs/sec: {results['urls_processed'] / results['total_time_seconds']:.2f}")
+```
+
+## Visualization Dependencies
+
+(This section remains the same)
+For full visualization capabilities in the HTML reports generated by `benchmark_report.py`, install additional dependencies...
+
+## Directory Structure
+
+```
+benchmarking/          # Or your top-level directory name
+├── benchmark_reports/ # Generated HTML reports (by benchmark_report.py)
+├── reports/           # Raw test result data (from test_stress_sdk.py)
+├── test_site/         # Generated test content (temporary)
+├── benchmark_report.py# Report generator
+├── run_benchmark.py   # Test runner with predefined configs
+├── test_stress_sdk.py # Main stress test implementation using arun_many
+└── run_all.sh         # Simple wrapper script (may need updates)
+#└── requirements.txt   # Optional: Visualization dependencies for benchmark_report.py
+```
+
+## Cleanup
+
+To clean up after testing:
+
+```bash
+# Remove the test site content (if not using --keep-site)
+rm -rf test_site
+
+# Remove all raw reports and generated benchmark reports
+rm -rf reports benchmark_reports
+
+# Or use the --clean flag with run_benchmark.py
+python run_benchmark.py medium --clean
+```
+
+## Use in CI/CD
+
+(This section remains conceptually the same, just update script names)
+These tests can be integrated into CI/CD pipelines:
+```bash
+# Example CI script
+python run_benchmark.py medium --no-report # Run test without interactive report gen
+# Check exit code
+if [ $? -ne 0 ]; then echo "Stress test failed!"; exit 1; fi
+# Optionally, run report generator and check its output/metrics
+# python benchmark_report.py
+# check_report_metrics.py reports/test_summary_*.json || exit 1
+exit 0
+```
+
+## Troubleshooting
+
+-   **HTTP Server Port Conflict**: Use `--port` with `run_benchmark.py` or `test_stress_sdk.py`.
+-   **Memory Tracking Issues**: The `SimpleMemoryTracker` uses platform commands (`ps`, `/proc`, `tasklist`). Ensure these are available and the script has permission. If it consistently fails, memory reporting will be limited.
+-   **Visualization Missing**: Related to `benchmark_report.py` and its dependencies.
+-   **Site Generation Issues**: Check permissions for creating `./test_site/`. Use `--skip-generation` if you want to manage the site manually.
+-   **Testing Against External Site**: Ensure the external site is running and use `--use-existing-site --port `.
diff --git a/tests/memory/benchmark_report.py b/tests/memory/benchmark_report.py
new file mode 100755
index 0000000..a634f99
--- /dev/null
+++ b/tests/memory/benchmark_report.py
@@ -0,0 +1,887 @@
+#!/usr/bin/env python3
+"""
+Benchmark reporting tool for Crawl4AI stress tests.
+Generates visual reports and comparisons between test runs.
+"""
+
+import os
+import json
+import glob
+import argparse
+import sys
+from datetime import datetime
+from pathlib import Path
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+
+# Initialize rich console
+console = Console()
+
+# Try to import optional visualization dependencies
+VISUALIZATION_AVAILABLE = True
+try:
+    import pandas as pd
+    import matplotlib.pyplot as plt
+    import matplotlib as mpl
+    import numpy as np
+    import seaborn as sns
+except ImportError:
+    VISUALIZATION_AVAILABLE = False
+    console.print("[yellow]Warning: Visualization dependencies not found. Install with:[/yellow]")
+    console.print("[yellow]pip install pandas matplotlib seaborn[/yellow]")
+    console.print("[yellow]Only text-based reports will be generated.[/yellow]")
+
+# Configure plotting if available
+if VISUALIZATION_AVAILABLE:
+    # Set plot style for dark theme
+    plt.style.use('dark_background')
+    sns.set_theme(style="darkgrid")
+    
+    # Custom color palette based on Nord theme
+    nord_palette = ["#88c0d0", "#81a1c1", "#a3be8c", "#ebcb8b", "#bf616a", "#b48ead", "#5e81ac"]
+    sns.set_palette(nord_palette)
+
+class BenchmarkReporter:
+    """Generates visual reports and comparisons for Crawl4AI stress tests."""
+    
+    def __init__(self, reports_dir="reports", output_dir="benchmark_reports"):
+        """Initialize the benchmark reporter.
+        
+        Args:
+            reports_dir: Directory containing test result files
+            output_dir: Directory to save generated reports
+        """
+        self.reports_dir = Path(reports_dir)
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Configure matplotlib if available
+        if VISUALIZATION_AVAILABLE:
+            # Ensure the matplotlib backend works in headless environments
+            mpl.use('Agg')
+            
+            # Set up styling for plots with dark theme
+            mpl.rcParams['figure.figsize'] = (12, 8)
+            mpl.rcParams['font.size'] = 12
+            mpl.rcParams['axes.labelsize'] = 14
+            mpl.rcParams['axes.titlesize'] = 16
+            mpl.rcParams['xtick.labelsize'] = 12
+            mpl.rcParams['ytick.labelsize'] = 12
+            mpl.rcParams['legend.fontsize'] = 12
+            mpl.rcParams['figure.facecolor'] = '#1e1e1e'
+            mpl.rcParams['axes.facecolor'] = '#2e3440'
+            mpl.rcParams['savefig.facecolor'] = '#1e1e1e'
+            mpl.rcParams['text.color'] = '#e0e0e0'
+            mpl.rcParams['axes.labelcolor'] = '#e0e0e0'
+            mpl.rcParams['xtick.color'] = '#e0e0e0'
+            mpl.rcParams['ytick.color'] = '#e0e0e0'
+            mpl.rcParams['grid.color'] = '#444444'
+            mpl.rcParams['figure.edgecolor'] = '#444444'
+        
+    def load_test_results(self, limit=None):
+        """Load all test results from the reports directory.
+        
+        Args:
+            limit: Optional limit on number of most recent tests to load
+            
+        Returns:
+            Dictionary mapping test IDs to result data
+        """
+        result_files = glob.glob(str(self.reports_dir / "test_results_*.json"))
+        
+        # Sort files by modification time (newest first)
+        result_files.sort(key=os.path.getmtime, reverse=True)
+        
+        if limit:
+            result_files = result_files[:limit]
+        
+        results = {}
+        for file_path in result_files:
+            try:
+                with open(file_path, 'r') as f:
+                    data = json.load(f)
+                    test_id = data.get('test_id')
+                    if test_id:
+                        results[test_id] = data
+                        
+                        # Try to load the corresponding memory samples
+                        csv_path = self.reports_dir / f"memory_samples_{test_id}.csv"
+                        if csv_path.exists():
+                            try:
+                                memory_df = pd.read_csv(csv_path)
+                                results[test_id]['memory_samples'] = memory_df
+                            except Exception as e:
+                                console.print(f"[yellow]Warning: Could not load memory samples for {test_id}: {e}[/yellow]")
+            except Exception as e:
+                console.print(f"[red]Error loading {file_path}: {e}[/red]")
+        
+        console.print(f"Loaded {len(results)} test results")
+        return results
+    
+    def generate_summary_table(self, results):
+        """Generate a summary table of test results.
+        
+        Args:
+            results: Dictionary mapping test IDs to result data
+            
+        Returns:
+            Rich Table object
+        """
+        table = Table(title="Crawl4AI Stress Test Summary", show_header=True)
+        
+        # Define columns
+        table.add_column("Test ID", style="cyan")
+        table.add_column("Date", style="bright_green")
+        table.add_column("URLs", justify="right")
+        table.add_column("Workers", justify="right")
+        table.add_column("Success %", justify="right")
+        table.add_column("Time (s)", justify="right")
+        table.add_column("Mem Growth", justify="right")
+        table.add_column("URLs/sec", justify="right")
+        
+        # Add rows
+        for test_id, data in sorted(results.items(), key=lambda x: x[0], reverse=True):
+            # Parse timestamp from test_id
+            try:
+                date_str = datetime.strptime(test_id, "%Y%m%d_%H%M%S").strftime("%Y-%m-%d %H:%M")
+            except:
+                date_str = "Unknown"
+            
+            # Calculate success percentage
+            total_urls = data.get('url_count', 0)
+            successful = data.get('successful_urls', 0)
+            success_pct = (successful / total_urls * 100) if total_urls > 0 else 0
+            
+            # Calculate memory growth if available
+            mem_growth = "N/A"
+            if 'memory_samples' in data:
+                samples = data['memory_samples']
+                if len(samples) >= 2:
+                    # Try to extract numeric values from memory_info strings
+                    try:
+                        first_mem = float(samples.iloc[0]['memory_info'].split()[0])
+                        last_mem = float(samples.iloc[-1]['memory_info'].split()[0])
+                        mem_growth = f"{last_mem - first_mem:.1f} MB"
+                    except:
+                        pass
+            
+            # Calculate URLs per second
+            time_taken = data.get('total_time_seconds', 0)
+            urls_per_sec = total_urls / time_taken if time_taken > 0 else 0
+            
+            table.add_row(
+                test_id,
+                date_str,
+                str(total_urls),
+                str(data.get('workers', 'N/A')),
+                f"{success_pct:.1f}%",
+                f"{data.get('total_time_seconds', 0):.2f}",
+                mem_growth,
+                f"{urls_per_sec:.1f}"
+            )
+        
+        return table
+    
+    def generate_performance_chart(self, results, output_file=None):
+        """Generate a performance comparison chart.
+        
+        Args:
+            results: Dictionary mapping test IDs to result data
+            output_file: File path to save the chart
+            
+        Returns:
+            Path to the saved chart file or None if visualization is not available
+        """
+        if not VISUALIZATION_AVAILABLE:
+            console.print("[yellow]Skipping performance chart - visualization dependencies not available[/yellow]")
+            return None
+            
+        # Extract relevant data
+        data = []
+        for test_id, result in results.items():
+            urls = result.get('url_count', 0)
+            workers = result.get('workers', 0)
+            time_taken = result.get('total_time_seconds', 0)
+            urls_per_sec = urls / time_taken if time_taken > 0 else 0
+            
+            # Parse timestamp from test_id for sorting
+            try:
+                timestamp = datetime.strptime(test_id, "%Y%m%d_%H%M%S")
+                data.append({
+                    'test_id': test_id,
+                    'timestamp': timestamp,
+                    'urls': urls,
+                    'workers': workers,
+                    'time_seconds': time_taken,
+                    'urls_per_sec': urls_per_sec
+                })
+            except:
+                console.print(f"[yellow]Warning: Could not parse timestamp from {test_id}[/yellow]")
+        
+        if not data:
+            console.print("[yellow]No valid data for performance chart[/yellow]")
+            return None
+        
+        # Convert to DataFrame and sort by timestamp
+        df = pd.DataFrame(data)
+        df = df.sort_values('timestamp')
+        
+        # Create the plot
+        fig, ax1 = plt.subplots(figsize=(12, 6))
+        
+        # Plot URLs per second as bars with properly set x-axis
+        x_pos = range(len(df['test_id']))
+        bars = ax1.bar(x_pos, df['urls_per_sec'], color='#88c0d0', alpha=0.8)
+        ax1.set_ylabel('URLs per Second', color='#88c0d0')
+        ax1.tick_params(axis='y', labelcolor='#88c0d0')
+        
+        # Properly set x-axis labels
+        ax1.set_xticks(x_pos)
+        ax1.set_xticklabels(df['test_id'].tolist(), rotation=45, ha='right')
+        
+        # Add worker count as text on each bar
+        for i, bar in enumerate(bars):
+            height = bar.get_height()
+            workers = df.iloc[i]['workers']
+            ax1.text(i, height + 0.1,
+                    f'W: {workers}', ha='center', va='bottom', fontsize=9, color='#e0e0e0')
+        
+        # Add a second y-axis for total URLs
+        ax2 = ax1.twinx()
+        ax2.plot(x_pos, df['urls'], '-', color='#bf616a', alpha=0.8, markersize=6, marker='o')
+        ax2.set_ylabel('Total URLs', color='#bf616a')
+        ax2.tick_params(axis='y', labelcolor='#bf616a')
+        
+        # Set title and layout
+        plt.title('Crawl4AI Performance Benchmarks')
+        plt.tight_layout()
+        
+        # Save the figure
+        if output_file is None:
+            output_file = self.output_dir / "performance_comparison.png"
+        plt.savefig(output_file, dpi=100, bbox_inches='tight')
+        plt.close()
+        
+        return output_file
+    
+    def generate_memory_charts(self, results, output_prefix=None):
+        """Generate memory usage charts for each test.
+        
+        Args:
+            results: Dictionary mapping test IDs to result data
+            output_prefix: Prefix for output file names
+            
+        Returns:
+            List of paths to the saved chart files
+        """
+        if not VISUALIZATION_AVAILABLE:
+            console.print("[yellow]Skipping memory charts - visualization dependencies not available[/yellow]")
+            return []
+            
+        output_files = []
+        
+        for test_id, result in results.items():
+            if 'memory_samples' not in result:
+                continue
+            
+            memory_df = result['memory_samples']
+            
+            # Check if we have enough data points
+            if len(memory_df) < 2:
+                continue
+            
+            # Try to extract numeric values from memory_info strings
+            try:
+                memory_values = []
+                for mem_str in memory_df['memory_info']:
+                    # Extract the number from strings like "142.8 MB"
+                    value = float(mem_str.split()[0])
+                    memory_values.append(value)
+                
+                memory_df['memory_mb'] = memory_values
+            except Exception as e:
+                console.print(f"[yellow]Could not parse memory values for {test_id}: {e}[/yellow]")
+                continue
+            
+            # Create the plot
+            plt.figure(figsize=(10, 6))
+            
+            # Plot memory usage over time
+            plt.plot(memory_df['elapsed_seconds'], memory_df['memory_mb'], 
+                     color='#88c0d0', marker='o', linewidth=2, markersize=4)
+            
+            # Add annotations for chunk processing
+            chunk_size = result.get('chunk_size', 0)
+            url_count = result.get('url_count', 0)
+            if chunk_size > 0 and url_count > 0:
+                # Estimate chunk processing times
+                num_chunks = (url_count + chunk_size - 1) // chunk_size  # Ceiling division
+                total_time = result.get('total_time_seconds', memory_df['elapsed_seconds'].max())
+                chunk_times = np.linspace(0, total_time, num_chunks + 1)[1:]
+                
+                for i, time_point in enumerate(chunk_times):
+                    if time_point <= memory_df['elapsed_seconds'].max():
+                        plt.axvline(x=time_point, color='#4c566a', linestyle='--', alpha=0.6)
+                        plt.text(time_point, memory_df['memory_mb'].min(), f'Chunk {i+1}', 
+                                rotation=90, verticalalignment='bottom', fontsize=8, color='#e0e0e0')
+            
+            # Set labels and title
+            plt.xlabel('Elapsed Time (seconds)', color='#e0e0e0')
+            plt.ylabel('Memory Usage (MB)', color='#e0e0e0')
+            plt.title(f'Memory Usage During Test {test_id}\n({url_count} URLs, {result.get("workers", "?")} Workers)', 
+                      color='#e0e0e0')
+            
+            # Add grid and set y-axis to start from zero
+            plt.grid(True, alpha=0.3, color='#4c566a')
+            
+            # Add test metadata as text
+            info_text = (
+                f"URLs: {url_count}\n"
+                f"Workers: {result.get('workers', 'N/A')}\n"
+                f"Chunk Size: {result.get('chunk_size', 'N/A')}\n"
+                f"Total Time: {result.get('total_time_seconds', 0):.2f}s\n"
+            )
+            
+            # Calculate memory growth
+            if len(memory_df) >= 2:
+                first_mem = memory_df.iloc[0]['memory_mb']
+                last_mem = memory_df.iloc[-1]['memory_mb']
+                growth = last_mem - first_mem
+                growth_rate = growth / result.get('total_time_seconds', 1)
+                
+                info_text += f"Memory Growth: {growth:.1f} MB\n"
+                info_text += f"Growth Rate: {growth_rate:.2f} MB/s"
+            
+            plt.figtext(0.02, 0.02, info_text, fontsize=9, color='#e0e0e0',
+                       bbox=dict(facecolor='#3b4252', alpha=0.8, edgecolor='#4c566a'))
+            
+            # Save the figure
+            if output_prefix is None:
+                output_file = self.output_dir / f"memory_chart_{test_id}.png"
+            else:
+                output_file = Path(f"{output_prefix}_memory_{test_id}.png")
+                
+            plt.tight_layout()
+            plt.savefig(output_file, dpi=100, bbox_inches='tight')
+            plt.close()
+            
+            output_files.append(output_file)
+        
+        return output_files
+    
+    def generate_comparison_report(self, results, title=None, output_file=None):
+        """Generate a comprehensive comparison report of multiple test runs.
+        
+        Args:
+            results: Dictionary mapping test IDs to result data
+            title: Optional title for the report
+            output_file: File path to save the report
+            
+        Returns:
+            Path to the saved report file
+        """
+        if not results:
+            console.print("[yellow]No results to generate comparison report[/yellow]")
+            return None
+        
+        if output_file is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            output_file = self.output_dir / f"comparison_report_{timestamp}.html"
+        
+        # Create data for the report
+        rows = []
+        for test_id, data in results.items():
+            # Calculate metrics
+            urls = data.get('url_count', 0)
+            workers = data.get('workers', 0)
+            successful = data.get('successful_urls', 0)
+            failed = data.get('failed_urls', 0)
+            time_seconds = data.get('total_time_seconds', 0)
+            
+            # Calculate additional metrics
+            success_rate = (successful / urls) * 100 if urls > 0 else 0
+            urls_per_second = urls / time_seconds if time_seconds > 0 else 0
+            urls_per_worker = urls / workers if workers > 0 else 0
+            
+            # Calculate memory growth if available
+            mem_start = None
+            mem_end = None
+            mem_growth = None
+            if 'memory_samples' in data:
+                samples = data['memory_samples']
+                if len(samples) >= 2:
+                    try:
+                        first_mem = float(samples.iloc[0]['memory_info'].split()[0])
+                        last_mem = float(samples.iloc[-1]['memory_info'].split()[0])
+                        mem_start = first_mem
+                        mem_end = last_mem
+                        mem_growth = last_mem - first_mem
+                    except:
+                        pass
+            
+            # Parse timestamp from test_id
+            try:
+                timestamp = datetime.strptime(test_id, "%Y%m%d_%H%M%S")
+            except:
+                timestamp = None
+            
+            rows.append({
+                'test_id': test_id,
+                'timestamp': timestamp,
+                'date': timestamp.strftime("%Y-%m-%d %H:%M:%S") if timestamp else "Unknown",
+                'urls': urls,
+                'workers': workers,
+                'chunk_size': data.get('chunk_size', 0),
+                'successful': successful,
+                'failed': failed,
+                'success_rate': success_rate,
+                'time_seconds': time_seconds,
+                'urls_per_second': urls_per_second,
+                'urls_per_worker': urls_per_worker,
+                'memory_start': mem_start,
+                'memory_end': mem_end,
+                'memory_growth': mem_growth
+            })
+        
+        # Sort data by timestamp if possible
+        if VISUALIZATION_AVAILABLE:
+            # Convert to DataFrame and sort by timestamp
+            df = pd.DataFrame(rows)
+            if 'timestamp' in df.columns and not df['timestamp'].isna().all():
+                df = df.sort_values('timestamp', ascending=False)
+        else:
+            # Simple sorting without pandas
+            rows.sort(key=lambda x: x.get('timestamp', datetime.now()), reverse=True)
+            df = None
+        
+        # Generate HTML report
+        html = []
+        html.append('')
+        html.append('')
+        html.append('')
+        html.append('')
+        html.append('')
+        html.append(f'{title or "Crawl4AI Benchmark Comparison"}')
+        html.append('')
+        html.append('')
+        html.append('')
+        
+        # Header
+        html.append(f'
      {title or "Crawl4AI Benchmark Comparison"}
      ')
+        html.append(f'
      Report generated on {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
      ')
+        
+        # Summary section
+        html.append('
      ')
+        html.append('
      Summary
      ')
+        html.append('
      This report compares the performance of Crawl4AI across multiple test runs.
      ')
+        
+        # Summary metrics
+        data_available = (VISUALIZATION_AVAILABLE and df is not None and not df.empty) or (not VISUALIZATION_AVAILABLE and len(rows) > 0)
+        if data_available:
+            # Get the latest test data
+            if VISUALIZATION_AVAILABLE and df is not None and not df.empty:
+                latest_test = df.iloc[0]
+                latest_id = latest_test['test_id']
+            else:
+                latest_test = rows[0]  # First row (already sorted by timestamp)
+                latest_id = latest_test['test_id']
+            
+            html.append('
      Latest Test Results
      ')
+            html.append('
        ')
+            html.append(f'
      • Test ID: {latest_id}
        • ')
+            html.append(f'
      • Date: {latest_test["date"]}
        • ')
+            html.append(f'
      • URLs: {latest_test["urls"]}
        • ')
+            html.append(f'
      • Workers: {latest_test["workers"]}
        • ')
+            html.append(f'
      • Success Rate: {latest_test["success_rate"]:.1f}%
        • ')
+            html.append(f'
      • Time: {latest_test["time_seconds"]:.2f} seconds
        • ')
+            html.append(f'
      • Performance: {latest_test["urls_per_second"]:.1f} URLs/second
        • ')
+            
+            # Check memory growth (handle both pandas and dict mode)
+            memory_growth_available = False
+            if VISUALIZATION_AVAILABLE and df is not None:
+                if pd.notna(latest_test["memory_growth"]):
+                    html.append(f'
      • Memory Growth: {latest_test["memory_growth"]:.1f} MB
        • ')
+                    memory_growth_available = True
+            else:
+                if latest_test["memory_growth"] is not None:
+                    html.append(f'
      • Memory Growth: {latest_test["memory_growth"]:.1f} MB
        • ')
+                    memory_growth_available = True
+            
+            html.append('
      ')
+            
+            # If we have more than one test, show trend
+            if (VISUALIZATION_AVAILABLE and df is not None and len(df) > 1) or (not VISUALIZATION_AVAILABLE and len(rows) > 1):
+                if VISUALIZATION_AVAILABLE and df is not None:
+                    prev_test = df.iloc[1]
+                else:
+                    prev_test = rows[1]
+                
+                # Calculate performance change
+                perf_change = ((latest_test["urls_per_second"] / prev_test["urls_per_second"]) - 1) * 100 if prev_test["urls_per_second"] > 0 else 0
+                
+                status_class = ""
+                if perf_change > 5:
+                    status_class = "status-good"
+                elif perf_change < -5:
+                    status_class = "status-bad"
+                
+                html.append('
      Performance Trend
      ')
+                html.append('
        ')
+                html.append(f'
      • Performance Change: {perf_change:+.1f}% compared to previous test
        • ')
+                
+                # Memory trend if available
+                memory_trend_available = False
+                if VISUALIZATION_AVAILABLE and df is not None:
+                    if pd.notna(latest_test["memory_growth"]) and pd.notna(prev_test["memory_growth"]):
+                        mem_change = latest_test["memory_growth"] - prev_test["memory_growth"]
+                        memory_trend_available = True
+                else:
+                    if latest_test["memory_growth"] is not None and prev_test["memory_growth"] is not None:
+                        mem_change = latest_test["memory_growth"] - prev_test["memory_growth"]
+                        memory_trend_available = True
+                
+                if memory_trend_available:
+                    mem_status = ""
+                    if mem_change < -1:  # Improved (less growth)
+                        mem_status = "status-good"
+                    elif mem_change > 1:  # Worse (more growth)
+                        mem_status = "status-bad"
+                    
+                    html.append(f'
      • Memory Trend: {mem_change:+.1f} MB change in memory growth
        • ')
+                
+                html.append('
      ')
+        
+        html.append('
      ')
+        
+        # Generate performance chart if visualization is available
+        if VISUALIZATION_AVAILABLE:
+            perf_chart = self.generate_performance_chart(results)
+            if perf_chart:
+                html.append('
      ')
+                html.append('
      Performance Comparison
      ')
+                html.append(f'Performance Comparison Chart')
+                html.append('
      ')
+        else:
+            html.append('
      ')
+            html.append('
      Performance Comparison
      ')
+            html.append('
      Charts not available - install visualization dependencies (pandas, matplotlib, seaborn) to enable.
      ')
+            html.append('
      ')
+        
+        # Generate memory charts if visualization is available
+        if VISUALIZATION_AVAILABLE:
+            memory_charts = self.generate_memory_charts(results)
+            if memory_charts:
+                html.append('
      ')
+                html.append('
      Memory Usage
      ')
+                
+                for chart in memory_charts:
+                    test_id = chart.stem.split('_')[-1]
+                    html.append(f'
      Test {test_id}
      ')
+                    html.append(f'Memory Chart for {test_id}')
+                
+                html.append('
      ')
+        else:
+            html.append('
      ')
+            html.append('
      Memory Usage
      ')
+            html.append('
      Charts not available - install visualization dependencies (pandas, matplotlib, seaborn) to enable.
      ')
+            html.append('
      ')
+        
+        # Detailed results table
+        html.append('
      Detailed Results
      ')
+        
+        # Add the results as an HTML table
+        html.append('')
+        
+        # Table headers
+        html.append('')
+        for col in ['Test ID', 'Date', 'URLs', 'Workers', 'Success %', 'Time (s)', 'URLs/sec', 'Mem Growth (MB)']:
+            html.append(f'')
+        html.append('')
+        
+        # Table rows - handle both pandas DataFrame and list of dicts
+        if VISUALIZATION_AVAILABLE and df is not None:
+            # Using pandas DataFrame
+            for _, row in df.iterrows():
+                html.append('')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                
+                # Memory growth cell
+                if pd.notna(row["memory_growth"]):
+                    html.append(f'')
+                else:
+                    html.append('')
+                    
+                html.append('')
+        else:
+            # Using list of dicts (when pandas is not available)
+            for row in rows:
+                html.append('')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                html.append(f'')
+                
+                # Memory growth cell
+                if row["memory_growth"] is not None:
+                    html.append(f'')
+                else:
+                    html.append('')
+                    
+                html.append('')
+        
+        html.append('
      {col}
      {row["test_id"]}{row["date"]}{row["urls"]}{row["workers"]}{row["success_rate"]:.1f}%{row["time_seconds"]:.2f}{row["urls_per_second"]:.1f}{row["memory_growth"]:.1f}N/A
      {row["test_id"]}{row["date"]}{row["urls"]}{row["workers"]}{row["success_rate"]:.1f}%{row["time_seconds"]:.2f}{row["urls_per_second"]:.1f}{row["memory_growth"]:.1f}N/A
      ')
+        
+        # Conclusion section
+        html.append('
      ')
+        html.append('
      Conclusion
      ')
+        
+        if VISUALIZATION_AVAILABLE and df is not None and not df.empty:
+            # Using pandas for statistics (when available)
+            # Calculate some overall statistics
+            avg_urls_per_sec = df['urls_per_second'].mean()
+            max_urls_per_sec = df['urls_per_second'].max()
+            
+            # Determine if we have a trend
+            if len(df) > 1:
+                trend_data = df.sort_values('timestamp')
+                first_perf = trend_data.iloc[0]['urls_per_second']
+                last_perf = trend_data.iloc[-1]['urls_per_second']
+                
+                perf_change = ((last_perf / first_perf) - 1) * 100 if first_perf > 0 else 0
+                
+                if perf_change > 10:
+                    trend_desc = "significantly improved"
+                    trend_class = "status-good"
+                elif perf_change > 5:
+                    trend_desc = "improved"
+                    trend_class = "status-good"
+                elif perf_change < -10:
+                    trend_desc = "significantly decreased"
+                    trend_class = "status-bad"
+                elif perf_change < -5:
+                    trend_desc = "decreased"
+                    trend_class = "status-bad"
+                else:
+                    trend_desc = "remained stable"
+                    trend_class = ""
+                
+                html.append(f'
      Overall performance has {trend_desc} over the test period.
      ')
+            
+            html.append(f'
      Average throughput: {avg_urls_per_sec:.1f} URLs/second
      ')
+            html.append(f'
      Maximum throughput: {max_urls_per_sec:.1f} URLs/second
      ')
+            
+            # Memory leak assessment
+            if 'memory_growth' in df.columns and not df['memory_growth'].isna().all():
+                avg_growth = df['memory_growth'].mean()
+                max_growth = df['memory_growth'].max()
+                
+                if avg_growth < 5:
+                    leak_assessment = "No significant memory leaks detected"
+                    leak_class = "status-good"
+                elif avg_growth < 10:
+                    leak_assessment = "Minor memory growth observed"
+                    leak_class = "status-warning"
+                else:
+                    leak_assessment = "Potential memory leak detected"
+                    leak_class = "status-bad"
+                
+                html.append(f'
      {leak_assessment}. Average memory growth: {avg_growth:.1f} MB per test.
      ')
+        else:
+            # Manual calculations without pandas
+            if rows:
+                # Calculate average and max throughput
+                total_urls_per_sec = sum(row['urls_per_second'] for row in rows)
+                avg_urls_per_sec = total_urls_per_sec / len(rows)
+                max_urls_per_sec = max(row['urls_per_second'] for row in rows)
+                
+                html.append(f'
      Average throughput: {avg_urls_per_sec:.1f} URLs/second
      ')
+                html.append(f'
      Maximum throughput: {max_urls_per_sec:.1f} URLs/second
      ')
+                
+                # Memory assessment (simplified without pandas)
+                growth_values = [row['memory_growth'] for row in rows if row['memory_growth'] is not None]
+                if growth_values:
+                    avg_growth = sum(growth_values) / len(growth_values)
+                    
+                    if avg_growth < 5:
+                        leak_assessment = "No significant memory leaks detected"
+                        leak_class = "status-good"
+                    elif avg_growth < 10:
+                        leak_assessment = "Minor memory growth observed"
+                        leak_class = "status-warning"
+                    else:
+                        leak_assessment = "Potential memory leak detected"
+                        leak_class = "status-bad"
+                    
+                    html.append(f'
      {leak_assessment}. Average memory growth: {avg_growth:.1f} MB per test.
      ')
+            else:
+                html.append('
      No test data available for analysis.
      ')
+        
+        html.append('
      ')
+        
+        # Footer
+        html.append('
      ')
+        html.append('
      Generated by Crawl4AI Benchmark Reporter
      ')
+        html.append('
      ')
+        
+        html.append('')
+        html.append('')
+        
+        # Write the HTML file
+        with open(output_file, 'w') as f:
+            f.write('\n'.join(html))
+        
+        # Print a clickable link for terminals that support it (iTerm, VS Code, etc.)
+        file_url = f"file://{os.path.abspath(output_file)}"
+        console.print(f"[green]Comparison report saved to: {output_file}[/green]")
+        console.print(f"[blue underline]Click to open report: {file_url}[/blue underline]")
+        return output_file
+    
+    def run(self, limit=None, output_file=None):
+        """Generate a full benchmark report.
+        
+        Args:
+            limit: Optional limit on number of most recent tests to include
+            output_file: Optional output file path
+            
+        Returns:
+            Path to the generated report file
+        """
+        # Load test results
+        results = self.load_test_results(limit=limit)
+        
+        if not results:
+            console.print("[yellow]No test results found. Run some tests first.[/yellow]")
+            return None
+        
+        # Generate and display summary table
+        summary_table = self.generate_summary_table(results)
+        console.print(summary_table)
+        
+        # Generate comparison report
+        title = f"Crawl4AI Benchmark Report ({len(results)} test runs)"
+        report_file = self.generate_comparison_report(results, title=title, output_file=output_file)
+        
+        if report_file:
+            console.print(f"[bold green]Report generated successfully: {report_file}[/bold green]")
+            return report_file
+        else:
+            console.print("[bold red]Failed to generate report[/bold red]")
+            return None
+
+
+def main():
+    """Main entry point for the benchmark reporter."""
+    parser = argparse.ArgumentParser(description="Generate benchmark reports for Crawl4AI stress tests")
+    
+    parser.add_argument("--reports-dir", type=str, default="reports",
+                      help="Directory containing test result files")
+    parser.add_argument("--output-dir", type=str, default="benchmark_reports",
+                      help="Directory to save generated reports")
+    parser.add_argument("--limit", type=int, default=None,
+                      help="Limit to most recent N test results")
+    parser.add_argument("--output-file", type=str, default=None,
+                      help="Custom output file path for the report")
+    
+    args = parser.parse_args()
+    
+    # Create the benchmark reporter
+    reporter = BenchmarkReporter(reports_dir=args.reports_dir, output_dir=args.output_dir)
+    
+    # Generate the report
+    report_file = reporter.run(limit=args.limit, output_file=args.output_file)
+    
+    if report_file:
+        print(f"Report generated at: {report_file}")
+        return 0
+    else:
+        print("Failed to generate report")
+        return 1
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main())
\ No newline at end of file
diff --git a/tests/memory/cap_test.py b/tests/memory/cap_test.py
new file mode 100644
index 0000000..56d7b26
--- /dev/null
+++ b/tests/memory/cap_test.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+"""
+Hammer /crawl with many concurrent requests to prove GLOBAL_SEM works.
+"""
+
+import asyncio, httpx, json, uuid, argparse
+
+API = "http://localhost:8020/crawl"
+URLS_PER_CALL = 1          # keep it minimal so each arun() == 1 page
+CONCURRENT_CALLS = 20      # way above your cap
+
+payload_template = {
+    "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {"cache_mode": "BYPASS", "verbose": False},
+    }
+}
+
+async def one_call(client):
+    payload = payload_template.copy()
+    payload["urls"] = [f"https://httpbin.org/anything/{uuid.uuid4()}"]
+    r = await client.post(API, json=payload)
+    r.raise_for_status()
+    return r.json()["server_peak_memory_mb"]
+
+async def main():
+    async with httpx.AsyncClient(timeout=60) as client:
+        tasks = [asyncio.create_task(one_call(client)) for _ in range(CONCURRENT_CALLS)]
+        mem_usages = await asyncio.gather(*tasks)
+        print("Calls finished OK, server peaks reported:", mem_usages)
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/memory/requirements.txt b/tests/memory/requirements.txt
new file mode 100644
index 0000000..230e0e1
--- /dev/null
+++ b/tests/memory/requirements.txt
@@ -0,0 +1,4 @@
+pandas>=1.5.0
+matplotlib>=3.5.0
+seaborn>=0.12.0
+rich>=12.0.0
\ No newline at end of file
diff --git a/tests/memory/run_benchmark.py b/tests/memory/run_benchmark.py
new file mode 100755
index 0000000..1e110dd
--- /dev/null
+++ b/tests/memory/run_benchmark.py
@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""
+Run a complete Crawl4AI benchmark test using test_stress_sdk.py and generate a report.
+"""
+
+import sys
+import os
+import glob
+import argparse
+import subprocess
+import time
+from datetime import datetime
+
+from rich.console import Console
+from rich.text import Text
+
+console = Console()
+
+# Updated TEST_CONFIGS to use max_sessions
+TEST_CONFIGS = {
+    "quick":   {"urls": 50,   "max_sessions": 4,  "chunk_size": 10, "description": "Quick test (50 URLs, 4 sessions)"},
+    "small":   {"urls": 100,  "max_sessions": 8,  "chunk_size": 20, "description": "Small test (100 URLs, 8 sessions)"},
+    "medium":  {"urls": 500,  "max_sessions": 16, "chunk_size": 50, "description": "Medium test (500 URLs, 16 sessions)"},
+    "large":   {"urls": 1000, "max_sessions": 32, "chunk_size": 100,"description": "Large test (1000 URLs, 32 sessions)"},
+    "extreme": {"urls": 2000, "max_sessions": 64, "chunk_size": 200,"description": "Extreme test (2000 URLs, 64 sessions)"},
+}
+
+# Arguments to forward directly if present in custom_args
+FORWARD_ARGS = {
+    "urls": "--urls",
+    "max_sessions": "--max-sessions",
+    "chunk_size": "--chunk-size",
+    "port": "--port",
+    "monitor_mode": "--monitor-mode",
+}
+# Boolean flags to forward if True
+FORWARD_FLAGS = {
+    "stream": "--stream",
+    "use_rate_limiter": "--use-rate-limiter",
+    "keep_server_alive": "--keep-server-alive",
+    "use_existing_site": "--use-existing-site",
+    "skip_generation": "--skip-generation",
+    "keep_site": "--keep-site",
+    "clean_reports": "--clean-reports", # Note: clean behavior is handled here, but pass flag if needed
+    "clean_site": "--clean-site",     # Note: clean behavior is handled here, but pass flag if needed
+}
+
+def run_benchmark(config_name, custom_args=None, compare=True, clean=False):
+    """Runs the stress test and optionally the report generator."""
+    if config_name not in TEST_CONFIGS and config_name != "custom":
+        console.print(f"[bold red]Unknown configuration: {config_name}[/bold red]")
+        return False
+
+    # Print header
+    title = "Crawl4AI SDK Benchmark Test"
+    if config_name != "custom":
+        title += f" - {TEST_CONFIGS[config_name]['description']}"
+    else:
+        # Safely get custom args for title
+        urls = custom_args.get('urls', '?') if custom_args else '?'
+        sessions = custom_args.get('max_sessions', '?') if custom_args else '?'
+        title += f" - Custom ({urls} URLs, {sessions} sessions)"
+
+    console.print(f"\n[bold blue]{title}[/bold blue]")
+    console.print("=" * (len(title) + 4)) # Adjust underline length
+
+    console.print("\n[bold white]Preparing test...[/bold white]")
+
+    # --- Command Construction ---
+    # Use the new script name
+    cmd = ["python", "test_stress_sdk.py"]
+
+    # Apply config or custom args
+    args_to_use = {}
+    if config_name != "custom":
+        args_to_use = TEST_CONFIGS[config_name].copy()
+        # If custom args are provided (e.g., boolean flags), overlay them
+        if custom_args:
+            args_to_use.update(custom_args)
+    elif custom_args: # Custom config
+        args_to_use = custom_args.copy()
+
+    # Add arguments with values
+    for key, arg_name in FORWARD_ARGS.items():
+        if key in args_to_use:
+            cmd.extend([arg_name, str(args_to_use[key])])
+
+    # Add boolean flags
+    for key, flag_name in FORWARD_FLAGS.items():
+        if args_to_use.get(key, False): # Check if key exists and is True
+             # Special handling for clean flags - apply locally, don't forward?
+             # Decide if test_stress_sdk.py also needs --clean flags or if run_benchmark handles it.
+             # For now, let's assume run_benchmark handles cleaning based on its own --clean flag.
+             # We'll forward other flags.
+            if key not in ["clean_reports", "clean_site"]:
+                 cmd.append(flag_name)
+
+    # Handle the top-level --clean flag for run_benchmark
+    if clean:
+        # Pass clean flags to the stress test script as well, if needed
+        # This assumes test_stress_sdk.py also uses --clean-reports and --clean-site
+        cmd.append("--clean-reports")
+        cmd.append("--clean-site")
+        console.print("[yellow]Applying --clean: Cleaning reports and site before test.[/yellow]")
+        # Actual cleaning logic might reside here or be delegated entirely
+
+    console.print(f"\n[bold white]Running stress test:[/bold white] {' '.join(cmd)}")
+    start = time.time()
+
+    # Execute the stress test script
+    # Use Popen to stream output
+    try:
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace')
+        while True:
+            line = proc.stdout.readline()
+            if not line:
+                break
+            console.print(line.rstrip()) # Print line by line
+        proc.wait() # Wait for the process to complete
+    except FileNotFoundError:
+         console.print(f"[bold red]Error: Script 'test_stress_sdk.py' not found. Make sure it's in the correct directory.[/bold red]")
+         return False
+    except Exception as e:
+         console.print(f"[bold red]Error running stress test subprocess: {e}[/bold red]")
+         return False
+
+
+    if proc.returncode != 0:
+        console.print(f"[bold red]Stress test failed with exit code {proc.returncode}[/bold red]")
+        return False
+
+    duration = time.time() - start
+    console.print(f"[bold green]Stress test completed in {duration:.1f} seconds[/bold green]")
+
+    # --- Report Generation (Optional) ---
+    if compare:
+        # Assuming benchmark_report.py exists and works with the generated reports
+        report_script = "benchmark_report.py" # Keep configurable if needed
+        report_cmd = ["python", report_script]
+        console.print(f"\n[bold white]Generating benchmark report: {' '.join(report_cmd)}[/bold white]")
+
+        # Run the report command and capture output
+        try:
+             report_proc = subprocess.run(report_cmd, capture_output=True, text=True, check=False, encoding='utf-8', errors='replace') # Use check=False to handle potential errors
+
+             # Print the captured output from benchmark_report.py
+             if report_proc.stdout:
+                 console.print("\n" + report_proc.stdout)
+             if report_proc.stderr:
+                 console.print("[yellow]Report generator stderr:[/yellow]\n" + report_proc.stderr)
+
+             if report_proc.returncode != 0:
+                 console.print(f"[bold yellow]Benchmark report generation script '{report_script}' failed with exit code {report_proc.returncode}[/bold yellow]")
+                 # Don't return False here, test itself succeeded
+             else:
+                  console.print(f"[bold green]Benchmark report script '{report_script}' completed.[/bold green]")
+
+             # Find and print clickable links to the reports
+             # Assuming reports are saved in 'benchmark_reports' by benchmark_report.py
+             report_dir = "benchmark_reports"
+             if os.path.isdir(report_dir):
+                 report_files = glob.glob(os.path.join(report_dir, "comparison_report_*.html"))
+                 if report_files:
+                     try:
+                         latest_report = max(report_files, key=os.path.getctime)
+                         report_path = os.path.abspath(latest_report)
+                         report_url = pathlib.Path(report_path).as_uri() # Better way to create file URI
+                         console.print(f"[bold cyan]Click to open report: [link={report_url}]{report_url}[/link][/bold cyan]")
+                     except Exception as e:
+                          console.print(f"[yellow]Could not determine latest report: {e}[/yellow]")
+
+                 chart_files = glob.glob(os.path.join(report_dir, "memory_chart_*.png"))
+                 if chart_files:
+                      try:
+                         latest_chart = max(chart_files, key=os.path.getctime)
+                         chart_path = os.path.abspath(latest_chart)
+                         chart_url = pathlib.Path(chart_path).as_uri()
+                         console.print(f"[cyan]Memory chart: [link={chart_url}]{chart_url}[/link][/cyan]")
+                      except Exception as e:
+                           console.print(f"[yellow]Could not determine latest chart: {e}[/yellow]")
+             else:
+                  console.print(f"[yellow]Benchmark report directory '{report_dir}' not found. Cannot link reports.[/yellow]")
+
+        except FileNotFoundError:
+             console.print(f"[bold red]Error: Report script '{report_script}' not found.[/bold red]")
+        except Exception as e:
+             console.print(f"[bold red]Error running report generation subprocess: {e}[/bold red]")
+
+
+    # Prompt to exit
+    console.print("\n[bold green]Benchmark run finished. Press Enter to exit.[/bold green]")
+    try:
+        input() # Wait for user input
+    except EOFError:
+        pass # Handle case where input is piped or unavailable
+
+    return True
+
+def main():
+    parser = argparse.ArgumentParser(description="Run a Crawl4AI SDK benchmark test and generate a report")
+
+    # --- Arguments ---
+    parser.add_argument("config", choices=list(TEST_CONFIGS) + ["custom"],
+                        help="Test configuration: quick, small, medium, large, extreme, or custom")
+
+    # Arguments for 'custom' config or to override presets
+    parser.add_argument("--urls", type=int, help="Number of URLs")
+    parser.add_argument("--max-sessions", type=int, help="Max concurrent sessions (replaces --workers)")
+    parser.add_argument("--chunk-size", type=int, help="URLs per batch (for non-stream logging)")
+    parser.add_argument("--port", type=int, help="HTTP server port")
+    parser.add_argument("--monitor-mode", type=str, choices=["DETAILED", "AGGREGATED"], help="Monitor display mode")
+
+    # Boolean flags / options
+    parser.add_argument("--stream", action="store_true", help="Enable streaming results (disables batch logging)")
+    parser.add_argument("--use-rate-limiter", action="store_true", help="Enable basic rate limiter")
+    parser.add_argument("--no-report", action="store_true", help="Skip generating comparison report")
+    parser.add_argument("--clean", action="store_true", help="Clean up reports and site before running")
+    parser.add_argument("--keep-server-alive", action="store_true", help="Keep HTTP server running after test")
+    parser.add_argument("--use-existing-site", action="store_true", help="Use existing site on specified port")
+    parser.add_argument("--skip-generation", action="store_true", help="Use existing site files without regenerating")
+    parser.add_argument("--keep-site", action="store_true", help="Keep generated site files after test")
+    # Removed url_level_logging as it's implicitly handled by stream/batch mode now
+
+    args = parser.parse_args()
+
+    custom_args = {}
+
+    # Populate custom_args from explicit command-line args
+    if args.urls is not None: custom_args["urls"] = args.urls
+    if args.max_sessions is not None: custom_args["max_sessions"] = args.max_sessions
+    if args.chunk_size is not None: custom_args["chunk_size"] = args.chunk_size
+    if args.port is not None: custom_args["port"] = args.port
+    if args.monitor_mode is not None: custom_args["monitor_mode"] = args.monitor_mode
+    if args.stream: custom_args["stream"] = True
+    if args.use_rate_limiter: custom_args["use_rate_limiter"] = True
+    if args.keep_server_alive: custom_args["keep_server_alive"] = True
+    if args.use_existing_site: custom_args["use_existing_site"] = True
+    if args.skip_generation: custom_args["skip_generation"] = True
+    if args.keep_site: custom_args["keep_site"] = True
+    # Clean flags are handled by the 'clean' argument passed to run_benchmark
+
+    # Validate custom config requirements
+    if args.config == "custom":
+        required_custom = ["urls", "max_sessions", "chunk_size"]
+        missing = [f"--{arg}" for arg in required_custom if arg not in custom_args]
+        if missing:
+            console.print(f"[bold red]Error: 'custom' config requires: {', '.join(missing)}[/bold red]")
+            return 1
+
+    success = run_benchmark(
+        config_name=args.config,
+        custom_args=custom_args, # Pass all collected custom args
+        compare=not args.no_report,
+        clean=args.clean
+    )
+    return 0 if success else 1
+
+if __name__ == "__main__":
+    sys.exit(main())
\ No newline at end of file
diff --git a/tests/memory/test_crawler_monitor.py b/tests/memory/test_crawler_monitor.py
new file mode 100644
index 0000000..89cc08b
--- /dev/null
+++ b/tests/memory/test_crawler_monitor.py
@@ -0,0 +1,168 @@
+"""
+Test script for the CrawlerMonitor component.
+This script simulates a crawler with multiple tasks to demonstrate the real-time monitoring capabilities.
+"""
+
+import time
+import uuid
+import random
+import threading
+import sys
+import os
+
+# Add the parent directory to the path to import crawl4ai
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
+
+from crawl4ai.components.crawler_monitor import CrawlerMonitor
+from crawl4ai.models import CrawlStatus
+
+def simulate_crawler_task(monitor, task_id, url, simulate_failure=False):
+    """Simulate a crawler task with different states."""
+    # Task starts in the QUEUED state
+    wait_time = random.uniform(0.5, 3.0)
+    time.sleep(wait_time)
+    
+    # Update to IN_PROGRESS state
+    monitor.update_task(
+        task_id=task_id,
+        status=CrawlStatus.IN_PROGRESS,
+        start_time=time.time(),
+        wait_time=wait_time
+    )
+    
+    # Simulate task running
+    process_time = random.uniform(1.0, 5.0)
+    for i in range(int(process_time * 2)):
+        # Simulate memory usage changes
+        memory_usage = random.uniform(5.0, 25.0)
+        monitor.update_task(
+            task_id=task_id,
+            memory_usage=memory_usage,
+            peak_memory=max(memory_usage, monitor.get_task_stats(task_id).get("peak_memory", 0))
+        )
+        time.sleep(0.5)
+    
+    # Update to COMPLETED or FAILED state
+    if simulate_failure and random.random() < 0.8:  # 80% chance of failure if simulate_failure is True
+        monitor.update_task(
+            task_id=task_id,
+            status=CrawlStatus.FAILED,
+            end_time=time.time(),
+            error_message="Simulated failure: Connection timeout",
+            memory_usage=0.0
+        )
+    else:
+        monitor.update_task(
+            task_id=task_id,
+            status=CrawlStatus.COMPLETED,
+            end_time=time.time(),
+            memory_usage=0.0
+        )
+
+def update_queue_stats(monitor, num_queued_tasks):
+    """Update queue statistics periodically."""
+    while monitor.is_running:
+        queued_tasks = [
+            task for task_id, task in monitor.get_all_task_stats().items()
+            if task["status"] == CrawlStatus.QUEUED.name
+        ]
+        
+        total_queued = len(queued_tasks)
+        
+        if total_queued > 0:
+            current_time = time.time()
+            wait_times = [
+                current_time - task.get("enqueue_time", current_time)
+                for task in queued_tasks
+            ]
+            highest_wait_time = max(wait_times) if wait_times else 0.0
+            avg_wait_time = sum(wait_times) / len(wait_times) if wait_times else 0.0
+        else:
+            highest_wait_time = 0.0
+            avg_wait_time = 0.0
+        
+        monitor.update_queue_statistics(
+            total_queued=total_queued,
+            highest_wait_time=highest_wait_time,
+            avg_wait_time=avg_wait_time
+        )
+        
+        # Simulate memory pressure based on number of active tasks
+        active_tasks = len([
+            task for task_id, task in monitor.get_all_task_stats().items()
+            if task["status"] == CrawlStatus.IN_PROGRESS.name
+        ])
+        
+        if active_tasks > 8:
+            monitor.update_memory_status("CRITICAL")
+        elif active_tasks > 4:
+            monitor.update_memory_status("PRESSURE")
+        else:
+            monitor.update_memory_status("NORMAL")
+            
+        time.sleep(1.0)
+
+def test_crawler_monitor():
+    """Test the CrawlerMonitor with simulated crawler tasks."""
+    # Total number of URLs to crawl
+    total_urls = 50
+    
+    # Initialize the monitor
+    monitor = CrawlerMonitor(urls_total=total_urls, refresh_rate=0.5)
+    
+    # Start the monitor
+    monitor.start()
+    
+    # Start thread to update queue statistics
+    queue_stats_thread = threading.Thread(target=update_queue_stats, args=(monitor, total_urls))
+    queue_stats_thread.daemon = True
+    queue_stats_thread.start()
+    
+    try:
+        # Create task threads
+        threads = []
+        for i in range(total_urls):
+            task_id = str(uuid.uuid4())
+            url = f"https://example.com/page{i}"
+            
+            # Add task to monitor
+            monitor.add_task(task_id, url)
+            
+            # Determine if this task should simulate failure
+            simulate_failure = (i % 10 == 0)  # Every 10th task
+            
+            # Create and start thread for this task
+            thread = threading.Thread(
+                target=simulate_crawler_task,
+                args=(monitor, task_id, url, simulate_failure)
+            )
+            thread.daemon = True
+            threads.append(thread)
+        
+        # Start threads with delay to simulate tasks being added over time
+        batch_size = 5
+        for i in range(0, len(threads), batch_size):
+            batch = threads[i:i+batch_size]
+            for thread in batch:
+                thread.start()
+                time.sleep(0.5)  # Small delay between starting threads
+            
+            # Wait a bit before starting the next batch
+            time.sleep(2.0)
+        
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+            
+        # Keep monitor running a bit longer to see the final state
+        time.sleep(5.0)
+        
+    except KeyboardInterrupt:
+        print("\nTest interrupted by user")
+    finally:
+        # Stop the monitor
+        monitor.stop()
+        print("\nCrawler monitor test completed")
+
+if __name__ == "__main__":
+    test_crawler_monitor()
\ No newline at end of file
diff --git a/tests/memory/test_dispatcher_stress.py b/tests/memory/test_dispatcher_stress.py
new file mode 100644
index 0000000..f81f78f
--- /dev/null
+++ b/tests/memory/test_dispatcher_stress.py
@@ -0,0 +1,410 @@
+import asyncio
+import time
+import psutil
+import logging
+import random
+from typing import List, Dict
+import uuid
+import sys
+import os
+
+# Import your crawler components
+from crawl4ai.models import DisplayMode, CrawlStatus, CrawlResult
+from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig, CacheMode
+from crawl4ai import AsyncWebCrawler
+from crawl4ai import MemoryAdaptiveDispatcher, CrawlerMonitor
+
+# Global configuration
+STREAM = False  # Toggle between streaming and non-streaming modes
+
+# Configure logging to file only (to avoid breaking the rich display)
+os.makedirs("logs", exist_ok=True)
+file_handler = logging.FileHandler("logs/memory_stress_test.log")
+file_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s'))
+
+# Root logger - only to file, not console
+root_logger = logging.getLogger()
+root_logger.setLevel(logging.INFO)
+root_logger.addHandler(file_handler)
+
+# Our test logger also writes to file only
+logger = logging.getLogger("memory_stress_test")
+logger.setLevel(logging.INFO)
+logger.addHandler(file_handler)
+logger.propagate = False  # Don't propagate to root logger
+
+# Create a memory restrictor to simulate limited memory environment
+class MemorySimulator:
+    def __init__(self, target_percent: float = 85.0, aggressive: bool = False):
+        """Simulates memory pressure by allocating memory"""
+        self.target_percent = target_percent
+        self.memory_blocks: List[bytearray] = []
+        self.aggressive = aggressive
+        
+    def apply_pressure(self, additional_percent: float = 0.0):
+        """Fill memory until we reach target percentage"""
+        current_percent = psutil.virtual_memory().percent
+        target = self.target_percent + additional_percent
+        
+        if current_percent >= target:
+            return  # Already at target
+            
+        logger.info(f"Current memory: {current_percent}%, target: {target}%")
+        
+        # Calculate how much memory we need to allocate
+        total_memory = psutil.virtual_memory().total
+        target_usage = (target / 100.0) * total_memory
+        current_usage = (current_percent / 100.0) * total_memory
+        bytes_to_allocate = int(target_usage - current_usage)
+        
+        if bytes_to_allocate <= 0:
+            return
+            
+        # Allocate in smaller chunks to avoid overallocation
+        if self.aggressive:
+            # Use larger chunks for faster allocation in aggressive mode
+            chunk_size = min(bytes_to_allocate, 200 * 1024 * 1024)  # 200MB chunks
+        else:
+            chunk_size = min(bytes_to_allocate, 50 * 1024 * 1024)   # 50MB chunks
+        
+        try:
+            logger.info(f"Allocating {chunk_size / (1024 * 1024):.1f}MB to reach target memory usage")
+            self.memory_blocks.append(bytearray(chunk_size))
+            time.sleep(0.5)  # Give system time to register the allocation
+        except MemoryError:
+            logger.warning("Unable to allocate more memory")
+            
+    def release_pressure(self, percent: float = None):
+        """
+        Release allocated memory
+        If percent is specified, release that percentage of blocks
+        """
+        if not self.memory_blocks:
+            return
+            
+        if percent is None:
+            # Release all
+            logger.info(f"Releasing all {len(self.memory_blocks)} memory blocks")
+            self.memory_blocks.clear()
+        else:
+            # Release specified percentage
+            blocks_to_release = int(len(self.memory_blocks) * (percent / 100.0))
+            if blocks_to_release > 0:
+                logger.info(f"Releasing {blocks_to_release} of {len(self.memory_blocks)} memory blocks ({percent}%)")
+                self.memory_blocks = self.memory_blocks[blocks_to_release:]
+                
+    def spike_pressure(self, duration: float = 5.0):
+        """
+        Create a temporary spike in memory pressure then release
+        Useful for forcing requeues
+        """
+        logger.info(f"Creating memory pressure spike for {duration} seconds")
+        # Save current blocks count
+        initial_blocks = len(self.memory_blocks)
+        
+        # Create spike with extra 5%
+        self.apply_pressure(additional_percent=5.0)
+        
+        # Schedule release after duration
+        asyncio.create_task(self._delayed_release(duration, initial_blocks))
+        
+    async def _delayed_release(self, delay: float, target_blocks: int):
+        """Helper for spike_pressure - releases extra blocks after delay"""
+        await asyncio.sleep(delay)
+        
+        # Remove blocks added since spike started
+        if len(self.memory_blocks) > target_blocks:
+            logger.info(f"Releasing memory spike ({len(self.memory_blocks) - target_blocks} blocks)")
+            self.memory_blocks = self.memory_blocks[:target_blocks]
+            
+# Test statistics collector
+class TestResults:
+    def __init__(self):
+        self.start_time = time.time()
+        self.completed_urls: List[str] = []
+        self.failed_urls: List[str] = []
+        self.requeued_count = 0
+        self.memory_warnings = 0
+        self.max_memory_usage = 0.0
+        self.max_queue_size = 0
+        self.max_wait_time = 0.0
+        self.url_to_attempt: Dict[str, int] = {}  # Track retries per URL
+        
+    def log_summary(self):
+        duration = time.time() - self.start_time
+        logger.info("===== TEST SUMMARY =====")
+        logger.info(f"Stream mode: {'ON' if STREAM else 'OFF'}")
+        logger.info(f"Total duration: {duration:.1f} seconds")
+        logger.info(f"Completed URLs: {len(self.completed_urls)}")
+        logger.info(f"Failed URLs: {len(self.failed_urls)}")
+        logger.info(f"Requeue events: {self.requeued_count}")
+        logger.info(f"Memory warnings: {self.memory_warnings}")
+        logger.info(f"Max memory usage: {self.max_memory_usage:.1f}%")
+        logger.info(f"Max queue size: {self.max_queue_size}")
+        logger.info(f"Max wait time: {self.max_wait_time:.1f} seconds")
+        
+        # Log URLs with multiple attempts
+        retried_urls = {url: count for url, count in self.url_to_attempt.items() if count > 1}
+        if retried_urls:
+            logger.info(f"URLs with retries: {len(retried_urls)}")
+            # Log the top 5 most retried
+            top_retries = sorted(retried_urls.items(), key=lambda x: x[1], reverse=True)[:5]
+            for url, count in top_retries:
+                logger.info(f"  URL {url[-30:]} had {count} attempts")
+        
+        # Write summary to a separate human-readable file
+        with open("logs/test_summary.txt", "w") as f:
+            f.write(f"Stream mode: {'ON' if STREAM else 'OFF'}\n")
+            f.write(f"Total duration: {duration:.1f} seconds\n")
+            f.write(f"Completed URLs: {len(self.completed_urls)}\n")
+            f.write(f"Failed URLs: {len(self.failed_urls)}\n")
+            f.write(f"Requeue events: {self.requeued_count}\n")
+            f.write(f"Memory warnings: {self.memory_warnings}\n")
+            f.write(f"Max memory usage: {self.max_memory_usage:.1f}%\n")
+            f.write(f"Max queue size: {self.max_queue_size}\n")
+            f.write(f"Max wait time: {self.max_wait_time:.1f} seconds\n")
+        
+# Custom monitor with stats tracking
+# Custom monitor that extends CrawlerMonitor with test-specific tracking
+class StressTestMonitor(CrawlerMonitor):
+    def __init__(self, test_results: TestResults, **kwargs):
+        # Initialize the parent CrawlerMonitor
+        super().__init__(**kwargs)
+        self.test_results = test_results
+        
+    def update_memory_status(self, status: str):
+        if status != self.memory_status:
+            logger.info(f"Memory status changed: {self.memory_status} -> {status}")
+            if "CRITICAL" in status or "PRESSURE" in status:
+                self.test_results.memory_warnings += 1
+                
+        # Track peak memory usage in test results
+        current_memory = psutil.virtual_memory().percent
+        self.test_results.max_memory_usage = max(self.test_results.max_memory_usage, current_memory)
+        
+        # Call parent method to update the dashboard
+        super().update_memory_status(status)
+        
+    def update_queue_statistics(self, total_queued: int, highest_wait_time: float, avg_wait_time: float):
+        # Track queue metrics in test results
+        self.test_results.max_queue_size = max(self.test_results.max_queue_size, total_queued)
+        self.test_results.max_wait_time = max(self.test_results.max_wait_time, highest_wait_time)
+        
+        # Call parent method to update the dashboard
+        super().update_queue_statistics(total_queued, highest_wait_time, avg_wait_time)
+        
+    def update_task(self, task_id: str, **kwargs):
+        # Track URL status changes for test results
+        if task_id in self.stats:
+            old_status = self.stats[task_id].status
+            
+            # If this is a requeue event (requeued due to memory pressure)
+            if 'error_message' in kwargs and 'requeued' in kwargs['error_message']:
+                if not hasattr(self.stats[task_id], 'counted_requeue') or not self.stats[task_id].counted_requeue:
+                    self.test_results.requeued_count += 1
+                    self.stats[task_id].counted_requeue = True
+                    
+            # Track completion status for test results
+            if 'status' in kwargs:
+                new_status = kwargs['status']
+                if old_status != new_status:
+                    if new_status == CrawlStatus.COMPLETED:
+                        if task_id not in self.test_results.completed_urls:
+                            self.test_results.completed_urls.append(task_id)
+                    elif new_status == CrawlStatus.FAILED:
+                        if task_id not in self.test_results.failed_urls:
+                            self.test_results.failed_urls.append(task_id)
+        
+        # Call parent method to update the dashboard
+        super().update_task(task_id, **kwargs)
+        self.live.update(self._create_table())
+
+# Generate test URLs - use example.com with unique paths to avoid browser caching
+def generate_test_urls(count: int) -> List[str]:
+    urls = []
+    for i in range(count):
+        # Add random path and query parameters to create unique URLs
+        path = f"/path/{uuid.uuid4()}"
+        query = f"?test={i}&random={random.randint(1, 100000)}"
+        urls.append(f"https://example.com{path}{query}")
+    return urls
+
+# Process result callback
+async def process_result(result, test_results: TestResults):
+    # Track attempt counts
+    if result.url not in test_results.url_to_attempt:
+        test_results.url_to_attempt[result.url] = 1
+    else:
+        test_results.url_to_attempt[result.url] += 1
+    
+    if "requeued" in result.error_message:
+        test_results.requeued_count += 1
+        logger.debug(f"Requeued due to memory pressure: {result.url}")
+    elif result.success:
+        test_results.completed_urls.append(result.url)
+        logger.debug(f"Successfully processed: {result.url}")
+    else:
+        test_results.failed_urls.append(result.url)
+        logger.warning(f"Failed to process: {result.url} - {result.error_message}")
+
+# Process multiple results (used in non-streaming mode)
+async def process_results(results, test_results: TestResults):
+    for result in results:
+        await process_result(result, test_results)
+
+# Main test function for extreme memory pressure simulation
+async def run_memory_stress_test(
+    url_count: int = 100,
+    target_memory_percent: float = 92.0,  # Push to dangerous levels
+    chunk_size: int = 20,  # Larger chunks for more chaos
+    aggressive: bool = False,
+    spikes: bool = True
+):
+    test_results = TestResults()
+    memory_simulator = MemorySimulator(target_percent=target_memory_percent, aggressive=aggressive)
+    
+    logger.info(f"Starting stress test with {url_count} URLs in {'STREAM' if STREAM else 'NON-STREAM'} mode")
+    logger.info(f"Target memory usage: {target_memory_percent}%")
+    
+    # First, elevate memory usage to create pressure
+    logger.info("Creating initial memory pressure...")
+    memory_simulator.apply_pressure()
+    
+    # Create test URLs in chunks to simulate real-world crawling where URLs are discovered
+    all_urls = generate_test_urls(url_count)
+    url_chunks = [all_urls[i:i+chunk_size] for i in range(0, len(all_urls), chunk_size)]
+    
+    # Set up the crawler components - low memory thresholds to create more requeues
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        verbose=False,
+        stream=STREAM  # Use the global STREAM variable to set mode
+    )
+    
+    # Create monitor with reference to test results
+    monitor = StressTestMonitor(
+        test_results=test_results,
+        display_mode=DisplayMode.DETAILED,
+        max_visible_rows=20,
+        total_urls=url_count  # Pass total URLs count
+    )
+    
+    # Create dispatcher with EXTREME settings - pure survival mode
+    # These settings are designed to create a memory battleground
+    dispatcher = MemoryAdaptiveDispatcher(
+        memory_threshold_percent=63.0,  # Start throttling at just 60% memory
+        critical_threshold_percent=70.0,  # Start requeuing at 70% - incredibly aggressive  
+        recovery_threshold_percent=55.0,  # Only resume normal ops when plenty of memory available
+        check_interval=0.1,  # Check extremely frequently (100ms)
+        max_session_permit=20 if aggressive else 10,  # Double the concurrent sessions - pure chaos
+        fairness_timeout=10.0,  # Extremely low timeout - rapid priority changes
+        monitor=monitor
+    )
+    
+    # Set up spike schedule if enabled
+    if spikes:
+        spike_intervals = []
+        # Create 3-5 random spike times
+        num_spikes = random.randint(3, 5)
+        for _ in range(num_spikes):
+            # Schedule spikes at random chunks
+            chunk_index = random.randint(1, len(url_chunks) - 1)
+            spike_intervals.append(chunk_index)
+        logger.info(f"Scheduled memory spikes at chunks: {spike_intervals}")
+    
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            # Process URLs in chunks to simulate discovering URLs over time
+            for chunk_index, url_chunk in enumerate(url_chunks):
+                logger.info(f"Processing chunk {chunk_index+1}/{len(url_chunks)} ({len(url_chunk)} URLs)")
+                
+                # Regular pressure increases
+                if chunk_index % 2 == 0:
+                    logger.info("Increasing memory pressure...")
+                    memory_simulator.apply_pressure()
+                
+                # Memory spike if scheduled for this chunk
+                if spikes and chunk_index in spike_intervals:
+                    logger.info(f"⚠️ CREATING MASSIVE MEMORY SPIKE at chunk {chunk_index+1} ⚠️")
+                    # Create a nightmare scenario - multiple overlapping spikes
+                    memory_simulator.spike_pressure(duration=10.0)  # 10-second spike
+                    
+                    # 50% chance of double-spike (pure evil)
+                    if random.random() < 0.5:
+                        await asyncio.sleep(2.0)  # Wait 2 seconds
+                        logger.info("💀 DOUBLE SPIKE - EXTREME MEMORY PRESSURE 💀")
+                        memory_simulator.spike_pressure(duration=8.0)  # 8-second overlapping spike
+                
+                if STREAM:
+                    # Stream mode - process results as they come in
+                    async for result in dispatcher.run_urls_stream(
+                        urls=url_chunk,
+                        crawler=crawler,
+                        config=run_config
+                    ):
+                        await process_result(result, test_results)
+                else:
+                    # Non-stream mode - get all results at once
+                    results = await dispatcher.run_urls(
+                        urls=url_chunk,
+                        crawler=crawler,
+                        config=run_config
+                    )
+                    await process_results(results, test_results)
+                    
+                # Simulate discovering more URLs while others are still processing
+                await asyncio.sleep(1)
+                
+                # RARELY release pressure - make the system fight for resources
+                if chunk_index % 5 == 4:  # Less frequent releases
+                    release_percent = random.choice([10, 15, 20])  # Smaller, inconsistent releases
+                    logger.info(f"Releasing {release_percent}% of memory blocks - brief respite")
+                    memory_simulator.release_pressure(percent=release_percent)
+    
+    except Exception as e:
+        logger.error(f"Test error: {str(e)}")
+        raise
+    finally:
+        # Release memory pressure
+        memory_simulator.release_pressure()
+        # Log final results
+        test_results.log_summary()
+        
+        # Check for success criteria
+        if len(test_results.completed_urls) + len(test_results.failed_urls) < url_count:
+            logger.error(f"TEST FAILED: Not all URLs were processed. {url_count - len(test_results.completed_urls) - len(test_results.failed_urls)} URLs missing.")
+            return False
+            
+        logger.info("TEST PASSED: All URLs were processed without crashing.")
+        return True
+
+# Command-line entry point
+if __name__ == "__main__":
+    # Parse command line arguments
+    url_count = int(sys.argv[1]) if len(sys.argv) > 1 else 100
+    target_memory = float(sys.argv[2]) if len(sys.argv) > 2 else 85.0
+    
+    # Check if stream mode is specified
+    if len(sys.argv) > 3:
+        STREAM = sys.argv[3].lower() in ('true', 'yes', '1', 'stream')
+    
+    # Check if aggressive mode is specified
+    aggressive = False
+    if len(sys.argv) > 4:
+        aggressive = sys.argv[4].lower() in ('true', 'yes', '1', 'aggressive')
+    
+    print(f"Starting test with {url_count} URLs, {target_memory}% memory target")
+    print(f"Stream mode: {STREAM}, Aggressive: {aggressive}")
+    print("Logs will be written to the logs directory")
+    print("Live display starting now...")
+    
+    # Run the test
+    result = asyncio.run(run_memory_stress_test(
+        url_count=url_count, 
+        target_memory_percent=target_memory,
+        aggressive=aggressive
+    ))
+    
+    # Exit with status code
+    sys.exit(0 if result else 1)
\ No newline at end of file
diff --git a/tests/memory/test_docker_config_gen.py b/tests/memory/test_docker_config_gen.py
new file mode 100644
index 0000000..41beb30
--- /dev/null
+++ b/tests/memory/test_docker_config_gen.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""
+Quick sanity‑check for /config/dump endpoint.
+
+Usage:
+    python test_config_dump.py  [http://localhost:8020]
+
+If the server isn’t running, start it first:
+    uvicorn deploy.docker.server:app --port 8020
+"""
+
+import sys, json, textwrap, requests
+
+# BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8020"
+BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11235"
+URL  = f"{BASE.rstrip('/')}/config/dump"
+
+CASES = [
+    # --- CrawlRunConfig variants ---
+    "CrawlerRunConfig()",
+    "CrawlerRunConfig(stream=True, cache_mode=CacheMode.BYPASS)",
+    "CrawlerRunConfig(js_only=True, wait_until='networkidle')",
+
+    # --- BrowserConfig variants ---
+    "BrowserConfig()",
+    "BrowserConfig(headless=False, extra_args=['--disable-gpu'])",
+    "BrowserConfig(browser_mode='builtin', proxy_config={'server': 'http://1.2.3.4:8080'})",
+]
+
+for code in CASES:
+    print("\n===  POST:", code)
+    resp = requests.post(URL, json={"code": code}, timeout=15)
+    if resp.ok:
+        print(json.dumps(resp.json(), indent=2)[:400] + "...")
+    else:
+        print("ERROR", resp.status_code, resp.text[:200])
diff --git a/tests/memory/test_stress_api.py b/tests/memory/test_stress_api.py
new file mode 100644
index 0000000..1b4f1a9
--- /dev/null
+++ b/tests/memory/test_stress_api.py
@@ -0,0 +1,520 @@
+#!/usr/bin/env python3
+"""
+Stress test for Crawl4AI's Docker API server (/crawl and /crawl/stream endpoints).
+
+This version targets a running Crawl4AI API server, sending concurrent requests
+to test its ability to handle multiple crawl jobs simultaneously.
+It uses httpx for async HTTP requests and logs results per batch of requests,
+including server-side memory usage reported by the API.
+"""
+
+import asyncio
+import time
+import uuid
+import argparse
+import json
+import sys
+import os
+import shutil
+from typing import List, Dict, Optional, Union, AsyncGenerator, Tuple
+import httpx
+import pathlib # Import pathlib explicitly
+from rich.console import Console
+from rich.panel import Panel
+from rich.syntax import Syntax
+
+# --- Constants ---
+DEFAULT_API_URL = "http://localhost:11235" # Default port
+DEFAULT_API_URL = "http://localhost:8020" # Default port
+DEFAULT_URL_COUNT = 100
+DEFAULT_MAX_CONCURRENT_REQUESTS = 1
+DEFAULT_CHUNK_SIZE = 10
+DEFAULT_REPORT_PATH = "reports_api"
+DEFAULT_STREAM_MODE = True
+REQUEST_TIMEOUT = 180.0
+
+# Initialize Rich console
+console = Console()
+
+# --- API Health Check (Unchanged) ---
+async def check_server_health(client: httpx.AsyncClient, health_endpoint: str = "/health"):
+    """Check if the API server is healthy."""
+    console.print(f"[bold cyan]Checking API server health at {client.base_url}{health_endpoint}...[/]", end="")
+    try:
+        response = await client.get(health_endpoint, timeout=10.0)
+        response.raise_for_status()
+        health_data = response.json()
+        version = health_data.get('version', 'N/A')
+        console.print(f"[bold green] Server OK! Version: {version}[/]")
+        return True
+    except (httpx.RequestError, httpx.HTTPStatusError) as e:
+        console.print(f"\n[bold red]Server health check FAILED:[/]")
+        console.print(f"Error: {e}")
+        console.print(f"Is the server running and accessible at {client.base_url}?")
+        return False
+    except Exception as e:
+        console.print(f"\n[bold red]An unexpected error occurred during health check:[/]")
+        console.print(e)
+        return False
+
+# --- API Stress Test Class ---
+class ApiStressTest:
+    """Orchestrates the stress test by sending concurrent requests to the API."""
+
+    def __init__(
+        self,
+        api_url: str,
+        url_count: int,
+        max_concurrent_requests: int,
+        chunk_size: int,
+        report_path: str,
+        stream_mode: bool,
+    ):
+        self.api_base_url = api_url.rstrip('/')
+        self.url_count = url_count
+        self.max_concurrent_requests = max_concurrent_requests
+        self.chunk_size = chunk_size
+        self.report_path = pathlib.Path(report_path)
+        self.report_path.mkdir(parents=True, exist_ok=True)
+        self.stream_mode = stream_mode
+        
+        # Ignore repo path and set it to current file path
+        self.repo_path = pathlib.Path(__file__).parent.resolve()
+
+
+        self.test_id = time.strftime("%Y%m%d_%H%M%S")
+        self.results_summary = {
+            "test_id": self.test_id, "api_url": api_url, "url_count": url_count,
+            "max_concurrent_requests": max_concurrent_requests, "chunk_size": chunk_size,
+            "stream_mode": stream_mode, "start_time": "", "end_time": "",
+            "total_time_seconds": 0, "successful_requests": 0, "failed_requests": 0,
+            "successful_urls": 0, "failed_urls": 0, "total_urls_processed": 0,
+            "total_api_calls": 0,
+            "server_memory_metrics": { # To store aggregated server memory info
+                 "batch_mode_avg_delta_mb": None,
+                 "batch_mode_max_delta_mb": None,
+                 "stream_mode_avg_max_snapshot_mb": None,
+                 "stream_mode_max_max_snapshot_mb": None,
+                 "samples": [] # Store individual request memory results
+             }
+        }
+        self.http_client = httpx.AsyncClient(base_url=self.api_base_url, timeout=REQUEST_TIMEOUT, limits=httpx.Limits(max_connections=max_concurrent_requests + 5, max_keepalive_connections=max_concurrent_requests))
+
+    async def close_client(self):
+        """Close the httpx client."""
+        await self.http_client.aclose()
+
+    async def run(self) -> Dict:
+        """Run the API stress test."""
+        # No client memory tracker needed
+        urls_to_process = [f"https://httpbin.org/anything/{uuid.uuid4()}" for _ in range(self.url_count)]
+        url_chunks = [urls_to_process[i:i+self.chunk_size] for i in range(0, len(urls_to_process), self.chunk_size)]
+
+        self.results_summary["start_time"] = time.strftime("%Y-%m-%d %H:%M:%S")
+        start_time = time.time()
+
+        console.print(f"\n[bold cyan]Crawl4AI API Stress Test - {self.url_count} URLs, {self.max_concurrent_requests} concurrent requests[/bold cyan]")
+        console.print(f"[bold cyan]Target API:[/bold cyan] {self.api_base_url}, [bold cyan]Mode:[/bold cyan] {'Streaming' if self.stream_mode else 'Batch'}, [bold cyan]URLs per Request:[/bold cyan] {self.chunk_size}")
+        # Removed client memory log
+
+        semaphore = asyncio.Semaphore(self.max_concurrent_requests)
+
+        # Updated Batch logging header
+        console.print("\n[bold]API Request Batch Progress:[/bold]")
+        # Adjusted spacing and added Peak
+        console.print("[bold] Batch | Progress | SrvMem Peak / Δ|Max (MB) | Reqs/sec | S/F URLs | Time (s) | Status  [/bold]")
+        # Adjust separator length if needed, looks okay for now
+        console.print("─" * 95) 
+
+        # No client memory monitor task needed
+
+        tasks = []
+        total_api_calls = len(url_chunks)
+        self.results_summary["total_api_calls"] = total_api_calls
+
+        try:
+            for i, chunk in enumerate(url_chunks):
+                task = asyncio.create_task(self._make_api_request(
+                    chunk=chunk,
+                    batch_idx=i + 1,
+                    total_batches=total_api_calls,
+                    semaphore=semaphore
+                    # No memory tracker passed
+                ))
+                tasks.append(task)
+
+            api_results = await asyncio.gather(*tasks)
+
+            # Process aggregated results including server memory
+            total_successful_requests = sum(1 for r in api_results if r['request_success'])
+            total_failed_requests = total_api_calls - total_successful_requests
+            total_successful_urls = sum(r['success_urls'] for r in api_results)
+            total_failed_urls = sum(r['failed_urls'] for r in api_results)
+            total_urls_processed = total_successful_urls + total_failed_urls
+
+            # Aggregate server memory metrics
+            valid_samples = [r for r in api_results if r.get('server_delta_or_max_mb') is not None] # Filter results with valid mem data
+            self.results_summary["server_memory_metrics"]["samples"] = valid_samples # Store raw samples with both peak and delta/max
+
+            if valid_samples:
+                 delta_or_max_values = [r['server_delta_or_max_mb'] for r in valid_samples]
+                 if self.stream_mode:
+                     # Stream mode: delta_or_max holds max snapshot
+                     self.results_summary["server_memory_metrics"]["stream_mode_avg_max_snapshot_mb"] = sum(delta_or_max_values) / len(delta_or_max_values)
+                     self.results_summary["server_memory_metrics"]["stream_mode_max_max_snapshot_mb"] = max(delta_or_max_values)
+                 else: # Batch mode
+                     # delta_or_max holds delta
+                     self.results_summary["server_memory_metrics"]["batch_mode_avg_delta_mb"] = sum(delta_or_max_values) / len(delta_or_max_values)
+                     self.results_summary["server_memory_metrics"]["batch_mode_max_delta_mb"] = max(delta_or_max_values)
+
+                     # Aggregate peak values for batch mode
+                     peak_values = [r['server_peak_memory_mb'] for r in valid_samples if r.get('server_peak_memory_mb') is not None]
+                     if peak_values:
+                          self.results_summary["server_memory_metrics"]["batch_mode_avg_peak_mb"] = sum(peak_values) / len(peak_values)
+                          self.results_summary["server_memory_metrics"]["batch_mode_max_peak_mb"] = max(peak_values)
+
+
+            self.results_summary.update({
+                "successful_requests": total_successful_requests,
+                "failed_requests": total_failed_requests,
+                "successful_urls": total_successful_urls,
+                "failed_urls": total_failed_urls,
+                "total_urls_processed": total_urls_processed,
+            })
+
+        except Exception as e:
+             console.print(f"[bold red]An error occurred during task execution: {e}[/bold red]")
+             import traceback
+             traceback.print_exc()
+        # No finally block needed for monitor task
+
+        end_time = time.time()
+        self.results_summary.update({
+            "end_time": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "total_time_seconds": end_time - start_time,
+            # No client memory report
+        })
+        self._save_results()
+        return self.results_summary
+
+    async def _make_api_request(
+        self,
+        chunk: List[str],
+        batch_idx: int,
+        total_batches: int,
+        semaphore: asyncio.Semaphore
+        # No memory tracker
+    ) -> Dict:
+        """Makes a single API request for a chunk of URLs, handling concurrency and logging server memory."""
+        request_success = False
+        success_urls = 0
+        failed_urls = 0
+        status = "Pending"
+        status_color = "grey"
+        server_memory_metric = None # Store delta (batch) or max snapshot (stream)
+        api_call_start_time = time.time()
+
+        async with semaphore:
+            try:
+                # No client memory sampling
+
+                endpoint = "/crawl/stream" if self.stream_mode else "/crawl"
+                payload = {
+                    "urls": chunk,
+                    "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+                    "crawler_config": {
+                        "type": "CrawlerRunConfig",
+                        "params": {"cache_mode": "BYPASS", "stream": self.stream_mode}
+                    }
+                }
+
+                if self.stream_mode:
+                    max_server_mem_snapshot = 0.0 # Track max memory seen in this stream
+                    async with self.http_client.stream("POST", endpoint, json=payload) as response:
+                        initial_status_code = response.status_code
+                        response.raise_for_status()
+
+                        completed_marker_received = False
+                        async for line in response.aiter_lines():
+                            if line:
+                                try:
+                                    data = json.loads(line)
+                                    if data.get("status") == "completed":
+                                        completed_marker_received = True
+                                        break
+                                    elif data.get("url"):
+                                        if data.get("success"): success_urls += 1
+                                        else: failed_urls += 1
+                                        # Extract server memory snapshot per result
+                                        mem_snapshot = data.get('server_memory_mb')
+                                        if mem_snapshot is not None:
+                                            max_server_mem_snapshot = max(max_server_mem_snapshot, float(mem_snapshot))
+                                except json.JSONDecodeError:
+                                    console.print(f"[Batch {batch_idx}] [red]Stream decode error for line:[/red] {line}")
+                                    failed_urls = len(chunk)
+                                    break
+                        request_success = completed_marker_received
+                        if not request_success:
+                             failed_urls = len(chunk) - success_urls
+                        server_memory_metric = max_server_mem_snapshot # Use max snapshot for stream logging
+
+                else: # Batch mode
+                    response = await self.http_client.post(endpoint, json=payload)
+                    response.raise_for_status()
+                    data = response.json()
+
+                    # Extract server memory delta from the response
+                    server_memory_metric = data.get('server_memory_delta_mb')
+                    server_peak_mem_mb = data.get('server_peak_memory_mb') 
+
+                    if data.get("success") and "results" in data:
+                        request_success = True
+                        results_list = data.get("results", [])
+                        for result_item in results_list:
+                            if result_item.get("success"): success_urls += 1
+                            else: failed_urls += 1
+                        if len(results_list) != len(chunk):
+                             console.print(f"[Batch {batch_idx}] [yellow]Warning: Result count ({len(results_list)}) doesn't match URL count ({len(chunk)})[/yellow]")
+                             failed_urls = len(chunk) - success_urls
+                    else:
+                        request_success = False
+                        failed_urls = len(chunk)
+                        # Try to get memory from error detail if available
+                        detail = data.get('detail')
+                        if isinstance(detail, str):
+                            try: detail_json = json.loads(detail)
+                            except: detail_json = {}
+                        elif isinstance(detail, dict):
+                            detail_json = detail
+                        else: detail_json = {}
+                        server_peak_mem_mb = detail_json.get('server_peak_memory_mb', None)
+                        server_memory_metric = detail_json.get('server_memory_delta_mb', None)
+                        console.print(f"[Batch {batch_idx}] [red]API request failed:[/red] {detail_json.get('error', 'No details')}")
+
+
+            except httpx.HTTPStatusError as e:
+                request_success = False
+                failed_urls = len(chunk)
+                console.print(f"[Batch {batch_idx}] [bold red]HTTP Error {e.response.status_code}:[/] {e.request.url}")
+                try:
+                    error_detail = e.response.json()
+                    # Attempt to extract memory info even from error responses
+                    detail_content = error_detail.get('detail', {})
+                    if isinstance(detail_content, str): # Handle if detail is stringified JSON
+                         try: detail_content = json.loads(detail_content)
+                         except: detail_content = {}
+                    server_memory_metric = detail_content.get('server_memory_delta_mb', None)
+                    server_peak_mem_mb = detail_content.get('server_peak_memory_mb', None)
+                    console.print(f"Response: {error_detail}")
+                except Exception:
+                     console.print(f"Response Text: {e.response.text[:200]}...")
+            except httpx.RequestError as e:
+                request_success = False
+                failed_urls = len(chunk)
+                console.print(f"[Batch {batch_idx}] [bold red]Request Error:[/bold] {e.request.url} - {e}")
+            except Exception as e:
+                request_success = False
+                failed_urls = len(chunk)
+                console.print(f"[Batch {batch_idx}] [bold red]Unexpected Error:[/bold] {e}")
+                import traceback
+                traceback.print_exc()
+
+            finally:
+                api_call_time = time.time() - api_call_start_time
+                total_processed_urls = success_urls + failed_urls
+
+                if request_success and failed_urls == 0: status_color, status = "green", "Success"
+                elif request_success and success_urls > 0: status_color, status = "yellow", "Partial"
+                else: status_color, status = "red", "Failed"
+
+                current_total_urls = batch_idx * self.chunk_size
+                progress_pct = min(100.0, (current_total_urls / self.url_count) * 100)
+                reqs_per_sec = 1.0 / api_call_time if api_call_time > 0 else float('inf')
+
+                # --- New Memory Formatting ---
+                mem_display = " N/A " # Default
+                peak_mem_value = None
+                delta_or_max_value = None
+
+                if self.stream_mode:
+                    # server_memory_metric holds max snapshot for stream
+                    if server_memory_metric is not None:
+                        mem_display = f"{server_memory_metric:.1f} (Max)"
+                        delta_or_max_value = server_memory_metric # Store for aggregation
+                else: # Batch mode - expect peak and delta
+                    # We need to get peak and delta from the API response
+                    peak_mem_value = locals().get('server_peak_mem_mb', None) # Get from response data if available
+                    delta_value = server_memory_metric # server_memory_metric holds delta for batch
+
+                    if peak_mem_value is not None and delta_value is not None:
+                        mem_display = f"{peak_mem_value:.1f} / {delta_value:+.1f}"
+                        delta_or_max_value = delta_value # Store delta for aggregation
+                    elif peak_mem_value is not None:
+                         mem_display = f"{peak_mem_value:.1f} / N/A"
+                    elif delta_value is not None:
+                         mem_display = f"N/A / {delta_value:+.1f}"
+                         delta_or_max_value = delta_value # Store delta for aggregation
+
+                # --- Updated Print Statement with Adjusted Padding ---
+                console.print(
+                    f" {batch_idx:<5} | {progress_pct:6.1f}% | {mem_display:>24} | {reqs_per_sec:8.1f} | " # Increased width for memory column
+                    f"{success_urls:^7}/{failed_urls:<6} | {api_call_time:8.2f} | [{status_color}]{status:<7}[/{status_color}] " # Added trailing space
+                )
+
+                # --- Updated Return Dictionary ---
+                return_data = {
+                    "batch_idx": batch_idx,
+                    "request_success": request_success,
+                    "success_urls": success_urls,
+                    "failed_urls": failed_urls,
+                    "time": api_call_time,
+                    # Return both peak (if available) and delta/max
+                    "server_peak_memory_mb": peak_mem_value, # Will be None for stream mode
+                    "server_delta_or_max_mb": delta_or_max_value # Delta for batch, Max for stream
+                }
+                # Add back the specific batch mode delta if needed elsewhere, but delta_or_max covers it
+                # if not self.stream_mode:
+                #    return_data["server_memory_delta_mb"] = delta_value
+                return return_data
+
+    # No _periodic_memory_sample needed
+
+    def _save_results(self) -> None:
+        """Saves the results summary to a JSON file."""
+        results_path = self.report_path / f"api_test_summary_{self.test_id}.json"
+        try:
+            # No client memory path to convert
+            with open(results_path, 'w', encoding='utf-8') as f:
+                json.dump(self.results_summary, f, indent=2, default=str)
+        except Exception as e:
+            console.print(f"[bold red]Failed to save results summary: {e}[/bold red]")
+
+
+# --- run_full_test Function ---
+async def run_full_test(args):
+    """Runs the full API stress test process."""
+    client = httpx.AsyncClient(base_url=args.api_url, timeout=REQUEST_TIMEOUT)
+
+    if not await check_server_health(client):
+        console.print("[bold red]Aborting test due to server health check failure.[/]")
+        await client.aclose()
+        return
+    await client.aclose()
+
+    test = ApiStressTest(
+        api_url=args.api_url,
+        url_count=args.urls,
+        max_concurrent_requests=args.max_concurrent_requests,
+        chunk_size=args.chunk_size,
+        report_path=args.report_path,
+        stream_mode=args.stream,
+    )
+    results = {}
+    try:
+        results = await test.run()
+    finally:
+        await test.close_client()
+
+    if not results:
+        console.print("[bold red]Test did not produce results.[/bold red]")
+        return
+
+    console.print("\n" + "=" * 80)
+    console.print("[bold green]API Stress Test Completed[/bold green]")
+    console.print("=" * 80)
+
+    success_rate_reqs = results["successful_requests"] / results["total_api_calls"] * 100 if results["total_api_calls"] > 0 else 0
+    success_rate_urls = results["successful_urls"] / results["url_count"] * 100 if results["url_count"] > 0 else 0
+    urls_per_second = results["total_urls_processed"] / results["total_time_seconds"] if results["total_time_seconds"] > 0 else 0
+    reqs_per_second = results["total_api_calls"] / results["total_time_seconds"] if results["total_time_seconds"] > 0 else 0
+
+
+    console.print(f"[bold cyan]Test ID:[/bold cyan] {results['test_id']}")
+    console.print(f"[bold cyan]Target API:[/bold cyan] {results['api_url']}")
+    console.print(f"[bold cyan]Configuration:[/bold cyan] {results['url_count']} URLs, {results['max_concurrent_requests']} concurrent client requests, URLs/Req: {results['chunk_size']}, Stream: {results['stream_mode']}")
+    console.print(f"[bold cyan]API Requests:[/bold cyan] {results['successful_requests']} successful, {results['failed_requests']} failed ({results['total_api_calls']} total, {success_rate_reqs:.1f}% success)")
+    console.print(f"[bold cyan]URL Processing:[/bold cyan] {results['successful_urls']} successful, {results['failed_urls']} failed ({results['total_urls_processed']} processed, {success_rate_urls:.1f}% success)")
+    console.print(f"[bold cyan]Performance:[/bold cyan] {results['total_time_seconds']:.2f}s total | Avg Reqs/sec: {reqs_per_second:.2f} | Avg URLs/sec: {urls_per_second:.2f}")
+
+    # Report Server Memory
+    mem_metrics = results.get("server_memory_metrics", {})
+    mem_samples = mem_metrics.get("samples", [])
+    if mem_samples:
+         num_samples = len(mem_samples)
+         if results['stream_mode']:
+             avg_mem = mem_metrics.get("stream_mode_avg_max_snapshot_mb")
+             max_mem = mem_metrics.get("stream_mode_max_max_snapshot_mb")
+             avg_str = f"{avg_mem:.1f}" if avg_mem is not None else "N/A"
+             max_str = f"{max_mem:.1f}" if max_mem is not None else "N/A"
+             console.print(f"[bold cyan]Server Memory (Stream):[/bold cyan] Avg Max Snapshot: {avg_str} MB | Max Max Snapshot: {max_str} MB (across {num_samples} requests)")
+         else: # Batch mode
+             avg_delta = mem_metrics.get("batch_mode_avg_delta_mb")
+             max_delta = mem_metrics.get("batch_mode_max_delta_mb")
+             avg_peak = mem_metrics.get("batch_mode_avg_peak_mb")
+             max_peak = mem_metrics.get("batch_mode_max_peak_mb")
+
+             avg_delta_str = f"{avg_delta:.1f}" if avg_delta is not None else "N/A"
+             max_delta_str = f"{max_delta:.1f}" if max_delta is not None else "N/A"
+             avg_peak_str = f"{avg_peak:.1f}" if avg_peak is not None else "N/A"
+             max_peak_str = f"{max_peak:.1f}" if max_peak is not None else "N/A"
+
+             console.print(f"[bold cyan]Server Memory (Batch):[/bold cyan] Avg Peak: {avg_peak_str} MB | Max Peak: {max_peak_str} MB | Avg Delta: {avg_delta_str} MB | Max Delta: {max_delta_str} MB (across {num_samples} requests)")
+    else:
+        console.print("[bold cyan]Server Memory:[/bold cyan] No memory data reported by server.")
+
+
+    # No client memory report
+    summary_path = pathlib.Path(args.report_path) / f"api_test_summary_{results['test_id']}.json"
+    console.print(f"[bold green]Results summary saved to {summary_path}[/bold green]")
+
+    if results["failed_requests"] > 0:
+        console.print(f"\n[bold yellow]Warning: {results['failed_requests']} API requests failed ({100-success_rate_reqs:.1f}% failure rate)[/bold yellow]")
+    if results["failed_urls"] > 0:
+         console.print(f"[bold yellow]Warning: {results['failed_urls']} URLs failed to process ({100-success_rate_urls:.1f}% URL failure rate)[/bold yellow]")
+    if results["total_urls_processed"] < results["url_count"]:
+        console.print(f"\n[bold red]Error: Only {results['total_urls_processed']} out of {results['url_count']} target URLs were processed![/bold red]")
+
+
+# --- main Function (Argument parsing mostly unchanged) ---
+def main():
+    """Main entry point for the script."""
+    parser = argparse.ArgumentParser(description="Crawl4AI API Server Stress Test")
+
+    parser.add_argument("--api-url", type=str, default=DEFAULT_API_URL, help=f"Base URL of the Crawl4AI API server (default: {DEFAULT_API_URL})")
+    parser.add_argument("--urls", type=int, default=DEFAULT_URL_COUNT, help=f"Total number of unique URLs to process via API calls (default: {DEFAULT_URL_COUNT})")
+    parser.add_argument("--max-concurrent-requests", type=int, default=DEFAULT_MAX_CONCURRENT_REQUESTS, help=f"Maximum concurrent API requests from this client (default: {DEFAULT_MAX_CONCURRENT_REQUESTS})")
+    parser.add_argument("--chunk-size", type=int, default=DEFAULT_CHUNK_SIZE, help=f"Number of URLs per API request payload (default: {DEFAULT_CHUNK_SIZE})")
+    parser.add_argument("--stream", action="store_true", default=DEFAULT_STREAM_MODE, help=f"Use the /crawl/stream endpoint instead of /crawl (default: {DEFAULT_STREAM_MODE})")
+    parser.add_argument("--report-path", type=str, default=DEFAULT_REPORT_PATH, help=f"Path to save reports and logs (default: {DEFAULT_REPORT_PATH})")
+    parser.add_argument("--clean-reports", action="store_true", help="Clean up report directory before running")
+
+    args = parser.parse_args()
+
+    console.print("[bold underline]Crawl4AI API Stress Test Configuration[/bold underline]")
+    console.print(f"API URL: {args.api_url}")
+    console.print(f"Total URLs: {args.urls}, Concurrent Client Requests: {args.max_concurrent_requests}, URLs per Request: {args.chunk_size}")
+    console.print(f"Mode: {'Streaming' if args.stream else 'Batch'}")
+    console.print(f"Report Path: {args.report_path}")
+    console.print("-" * 40)
+    if args.clean_reports: console.print("[cyan]Option: Clean reports before test[/cyan]")
+    console.print("-" * 40)
+
+    if args.clean_reports:
+        report_dir = pathlib.Path(args.report_path)
+        if report_dir.exists():
+            console.print(f"[yellow]Cleaning up reports directory: {args.report_path}[/yellow]")
+            shutil.rmtree(args.report_path)
+        report_dir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        asyncio.run(run_full_test(args))
+    except KeyboardInterrupt:
+        console.print("\n[bold yellow]Test interrupted by user.[/bold yellow]")
+    except Exception as e:
+        console.print(f"\n[bold red]An unexpected error occurred:[/bold red] {e}")
+        import traceback
+        traceback.print_exc()
+
+if __name__ == "__main__":
+    # No need to modify sys.path for SimpleMemoryTracker as it's removed
+    main()
\ No newline at end of file
diff --git a/tests/memory/test_stress_api_xs.py b/tests/memory/test_stress_api_xs.py
new file mode 100644
index 0000000..2724888
--- /dev/null
+++ b/tests/memory/test_stress_api_xs.py
@@ -0,0 +1,203 @@
+"""Lite Crawl4AI API stress‑tester.
+
+✔ batch or stream mode (single unified path)
+✔ global stats + JSON summary
+✔ rich table progress
+✔ Typer CLI with presets (quick / soak)
+
+Usage examples:
+    python api_stress_test.py               # uses quick preset
+    python api_stress_test.py soak          # 5 K URLs stress run
+    python api_stress_test.py --urls 200 --concurrent 10 --chunk 20
+"""
+
+from __future__ import annotations
+
+import asyncio, json, time, uuid, pathlib, statistics
+from typing import List, Dict, Optional
+
+import httpx, typer
+from rich.console import Console
+from rich.table import Table
+
+# ───────────────────────── defaults / presets ──────────────────────────
+PRESETS = {
+    "quick": dict(urls=1, concurrent=1, chunk=1, stream=False),
+    "debug": dict(urls=10, concurrent=2, chunk=5, stream=False),
+    "soak": dict(urls=5000, concurrent=20, chunk=50, stream=True),
+}
+
+API_HEALTH_ENDPOINT = "/health"
+REQUEST_TIMEOUT = 180.0
+
+console = Console()
+app = typer.Typer(add_completion=False, rich_markup_mode="rich")
+
+# ───────────────────────── helpers ─────────────────────────────────────
+async def _check_health(client: httpx.AsyncClient) -> None:
+    resp = await client.get(API_HEALTH_ENDPOINT, timeout=10)
+    resp.raise_for_status()
+    console.print(f"[green]Server healthy — version {resp.json().get('version','?')}[/]")
+
+async def _iter_results(resp: httpx.Response, stream: bool):
+    """Yield result dicts from batch JSON or ND‑JSON stream."""
+    if stream:
+        async for line in resp.aiter_lines():
+            if not line:
+                continue
+            rec = json.loads(line)
+            if rec.get("status") == "completed":
+                break
+            yield rec
+    else:
+        data = resp.json()
+        for rec in data.get("results", []):
+            yield rec, data  # rec + whole payload for memory delta/peak
+
+async def _consume_stream(resp: httpx.Response) -> Dict:
+    stats = {"success_urls": 0, "failed_urls": 0, "mem_metric": 0.0}
+    async for line in resp.aiter_lines():
+        if not line:
+            continue
+        rec = json.loads(line)
+        if rec.get("status") == "completed":
+            break
+        if rec.get("success"):
+            stats["success_urls"] += 1
+        else:
+            stats["failed_urls"] += 1
+        mem = rec.get("server_memory_mb")
+        if mem is not None:
+            stats["mem_metric"] = max(stats["mem_metric"], float(mem))
+    return stats
+
+def _consume_batch(body: Dict) -> Dict:
+    stats = {"success_urls": 0, "failed_urls": 0}
+    for rec in body.get("results", []):
+        if rec.get("success"):
+            stats["success_urls"] += 1
+        else:
+            stats["failed_urls"] += 1
+    stats["mem_metric"] = body.get("server_memory_delta_mb")
+    stats["peak"] = body.get("server_peak_memory_mb")
+    return stats
+
+async def _fetch_chunk(
+    client: httpx.AsyncClient,
+    urls: List[str],
+    stream: bool,
+    semaphore: asyncio.Semaphore,
+) -> Dict:
+    endpoint = "/crawl/stream" if stream else "/crawl"
+    payload = {
+        "urls": urls,
+        "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
+        "crawler_config": {"type": "CrawlerRunConfig",
+                           "params": {"cache_mode": "BYPASS", "stream": stream}},
+    }
+
+    async with semaphore:
+        start = time.perf_counter()
+
+        if stream:
+            # ---- streaming request ----
+            async with client.stream("POST", endpoint, json=payload) as resp:
+                resp.raise_for_status()
+                stats = await _consume_stream(resp)
+        else:
+            # ---- batch request ----
+            resp = await client.post(endpoint, json=payload)
+            resp.raise_for_status()
+            stats = _consume_batch(resp.json())
+
+        stats["elapsed"] = time.perf_counter() - start
+        return stats
+
+
+# ───────────────────────── core runner ─────────────────────────────────
+async def _run(api: str, urls: int, concurrent: int, chunk: int, stream: bool, report: pathlib.Path):
+    client = httpx.AsyncClient(base_url=api, timeout=REQUEST_TIMEOUT, limits=httpx.Limits(max_connections=concurrent+5))
+    await _check_health(client)
+
+    url_list = [f"https://httpbin.org/anything/{uuid.uuid4()}" for _ in range(urls)]
+    chunks = [url_list[i:i+chunk] for i in range(0, len(url_list), chunk)]
+    sem = asyncio.Semaphore(concurrent)
+
+    table = Table(show_header=True, header_style="bold magenta")
+    table.add_column("Batch", style="dim", width=6)
+    table.add_column("Success/Fail", width=12)
+    table.add_column("Mem", width=14)
+    table.add_column("Time (s)")
+
+    agg_success = agg_fail = 0
+    deltas, peaks = [], []
+
+    start = time.perf_counter()
+    tasks = [asyncio.create_task(_fetch_chunk(client, c, stream, sem)) for c in chunks]
+    for idx, coro in enumerate(asyncio.as_completed(tasks), 1):
+        res = await coro
+        agg_success += res["success_urls"]
+        agg_fail += res["failed_urls"]
+        if res["mem_metric"] is not None:
+            deltas.append(res["mem_metric"])
+        if res["peak"] is not None:
+            peaks.append(res["peak"])
+
+        mem_txt = f"{res['mem_metric']:.1f}" if res["mem_metric"] is not None else "‑"
+        if res["peak"] is not None:
+            mem_txt = f"{res['peak']:.1f}/{mem_txt}"
+
+        table.add_row(str(idx), f"{res['success_urls']}/{res['failed_urls']}", mem_txt, f"{res['elapsed']:.2f}")
+
+    console.print(table)
+    total_time = time.perf_counter() - start
+
+    summary = {
+        "urls": urls,
+        "concurrent": concurrent,
+        "chunk": chunk,
+        "stream": stream,
+        "success_urls": agg_success,
+        "failed_urls": agg_fail,
+        "elapsed_sec": round(total_time, 2),
+        "avg_mem": round(statistics.mean(deltas), 2) if deltas else None,
+        "max_mem": max(deltas) if deltas else None,
+        "avg_peak": round(statistics.mean(peaks), 2) if peaks else None,
+        "max_peak": max(peaks) if peaks else None,
+    }
+    console.print("\n[bold green]Done:[/]" , summary)
+
+    report.mkdir(parents=True, exist_ok=True)
+    path = report / f"api_test_{int(time.time())}.json"
+    path.write_text(json.dumps(summary, indent=2))
+    console.print(f"[green]Summary → {path}")
+
+    await client.aclose()
+
+# ───────────────────────── Typer CLI ──────────────────────────────────
+@app.command()
+def main(
+    preset: str = typer.Argument("quick", help="quick / debug / soak or custom"),
+    api_url: str = typer.Option("http://localhost:8020", show_default=True),
+    urls: int = typer.Option(None, help="Total URLs to crawl"),
+    concurrent: int = typer.Option(None, help="Concurrent API requests"),
+    chunk: int = typer.Option(None, help="URLs per request"),
+    stream: bool = typer.Option(None, help="Use /crawl/stream"),
+    report: pathlib.Path = typer.Option("reports_api", help="Where to save JSON summary"),
+):
+    """Run a stress test against a running Crawl4AI API server."""
+    if preset not in PRESETS and any(v is None for v in (urls, concurrent, chunk, stream)):
+        console.print(f"[red]Unknown preset '{preset}' and custom params missing[/]")
+        raise typer.Exit(1)
+
+    cfg = PRESETS.get(preset, {})
+    urls = urls or cfg.get("urls")
+    concurrent = concurrent or cfg.get("concurrent")
+    chunk = chunk or cfg.get("chunk")
+    stream = stream if stream is not None else cfg.get("stream", False)
+
+    console.print(f"[cyan]API:[/] {api_url} | URLs: {urls} | Concurrency: {concurrent} | Chunk: {chunk} | Stream: {stream}")
+    asyncio.run(_run(api_url, urls, concurrent, chunk, stream, report))
+
+if __name__ == "__main__":
+    app()
diff --git a/tests/memory/test_stress_docker_api.py b/tests/memory/test_stress_docker_api.py
new file mode 100644
index 0000000..05b3bea
--- /dev/null
+++ b/tests/memory/test_stress_docker_api.py
@@ -0,0 +1,129 @@
+"""
+Crawl4AI Docker API stress tester.
+
+Examples
+--------
+python test_stress_docker_api.py --urls 1000 --concurrency 32
+python test_stress_docker_api.py --urls 1000 --concurrency 32 --stream
+python test_stress_docker_api.py --base-url http://10.0.0.42:11235 --http2
+"""
+
+import argparse, asyncio, json, secrets, statistics, time
+from typing import List, Tuple
+import httpx
+from rich.console import Console
+from rich.progress import Progress, BarColumn, TimeElapsedColumn, TimeRemainingColumn
+from rich.table import Table
+
+console = Console()
+
+
+# ───────────────────────── helpers ─────────────────────────
+def make_fake_urls(n: int) -> List[str]:
+    base = "https://httpbin.org/anything/"
+    return [f"{base}{secrets.token_hex(8)}" for _ in range(n)]
+
+
+async def fire(
+    client: httpx.AsyncClient, endpoint: str, payload: dict, sem: asyncio.Semaphore
+) -> Tuple[bool, float]:
+    async with sem:
+        print(f"POST {endpoint} with {len(payload['urls'])} URLs")
+        t0 = time.perf_counter()
+        try:
+            if endpoint.endswith("/stream"):
+                async with client.stream("POST", endpoint, json=payload) as r:
+                    r.raise_for_status()
+                    async for _ in r.aiter_lines():
+                        pass
+            else:
+                r = await client.post(endpoint, json=payload)                
+                r.raise_for_status()
+            return True, time.perf_counter() - t0
+        except Exception:
+            return False, time.perf_counter() - t0
+
+
+def pct(lat: List[float], p: float) -> str:
+    """Return percentile string even for tiny samples."""
+    if not lat:
+        return "-"
+    if len(lat) == 1:
+        return f"{lat[0]:.2f}s"
+    lat_sorted = sorted(lat)
+    k = (p / 100) * (len(lat_sorted) - 1)
+    lo = int(k)
+    hi = min(lo + 1, len(lat_sorted) - 1)
+    frac = k - lo
+    val = lat_sorted[lo] * (1 - frac) + lat_sorted[hi] * frac
+    return f"{val:.2f}s"
+
+
+# ───────────────────────── main ─────────────────────────
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Stress test Crawl4AI Docker API")
+    p.add_argument("--urls", type=int, default=100, help="number of URLs")
+    p.add_argument("--concurrency", type=int, default=1, help="max POSTs in flight")
+    p.add_argument("--chunk-size", type=int, default=50, help="URLs per request")
+    p.add_argument("--base-url", default="http://localhost:11235", help="API root")
+    # p.add_argument("--base-url", default="http://localhost:8020", help="API root")
+    p.add_argument("--stream", action="store_true", help="use /crawl/stream")
+    p.add_argument("--http2", action="store_true", help="enable HTTP/2")
+    p.add_argument("--headless", action="store_true", default=True)
+    return p.parse_args()
+
+
+async def main() -> None:
+    args = parse_args()
+
+    urls = make_fake_urls(args.urls)
+    batches = [urls[i : i + args.chunk_size] for i in range(0, len(urls), args.chunk_size)]
+    endpoint = "/crawl/stream" if args.stream else "/crawl"
+    sem = asyncio.Semaphore(args.concurrency)
+
+    async with httpx.AsyncClient(base_url=args.base_url, http2=args.http2, timeout=None) as client:
+        with Progress(
+            "[progress.description]{task.description}",
+            BarColumn(),
+            "[progress.percentage]{task.percentage:>3.0f}%",
+            TimeElapsedColumn(),
+            TimeRemainingColumn(),
+        ) as progress:
+            task_id = progress.add_task("[cyan]bombarding…", total=len(batches))
+            tasks = []
+            for chunk in batches:
+                payload = {
+                    "urls": chunk,
+                    "browser_config": {"type": "BrowserConfig", "params": {"headless": args.headless}},
+                    "crawler_config": {"type": "CrawlerRunConfig", "params": {"cache_mode": "BYPASS", "stream": args.stream}},
+                }
+                tasks.append(asyncio.create_task(fire(client, endpoint, payload, sem)))
+                progress.advance(task_id)
+
+            results = await asyncio.gather(*tasks)
+
+    ok_latencies = [dt for ok, dt in results if ok]
+    err_count = sum(1 for ok, _ in results if not ok)
+
+    table = Table(title="Docker API Stress‑Test Summary")
+    table.add_column("total", justify="right")
+    table.add_column("errors", justify="right")
+    table.add_column("p50", justify="right")
+    table.add_column("p95", justify="right")
+    table.add_column("max", justify="right")
+
+    table.add_row(
+        str(len(results)),
+        str(err_count),
+        pct(ok_latencies, 50),
+        pct(ok_latencies, 95),
+        f"{max(ok_latencies):.2f}s" if ok_latencies else "-",
+    )
+    console.print(table)
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        console.print("\n[yellow]aborted by user[/]")
diff --git a/tests/memory/test_stress_sdk.py b/tests/memory/test_stress_sdk.py
new file mode 100644
index 0000000..14da94a
--- /dev/null
+++ b/tests/memory/test_stress_sdk.py
@@ -0,0 +1,500 @@
+#!/usr/bin/env python3
+"""
+Stress test for Crawl4AI's arun_many and dispatcher system.
+This version uses a local HTTP server and focuses on testing
+the SDK's ability to handle multiple URLs concurrently, with per-batch logging.
+"""
+
+import asyncio
+import os
+import time
+import pathlib
+import random
+import secrets
+import argparse
+import json
+import sys
+import subprocess
+import signal
+from typing import List, Dict, Optional, Union, AsyncGenerator
+import shutil
+from rich.console import Console
+
+# Crawl4AI components
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    BrowserConfig,
+    MemoryAdaptiveDispatcher,
+    CrawlerMonitor,
+    DisplayMode,
+    CrawlResult,
+    RateLimiter,
+    CacheMode,
+)
+
+# Constants
+DEFAULT_SITE_PATH = "test_site"
+DEFAULT_PORT = 8000
+DEFAULT_MAX_SESSIONS = 16
+DEFAULT_URL_COUNT = 1
+DEFAULT_CHUNK_SIZE = 1 # Define chunk size for batch logging
+DEFAULT_REPORT_PATH = "reports"
+DEFAULT_STREAM_MODE = False
+DEFAULT_MONITOR_MODE = "DETAILED"
+
+# Initialize Rich console
+console = Console()
+
+# --- SiteGenerator Class (Unchanged) ---
+class SiteGenerator:
+    """Generates a local test site with heavy pages for stress testing."""
+
+    def __init__(self, site_path: str = DEFAULT_SITE_PATH, page_count: int = DEFAULT_URL_COUNT):
+        self.site_path = pathlib.Path(site_path)
+        self.page_count = page_count
+        self.images_dir = self.site_path / "images"
+        self.lorem_words = " ".join("lorem ipsum dolor sit amet " * 100).split()
+
+        self.html_template = """
+
+
+    Test Page {page_num}
+    
+
+
+    
      Test Page {page_num}
      
+    {paragraphs}
+    {images}
+
+
+"""
+
+    def generate_site(self) -> None:
+        self.site_path.mkdir(parents=True, exist_ok=True)
+        self.images_dir.mkdir(exist_ok=True)
+        console.print(f"Generating {self.page_count} test pages...")
+        for i in range(self.page_count):
+            paragraphs = "\n".join(f"
      {' '.join(random.choices(self.lorem_words, k=200))}
      " for _ in range(5))
+            images = "\n".join(f'Random image {j}' for j in range(3))
+            page_path = self.site_path / f"page_{i}.html"
+            page_path.write_text(self.html_template.format(page_num=i, paragraphs=paragraphs, images=images), encoding="utf-8")
+            if (i + 1) % (self.page_count // 10 or 1) == 0 or i == self.page_count - 1:
+                 console.print(f"Generated {i+1}/{self.page_count} pages")
+        self._create_index_page()
+        console.print(f"[bold green]Successfully generated {self.page_count} test pages in [cyan]{self.site_path}[/cyan][/bold green]")
+
+    def _create_index_page(self) -> None:
+        index_content = """Test Site Index
      Test Site Index
      This is an automatically generated site for testing Crawl4AI.
      \n"""
+        for i in range(self.page_count):
+            index_content += f'        Test Page {i}
      \n'
+        index_content += """    
      """
+        (self.site_path / "index.html").write_text(index_content, encoding="utf-8")
+
+# --- LocalHttpServer Class (Unchanged) ---
+class LocalHttpServer:
+    """Manages a local HTTP server for serving test pages."""
+    def __init__(self, site_path: str = DEFAULT_SITE_PATH, port: int = DEFAULT_PORT):
+        self.site_path = pathlib.Path(site_path)
+        self.port = port
+        self.process = None
+
+    def start(self) -> None:
+        if not self.site_path.exists(): raise FileNotFoundError(f"Site directory {self.site_path} does not exist")
+        console.print(f"Attempting to start HTTP server in [cyan]{self.site_path}[/cyan] on port {self.port}...")
+        try:
+            cmd = ["python", "-m", "http.server", str(self.port)]
+            creationflags = 0; preexec_fn = None
+            if sys.platform == 'win32': creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
+            self.process = subprocess.Popen(cmd, cwd=str(self.site_path), stdout=subprocess.PIPE, stderr=subprocess.PIPE, creationflags=creationflags)
+            time.sleep(1.5)
+            if self.is_running(): console.print(f"[bold green]HTTP server started successfully (PID: {self.process.pid})[/bold green]")
+            else:
+                console.print("[bold red]Failed to start HTTP server. Checking logs...[/bold red]")
+                stdout, stderr = self.process.communicate(); print(stdout.decode(errors='ignore')); print(stderr.decode(errors='ignore'))
+                self.stop(); raise RuntimeError("HTTP server failed to start.")
+        except Exception as e: console.print(f"[bold red]Error starting HTTP server: {str(e)}[/bold red]"); self.stop(); raise
+
+    def stop(self) -> None:
+        if self.process and self.is_running():
+            console.print(f"Stopping HTTP server (PID: {self.process.pid})...")
+            try:
+                if sys.platform == 'win32': self.process.send_signal(signal.CTRL_BREAK_EVENT); time.sleep(0.5)
+                self.process.terminate()
+                try: stdout, stderr = self.process.communicate(timeout=5); console.print("[bold yellow]HTTP server stopped[/bold yellow]")
+                except subprocess.TimeoutExpired: console.print("[bold red]Server did not terminate gracefully, killing...[/bold red]"); self.process.kill(); stdout, stderr = self.process.communicate(); console.print("[bold yellow]HTTP server killed[/bold yellow]")
+            except Exception as e: console.print(f"[bold red]Error stopping HTTP server: {str(e)}[/bold red]"); self.process.kill()
+            finally: self.process = None
+        elif self.process: console.print("[dim]HTTP server process already stopped.[/dim]"); self.process = None
+
+    def is_running(self) -> bool:
+        if not self.process: return False
+        return self.process.poll() is None
+
+# --- SimpleMemoryTracker Class (Unchanged) ---
+class SimpleMemoryTracker:
+    """Basic memory tracker that doesn't rely on psutil."""
+    def __init__(self, report_path: str = DEFAULT_REPORT_PATH, test_id: Optional[str] = None):
+        self.report_path = pathlib.Path(report_path); self.report_path.mkdir(parents=True, exist_ok=True)
+        self.test_id = test_id or time.strftime("%Y%m%d_%H%M%S")
+        self.start_time = time.time(); self.memory_samples = []; self.pid = os.getpid()
+        self.csv_path = self.report_path / f"memory_samples_{self.test_id}.csv"
+        with open(self.csv_path, 'w', encoding='utf-8') as f: f.write("timestamp,elapsed_seconds,memory_info_mb\n")
+
+    def sample(self) -> Dict:
+        try:
+            memory_mb = self._get_memory_info_mb()
+            memory_str = f"{memory_mb:.1f} MB" if memory_mb is not None else "Unknown"
+            timestamp = time.time(); elapsed = timestamp - self.start_time
+            sample = {"timestamp": timestamp, "elapsed_seconds": elapsed, "memory_mb": memory_mb, "memory_str": memory_str}
+            self.memory_samples.append(sample)
+            with open(self.csv_path, 'a', encoding='utf-8') as f: f.write(f"{timestamp},{elapsed:.2f},{memory_mb if memory_mb is not None else ''}\n")
+            return sample
+        except Exception as e: return {"memory_mb": None, "memory_str": "Error"}
+
+    def _get_memory_info_mb(self) -> Optional[float]:
+        pid_str = str(self.pid)
+        try:
+            if sys.platform == 'darwin': result = subprocess.run(["ps", "-o", "rss=", "-p", pid_str], capture_output=True, text=True, check=True, encoding='utf-8'); return int(result.stdout.strip()) / 1024.0
+            elif sys.platform == 'linux':
+                with open(f"/proc/{pid_str}/status", encoding='utf-8') as f:
+                    for line in f:
+                        if line.startswith("VmRSS:"): return int(line.split()[1]) / 1024.0
+                return None
+            elif sys.platform == 'win32': result = subprocess.run(["tasklist", "/fi", f"PID eq {pid_str}", "/fo", "csv", "/nh"], capture_output=True, text=True, check=True, encoding='cp850', errors='ignore'); parts = result.stdout.strip().split('","'); return int(parts[4].strip().replace('"', '').replace(' K', '').replace(',', '')) / 1024.0 if len(parts) >= 5 else None
+            else: return None
+        except: return None # Catch all exceptions for robustness
+
+    def get_report(self) -> Dict:
+        if not self.memory_samples: return {"error": "No memory samples collected"}
+        total_time = time.time() - self.start_time; valid_samples = [s['memory_mb'] for s in self.memory_samples if s['memory_mb'] is not None]
+        start_mem = valid_samples[0] if valid_samples else None; end_mem = valid_samples[-1] if valid_samples else None
+        max_mem = max(valid_samples) if valid_samples else None; avg_mem = sum(valid_samples) / len(valid_samples) if valid_samples else None
+        growth = (end_mem - start_mem) if start_mem is not None and end_mem is not None else None
+        return {"test_id": self.test_id, "total_time_seconds": total_time, "sample_count": len(self.memory_samples), "valid_sample_count": len(valid_samples), "csv_path": str(self.csv_path), "platform": sys.platform, "start_memory_mb": start_mem, "end_memory_mb": end_mem, "max_memory_mb": max_mem, "average_memory_mb": avg_mem, "memory_growth_mb": growth}
+
+
+# --- CrawlerStressTest Class (Refactored for Per-Batch Logging) ---
+class CrawlerStressTest:
+    """Orchestrates the stress test using arun_many per chunk and a dispatcher."""
+
+    def __init__(
+        self,
+        url_count: int = DEFAULT_URL_COUNT,
+        port: int = DEFAULT_PORT,
+        max_sessions: int = DEFAULT_MAX_SESSIONS,
+        chunk_size: int = DEFAULT_CHUNK_SIZE, # Added chunk_size
+        report_path: str = DEFAULT_REPORT_PATH,
+        stream_mode: bool = DEFAULT_STREAM_MODE,
+        monitor_mode: str = DEFAULT_MONITOR_MODE,
+        use_rate_limiter: bool = False
+    ):
+        self.url_count = url_count
+        self.server_port = port
+        self.max_sessions = max_sessions
+        self.chunk_size = chunk_size # Store chunk size
+        self.report_path = pathlib.Path(report_path)
+        self.report_path.mkdir(parents=True, exist_ok=True)
+        self.stream_mode = stream_mode
+        self.monitor_mode = DisplayMode[monitor_mode.upper()]
+        self.use_rate_limiter = use_rate_limiter
+
+        self.test_id = time.strftime("%Y%m%d_%H%M%S")
+        self.results_summary = {
+            "test_id": self.test_id, "url_count": url_count, "max_sessions": max_sessions,
+            "chunk_size": chunk_size, "stream_mode": stream_mode, "monitor_mode": monitor_mode,
+            "rate_limiter_used": use_rate_limiter, "start_time": "", "end_time": "",
+            "total_time_seconds": 0, "successful_urls": 0, "failed_urls": 0,
+            "urls_processed": 0, "chunks_processed": 0
+        }
+
+    async def run(self) -> Dict:
+        """Run the stress test and return results."""
+        memory_tracker = SimpleMemoryTracker(report_path=self.report_path, test_id=self.test_id)
+        urls = [f"http://localhost:{self.server_port}/page_{i}.html" for i in range(self.url_count)]
+        # Split URLs into chunks based on self.chunk_size
+        url_chunks = [urls[i:i+self.chunk_size] for i in range(0, len(urls), self.chunk_size)]
+
+        self.results_summary["start_time"] = time.strftime("%Y-%m-%d %H:%M:%S")
+        start_time = time.time()
+
+        config = CrawlerRunConfig(
+            wait_for_images=False, verbose=False,
+            stream=self.stream_mode, # Still pass stream mode, affects arun_many return type
+            cache_mode=CacheMode.BYPASS
+        )
+
+        total_successful_urls = 0
+        total_failed_urls = 0
+        total_urls_processed = 0
+        start_memory_sample = memory_tracker.sample()
+        start_memory_str = start_memory_sample.get("memory_str", "Unknown")
+
+        # monitor = CrawlerMonitor(display_mode=self.monitor_mode, total_urls=self.url_count)
+        monitor = None
+        rate_limiter = RateLimiter(base_delay=(0.1, 0.3)) if self.use_rate_limiter else None
+        dispatcher = MemoryAdaptiveDispatcher(max_session_permit=self.max_sessions, monitor=monitor, rate_limiter=rate_limiter)
+
+        console.print(f"\n[bold cyan]Crawl4AI Stress Test - {self.url_count} URLs, {self.max_sessions} max sessions[/bold cyan]")
+        console.print(f"[bold cyan]Mode:[/bold cyan] {'Streaming' if self.stream_mode else 'Batch'}, [bold cyan]Monitor:[/bold cyan] {self.monitor_mode.name}, [bold cyan]Chunk Size:[/bold cyan] {self.chunk_size}")
+        console.print(f"[bold cyan]Initial Memory:[/bold cyan] {start_memory_str}")
+
+        # Print batch log header only if not streaming
+        if not self.stream_mode:
+            console.print("\n[bold]Batch Progress:[/bold] (Monitor below shows overall progress)")
+            console.print("[bold] Batch | Progress | Start Mem | End Mem   | URLs/sec | Success/Fail | Time (s) | Status [/bold]")
+            console.print("─" * 90)
+
+        monitor_task = asyncio.create_task(self._periodic_memory_sample(memory_tracker, 2.0))
+
+        try:
+            async with AsyncWebCrawler(
+                    config=BrowserConfig( verbose = False)
+                ) as crawler:
+                # Process URLs chunk by chunk
+                for chunk_idx, url_chunk in enumerate(url_chunks):
+                    batch_start_time = time.time()
+                    chunk_success = 0
+                    chunk_failed = 0
+
+                    # Sample memory before the chunk
+                    start_mem_sample = memory_tracker.sample()
+                    start_mem_str = start_mem_sample.get("memory_str", "Unknown")
+
+                    # --- Call arun_many for the current chunk ---
+                    try:
+                        # Note: dispatcher/monitor persist across calls
+                        results_gen_or_list: Union[AsyncGenerator[CrawlResult, None], List[CrawlResult]] = \
+                            await crawler.arun_many(
+                                urls=url_chunk,
+                                config=config,
+                                dispatcher=dispatcher # Reuse the same dispatcher
+                            )
+
+                        if self.stream_mode:
+                            # Process stream results if needed, but batch logging is less relevant
+                            async for result in results_gen_or_list:
+                                total_urls_processed += 1
+                                if result.success: chunk_success += 1
+                                else: chunk_failed += 1
+                            # In stream mode, batch summary isn't as meaningful here
+                            # We could potentially track completion per chunk async, but it's complex
+
+                        else: # Batch mode
+                            # Process the list of results for this chunk
+                            for result in results_gen_or_list:
+                                total_urls_processed += 1
+                                if result.success: chunk_success += 1
+                                else: chunk_failed += 1
+
+                    except Exception as e:
+                        console.print(f"[bold red]Error processing chunk {chunk_idx+1}: {e}[/bold red]")
+                        chunk_failed = len(url_chunk) # Assume all failed in the chunk on error
+                        total_urls_processed += len(url_chunk) # Count them as processed (failed)
+
+                    # --- Log batch results (only if not streaming) ---
+                    if not self.stream_mode:
+                        batch_time = time.time() - batch_start_time
+                        urls_per_sec = len(url_chunk) / batch_time if batch_time > 0 else 0
+                        end_mem_sample = memory_tracker.sample()
+                        end_mem_str = end_mem_sample.get("memory_str", "Unknown")
+
+                        progress_pct = (total_urls_processed / self.url_count) * 100
+
+                        if chunk_failed == 0: status_color, status = "green", "Success"
+                        elif chunk_success == 0: status_color, status = "red", "Failed"
+                        else: status_color, status = "yellow", "Partial"
+
+                        console.print(
+                             f" {chunk_idx+1:<5} | {progress_pct:6.1f}% | {start_mem_str:>9} | {end_mem_str:>9} | {urls_per_sec:8.1f} | "
+                            f"{chunk_success:^7}/{chunk_failed:<6} | {batch_time:8.2f} | [{status_color}]{status:<7}[/{status_color}]"
+                        )
+
+                    # Accumulate totals
+                    total_successful_urls += chunk_success
+                    total_failed_urls += chunk_failed
+                    self.results_summary["chunks_processed"] += 1
+
+                    # Optional small delay between starting chunks if needed
+                    # await asyncio.sleep(0.1)
+
+        except Exception as e:
+             console.print(f"[bold red]An error occurred during the main crawl loop: {e}[/bold red]")
+        finally:
+            if 'monitor_task' in locals() and not monitor_task.done():
+                 monitor_task.cancel()
+                 try: await monitor_task
+                 except asyncio.CancelledError: pass
+
+        end_time = time.time()
+        self.results_summary.update({
+            "end_time": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "total_time_seconds": end_time - start_time,
+            "successful_urls": total_successful_urls,
+            "failed_urls": total_failed_urls,
+            "urls_processed": total_urls_processed,
+            "memory": memory_tracker.get_report()
+        })
+        self._save_results()
+        return self.results_summary
+
+    async def _periodic_memory_sample(self, tracker: SimpleMemoryTracker, interval: float):
+        """Background task to sample memory periodically."""
+        while True:
+            tracker.sample()
+            try:
+                await asyncio.sleep(interval)
+            except asyncio.CancelledError:
+                break # Exit loop on cancellation
+
+    def _save_results(self) -> None:
+        results_path = self.report_path / f"test_summary_{self.test_id}.json"
+        try:
+            with open(results_path, 'w', encoding='utf-8') as f: json.dump(self.results_summary, f, indent=2, default=str)
+            # console.print(f"\n[bold green]Results summary saved to {results_path}[/bold green]") # Moved summary print to run_full_test
+        except Exception as e: console.print(f"[bold red]Failed to save results summary: {e}[/bold red]")
+
+
+# --- run_full_test Function (Adjusted) ---
+async def run_full_test(args):
+    """Run the complete test process from site generation to crawling."""
+    server = None
+    site_generated = False
+
+    # --- Site Generation --- (Same as before)
+    if not args.use_existing_site and not args.skip_generation:
+        if os.path.exists(args.site_path): console.print(f"[yellow]Removing existing site directory: {args.site_path}[/yellow]"); shutil.rmtree(args.site_path)
+        site_generator = SiteGenerator(site_path=args.site_path, page_count=args.urls); site_generator.generate_site(); site_generated = True
+    elif args.use_existing_site: console.print(f"[cyan]Using existing site assumed to be running on port {args.port}[/cyan]")
+    elif args.skip_generation:
+         console.print(f"[cyan]Skipping site generation, using existing directory: {args.site_path}[/cyan]")
+         if not os.path.exists(args.site_path) or not os.path.isdir(args.site_path): console.print(f"[bold red]Error: Site path '{args.site_path}' does not exist or is not a directory.[/bold red]"); return
+
+    # --- Start Local Server --- (Same as before)
+    server_started = False
+    if not args.use_existing_site:
+        server = LocalHttpServer(site_path=args.site_path, port=args.port)
+        try: server.start(); server_started = True
+        except Exception as e:
+            console.print(f"[bold red]Failed to start local server. Aborting test.[/bold red]")
+            if site_generated and not args.keep_site: console.print(f"[yellow]Cleaning up generated site: {args.site_path}[/yellow]"); shutil.rmtree(args.site_path)
+            return
+
+    try:
+        # --- Run the Stress Test ---
+        test = CrawlerStressTest(
+            url_count=args.urls,
+            port=args.port,
+            max_sessions=args.max_sessions,
+            chunk_size=args.chunk_size, # Pass chunk_size
+            report_path=args.report_path,
+            stream_mode=args.stream,
+            monitor_mode=args.monitor_mode,
+            use_rate_limiter=args.use_rate_limiter
+        )
+        results = await test.run() # Run the test which now handles chunks internally
+
+        # --- Print Summary ---
+        console.print("\n" + "=" * 80)
+        console.print("[bold green]Test Completed[/bold green]")
+        console.print("=" * 80)
+
+        # (Summary printing logic remains largely the same)
+        success_rate = results["successful_urls"] / results["url_count"] * 100 if results["url_count"] > 0 else 0
+        urls_per_second = results["urls_processed"] / results["total_time_seconds"] if results["total_time_seconds"] > 0 else 0
+
+        console.print(f"[bold cyan]Test ID:[/bold cyan] {results['test_id']}")
+        console.print(f"[bold cyan]Configuration:[/bold cyan] {results['url_count']} URLs, {results['max_sessions']} sessions, Chunk: {results['chunk_size']}, Stream: {results['stream_mode']}, Monitor: {results['monitor_mode']}")
+        console.print(f"[bold cyan]Results:[/bold cyan] {results['successful_urls']} successful, {results['failed_urls']} failed ({results['urls_processed']} processed, {success_rate:.1f}% success)")
+        console.print(f"[bold cyan]Performance:[/bold cyan] {results['total_time_seconds']:.2f} seconds total, {urls_per_second:.2f} URLs/second avg")
+
+        mem_report = results.get("memory", {})
+        mem_info_str = "Memory tracking data unavailable."
+        if mem_report and not mem_report.get("error"):
+            start_mb = mem_report.get('start_memory_mb'); end_mb = mem_report.get('end_memory_mb'); max_mb = mem_report.get('max_memory_mb'); growth_mb = mem_report.get('memory_growth_mb')
+            mem_parts = []
+            if start_mb is not None: mem_parts.append(f"Start: {start_mb:.1f} MB")
+            if end_mb is not None: mem_parts.append(f"End: {end_mb:.1f} MB")
+            if max_mb is not None: mem_parts.append(f"Max: {max_mb:.1f} MB")
+            if growth_mb is not None: mem_parts.append(f"Growth: {growth_mb:.1f} MB")
+            if mem_parts: mem_info_str = ", ".join(mem_parts)
+            csv_path = mem_report.get('csv_path')
+            if csv_path: console.print(f"[dim]Memory samples saved to: {csv_path}[/dim]")
+
+        console.print(f"[bold cyan]Memory Usage:[/bold cyan] {mem_info_str}")
+        console.print(f"[bold green]Results summary saved to {results['memory']['csv_path'].replace('memory_samples', 'test_summary').replace('.csv', '.json')}[/bold green]") # Infer summary path
+
+
+        if results["failed_urls"] > 0: console.print(f"\n[bold yellow]Warning: {results['failed_urls']} URLs failed to process ({100-success_rate:.1f}% failure rate)[/bold yellow]")
+        if results["urls_processed"] < results["url_count"]: console.print(f"\n[bold red]Error: Only {results['urls_processed']} out of {results['url_count']} URLs were processed![/bold red]")
+
+
+    finally:
+        # --- Stop Server / Cleanup --- (Same as before)
+        if server_started and server and not args.keep_server_alive: server.stop()
+        elif server_started and server and args.keep_server_alive:
+            console.print(f"[bold cyan]Server is kept running on port {args.port}. Press Ctrl+C to stop it.[/bold cyan]")
+            try: await asyncio.Future() # Keep running indefinitely
+            except KeyboardInterrupt: console.print("\n[bold yellow]Stopping server due to user interrupt...[/bold yellow]"); server.stop()
+
+        if site_generated and not args.keep_site: console.print(f"[yellow]Cleaning up generated site: {args.site_path}[/yellow]"); shutil.rmtree(args.site_path)
+        elif args.clean_site and os.path.exists(args.site_path): console.print(f"[yellow]Cleaning up site directory as requested: {args.site_path}[/yellow]"); shutil.rmtree(args.site_path)
+
+
+# --- main Function (Added chunk_size argument) ---
+def main():
+    """Main entry point for the script."""
+    parser = argparse.ArgumentParser(description="Crawl4AI SDK High Volume Stress Test using arun_many")
+
+    # Test parameters
+    parser.add_argument("--urls", type=int, default=DEFAULT_URL_COUNT, help=f"Number of URLs to test (default: {DEFAULT_URL_COUNT})")
+    parser.add_argument("--max-sessions", type=int, default=DEFAULT_MAX_SESSIONS, help=f"Maximum concurrent crawling sessions (default: {DEFAULT_MAX_SESSIONS})")
+    parser.add_argument("--chunk-size", type=int, default=DEFAULT_CHUNK_SIZE, help=f"Number of URLs per batch for logging (default: {DEFAULT_CHUNK_SIZE})") # Added
+    parser.add_argument("--stream", action="store_true", default=DEFAULT_STREAM_MODE, help=f"Enable streaming mode (disables batch logging) (default: {DEFAULT_STREAM_MODE})")
+    parser.add_argument("--monitor-mode", type=str, default=DEFAULT_MONITOR_MODE, choices=["DETAILED", "AGGREGATED"], help=f"Display mode for the live monitor (default: {DEFAULT_MONITOR_MODE})")
+    parser.add_argument("--use-rate-limiter", action="store_true", default=False, help="Enable a basic rate limiter (default: False)")
+
+    # Environment parameters
+    parser.add_argument("--site-path", type=str, default=DEFAULT_SITE_PATH, help=f"Path to generate/use the test site (default: {DEFAULT_SITE_PATH})")
+    parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"Port for the local HTTP server (default: {DEFAULT_PORT})")
+    parser.add_argument("--report-path", type=str, default=DEFAULT_REPORT_PATH, help=f"Path to save reports and logs (default: {DEFAULT_REPORT_PATH})")
+
+    # Site/Server management
+    parser.add_argument("--skip-generation", action="store_true", help="Use existing test site folder without regenerating")
+    parser.add_argument("--use-existing-site", action="store_true", help="Do not generate site or start local server; assume site exists on --port")
+    parser.add_argument("--keep-server-alive", action="store_true", help="Keep the local HTTP server running after test")
+    parser.add_argument("--keep-site", action="store_true", help="Keep the generated test site files after test")
+    parser.add_argument("--clean-reports", action="store_true", help="Clean up report directory before running")
+    parser.add_argument("--clean-site", action="store_true", help="Clean up site directory before running (if generating) or after")
+
+    args = parser.parse_args()
+
+    # Display config
+    console.print("[bold underline]Crawl4AI SDK Stress Test Configuration[/bold underline]")
+    console.print(f"URLs: {args.urls}, Max Sessions: {args.max_sessions}, Chunk Size: {args.chunk_size}") # Added chunk size
+    console.print(f"Mode: {'Streaming' if args.stream else 'Batch'}, Monitor: {args.monitor_mode}, Rate Limit: {args.use_rate_limiter}")
+    console.print(f"Site Path: {args.site_path}, Port: {args.port}, Report Path: {args.report_path}")
+    console.print("-" * 40)
+    # (Rest of config display and cleanup logic is the same)
+    if args.use_existing_site: console.print("[cyan]Mode: Using existing external site/server[/cyan]")
+    elif args.skip_generation: console.print("[cyan]Mode: Using existing site files, starting local server[/cyan]")
+    else: console.print("[cyan]Mode: Generating site files, starting local server[/cyan]")
+    if args.keep_server_alive: console.print("[cyan]Option: Keep server alive after test[/cyan]")
+    if args.keep_site: console.print("[cyan]Option: Keep site files after test[/cyan]")
+    if args.clean_reports: console.print("[cyan]Option: Clean reports before test[/cyan]")
+    if args.clean_site: console.print("[cyan]Option: Clean site directory[/cyan]")
+    console.print("-" * 40)
+
+    if args.clean_reports:
+        if os.path.exists(args.report_path): console.print(f"[yellow]Cleaning up reports directory: {args.report_path}[/yellow]"); shutil.rmtree(args.report_path)
+        os.makedirs(args.report_path, exist_ok=True)
+    if args.clean_site and not args.use_existing_site:
+         if os.path.exists(args.site_path): console.print(f"[yellow]Cleaning up site directory as requested: {args.site_path}[/yellow]"); shutil.rmtree(args.site_path)
+
+    # Run
+    try: asyncio.run(run_full_test(args))
+    except KeyboardInterrupt: console.print("\n[bold yellow]Test interrupted by user.[/bold yellow]")
+    except Exception as e: console.print(f"\n[bold red]An unexpected error occurred:[/bold red] {e}"); import traceback; traceback.print_exc()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tests/profiler/test_create_profile.py b/tests/profiler/test_create_profile.py
new file mode 100644
index 0000000..e441ea4
--- /dev/null
+++ b/tests/profiler/test_create_profile.py
@@ -0,0 +1,32 @@
+from crawl4ai import BrowserProfiler
+import asyncio
+
+
+if __name__ == "__main__":
+    # Example usage
+    profiler = BrowserProfiler()
+    
+    # Create a new profile
+    import os
+    from pathlib import Path
+    home_dir = Path.home()
+    profile_path = asyncio.run(profiler.create_profile( str(home_dir / ".crawl4ai/profiles/test-profile")))
+    
+    print(f"Profile created at: {profile_path}")
+
+        
+            
+    # # Launch a standalone browser
+    # asyncio.run(profiler.launch_standalone_browser())
+    
+    # # List profiles
+    # profiles = profiler.list_profiles()
+    # for profile in profiles:
+    #     print(f"Profile: {profile['name']}, Path: {profile['path']}")
+    
+    # # Delete a profile
+    # success = profiler.delete_profile("my-profile")
+    # if success:
+    #     print("Profile deleted successfully")
+    # else:
+    #     print("Failed to delete profile")
\ No newline at end of file
diff --git a/tests/profiler/test_keyboard_handle.py b/tests/profiler/test_keyboard_handle.py
new file mode 100644
index 0000000..8845c10
--- /dev/null
+++ b/tests/profiler/test_keyboard_handle.py
@@ -0,0 +1,55 @@
+import sys
+import pytest
+import asyncio
+from unittest.mock import patch, MagicMock
+from crawl4ai.browser_profiler import BrowserProfiler
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(sys.platform != "win32", reason="Windows-specific msvcrt test")
+async def test_keyboard_input_handling():
+    # Mock sequence of keystrokes: arrow key followed by 'q'
+    mock_keys = [b'\x00K', b'q']
+    mock_kbhit = MagicMock(side_effect=[True, True, False])
+    mock_getch = MagicMock(side_effect=mock_keys)
+
+    with patch('msvcrt.kbhit', mock_kbhit), patch('msvcrt.getch', mock_getch):
+        # profiler = BrowserProfiler()
+        user_done_event = asyncio.Event()
+        
+        # Create a local async function to simulate the keyboard input handling
+        async def test_listen_for_quit_command():
+            if sys.platform == "win32":
+                while True:
+                    try:
+                        if mock_kbhit():
+                            raw = mock_getch()
+                            try:
+                                key = raw.decode("utf-8")
+                            except UnicodeDecodeError:
+                                continue
+
+                            if len(key) != 1 or not key.isprintable():
+                                continue
+
+                            if key.lower() == "q":
+                                user_done_event.set()
+                                return
+
+                        await asyncio.sleep(0.1)
+                    except Exception as e:
+                        continue
+
+        # Run the listener
+        listener_task = asyncio.create_task(test_listen_for_quit_command())
+        
+        # Wait for the event to be set
+        try:
+            await asyncio.wait_for(user_done_event.wait(), timeout=1.0)
+            assert user_done_event.is_set()
+        finally:
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
\ No newline at end of file
diff --git a/tests/proxy/test_antibot_detector.py b/tests/proxy/test_antibot_detector.py
new file mode 100644
index 0000000..a1278e2
--- /dev/null
+++ b/tests/proxy/test_antibot_detector.py
@@ -0,0 +1,334 @@
+"""
+Unit tests for antibot_detector.is_blocked().
+
+Tests are organized into:
+  - TRUE POSITIVES:  Real block pages that MUST be detected
+  - TRUE NEGATIVES:  Legitimate pages that MUST NOT be flagged
+  - EDGE CASES:      Boundary conditions
+"""
+
+import sys, os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../.."))
+
+from crawl4ai.antibot_detector import is_blocked
+
+PASS = 0
+FAIL = 0
+
+def check(name, result, expected_blocked, expected_substr=None):
+    global PASS, FAIL
+    blocked, reason = result
+    ok = blocked == expected_blocked
+    if expected_substr and blocked:
+        ok = ok and expected_substr.lower() in reason.lower()
+    status = "PASS" if ok else "FAIL"
+    if not ok:
+        FAIL += 1
+        print(f"  {status}: {name}")
+        print(f"         got blocked={blocked}, reason={reason!r}")
+        print(f"         expected blocked={expected_blocked}" +
+              (f", substr={expected_substr!r}" if expected_substr else ""))
+    else:
+        PASS += 1
+        if blocked:
+            print(f"  {status}: {name} -> {reason}")
+        else:
+            print(f"  {status}: {name} -> not blocked")
+
+
+# =========================================================================
+# TRUE POSITIVES — real block pages that MUST be detected
+# =========================================================================
+print("\n=== TRUE POSITIVES (must detect as blocked) ===\n")
+
+# --- Akamai ---
+check("Akamai Reference #",
+    is_blocked(403, 'Access Denied\nYour request was blocked.\nReference #18.2d351ab8.1557333295.a4e16ab'),
+    True, "Akamai")
+
+check("Akamai Pardon Our Interruption",
+    is_blocked(403, 'Pardon Our Interruption
      Please verify you are human
      '),
+    True, "Pardon")
+
+check("Akamai 403 short Access Denied",
+    is_blocked(403, '
      Access Denied
      '),
+    True)  # Detected via near-empty 403 or Access Denied pattern
+
+# --- Cloudflare ---
+check("Cloudflare challenge form",
+    is_blocked(403, '''
+        
      
+        
+        
      '''),
+    True, "Cloudflare challenge")
+
+check("Cloudflare error 1020",
+    is_blocked(403, '''
+        
      1020
      
+        
      Access denied
      '''),
+    True, "Cloudflare firewall")
+
+check("Cloudflare IUAM script",
+    is_blocked(403, ''),
+    True, "Cloudflare JS challenge")
+
+check("Cloudflare Just a moment",
+    is_blocked(403, 'Just a moment...Checking your browser'),
+    True)  # Detected via near-empty 403 or Cloudflare pattern
+
+check("Cloudflare Checking your browser (short 503)",
+    is_blocked(503, 'Checking your browser before accessing the site.'),
+    True, "503")
+
+# --- PerimeterX ---
+check("PerimeterX block page",
+    is_blocked(403, '''Access to This Page Has Been Blocked
+        
      
+        '''),
+    True, "PerimeterX")
+
+check("PerimeterX captcha CDN",
+    is_blocked(403, ''),
+    True, "PerimeterX captcha")
+
+# --- DataDome ---
+check("DataDome captcha delivery",
+    is_blocked(403, ''''''),
+    True, "DataDome")
+
+# --- Imperva/Incapsula ---
+check("Imperva Incapsula Resource",
+    is_blocked(403, ''),
+    True, "Imperva")
+
+check("Imperva incident ID",
+    is_blocked(200, 'Request unsuccessful. Incapsula incident ID: 12345-67890'),
+    True, "Incapsula incident")
+
+# --- Sucuri ---
+check("Sucuri firewall",
+    is_blocked(403, '
      Sucuri WebSite Firewall - Access Denied
      '),
+    True, "Sucuri")
+
+# --- Kasada ---
+check("Kasada challenge",
+    is_blocked(403, ''),
+    True, "Kasada")
+
+# --- Reddit / Network Security ---
+check("Reddit blocked by network security (small)",
+    is_blocked(403, 'You\'ve been blocked by network security.'),
+    True, "Network security block")
+
+check("Reddit blocked by network security (190KB SPA shell)",
+    is_blocked(403, '' +
+        'You\'ve been blocked by network security. Log in to continue.'),
+    True, "Network security block")
+
+check("Network security block on HTTP 200 (buried in large page)",
+    is_blocked(200, '' +
+        '
      blocked by network security
      '),
+    True, "Network security block")
+
+# --- HTTP 429 ---
+check("HTTP 429 rate limit",
+    is_blocked(429, 'Rate limit exceeded'),
+    True, "429")
+
+check("HTTP 429 empty body",
+    is_blocked(429, ''),
+    True, "429")
+
+# --- Empty 200 ---
+check("HTTP 200 empty page",
+    is_blocked(200, ''),
+    True, "empty")
+
+check("HTTP 200 whitespace only",
+    is_blocked(200, '   \n\n   '),
+    True, "empty")
+
+# --- 403 near-empty ---
+check("HTTP 403 near-empty (10 bytes)",
+    is_blocked(403, ''),
+    True, "403")
+
+
+# =========================================================================
+# TRUE NEGATIVES — legitimate pages that MUST NOT be flagged
+# =========================================================================
+print("\n=== TRUE NEGATIVES (must NOT detect as blocked) ===\n")
+
+# --- Normal pages ---
+check("Normal 200 page (example.com size)",
+    is_blocked(200, 'Example
      ' + 'x' * 500 + '
      '),
+    False)
+
+check("Normal 200 large page",
+    is_blocked(200, '' + '
      Some content here.
      \n' * 5000 + ''),
+    False)
+
+# --- Security articles (false positive trap!) ---
+check("Article about bot detection (large page)",
+    is_blocked(200, 'How to Detect Bots' +
+        '
      How to Detect Bots on Your Website
      ' +
+        '
      Anti-bot solutions like DataDome, PerimeterX, and Cloudflare ' +
+        'help detect and block bot traffic. When a bot is detected, ' +
+        'services show a CAPTCHA or Access Denied page. ' +
+        'Common signals include blocked by security warnings.
      ' +
+        '
      The g-recaptcha and h-captcha widgets are used for challenges.
      ' +
+        '
      ' + 'More article content. ' * 500 + '
      ' +
+        ''),
+    False)
+
+check("DataDome marketing page (large)",
+    is_blocked(200, '
      DataDome Bot Protection
      ' +
+        '
      DataDome protects websites from bot attacks. ' +
+        'Our solution detects automated traffic using advanced fingerprinting. ' +
+        'Competitors like PerimeterX use window._pxAppId for tracking.
      ' +
+        '
      ' + 'Marketing content. ' * 1000 + '
      ' +
+        ''),
+    False)
+
+
+# --- Login pages with CAPTCHA (not a block!) ---
+check("Login page with reCAPTCHA (large page)",
+    is_blocked(200, 'Sign In' +
+        '' +
+        '
      ' +
+        '' +
+        '' +
+        '
      ' +
+        '' +
+        '
      ' +
+        '
      Copyright 2024
      ' +
+        '
      ' + 'Page content. ' * 500 + '
      ' +
+        ''),
+    False)
+
+check("Signup page with hCaptcha (large page)",
+    is_blocked(200, '' +
+        '
      Create Account
      ' +
+        '
      ' +
+        '
      ' + 'Registration info. ' * 500 + '
      ' +
+        ''),
+    False)
+
+# --- 403 pages — ALL non-data 403 HTML is now treated as blocked ---
+# Rationale: 403 is never the content the user wants. Even for legitimate
+# auth errors (Apache/Nginx), the fallback will also get 403 and we report
+# failure correctly. False positives are cheap; false negatives are catastrophic.
+check("Apache directory listing denied (403, large-ish)",
+    is_blocked(403, '403 Forbidden' +
+        '
      Forbidden
      ' +
+        '
      You don\'t have permission to access this resource on this server.
      ' +
+        '
      Apache/2.4.41 (Ubuntu) Server at example.com Port 80
      ' +
+        '
      ' + 'Server info. ' * 500 + '
      ' +
+        ''),
+    True, "403")
+
+check("Nginx 403 (large page)",
+    is_blocked(403, '403 Forbidden' +
+        '
      403 Forbidden
      ' +
+        '
      nginx/1.18.0
      ' +
+        '
      ' + 'Content. ' * 500 + '
      ' +
+        ''),
+    True, "403")
+
+check("API 403 auth required (JSON)",
+    is_blocked(403, '{"error": "Forbidden", "message": "Invalid API key", "code": 403}'),
+    False)
+
+# --- Cloudflare-served normal pages (not blocked!) ---
+check("Cloudflare-served normal page with footer",
+    is_blocked(200, '' +
+        '
      Welcome to Our Site
      ' +
+        '
      This is a normal page served through Cloudflare CDN.
      ' +
+        '
      Performance & security by Cloudflare
      ' +
+        '
      ' + 'Normal content. ' * 500 + '
      ' +
+        ''),
+    False)
+
+# --- Small but legitimate pages ---
+check("Small valid 200 page (with content element)",
+    is_blocked(200, 'OK
      Your request was processed successfully. Everything is fine.
      '),
+    False)
+
+check("Small JSON 200 response",
+    is_blocked(200, '{"status": "ok", "data": {"id": 123, "name": "test"}, "timestamp": "2024-01-01T00:00:00Z"}'),
+    False)
+
+check("Redirect page 200",
+    is_blocked(200, '
      Redirecting to your dashboard. Please wait while we prepare your personalized experience.
      '),
+    False)
+
+# --- 503 pages — ALL non-data 503 HTML is now treated as blocked ---
+# Same rationale as 403: 503 is never desired content. Fallback rescues false positives.
+check("503 maintenance page (treated as blocked)",
+    is_blocked(503, '
      Service Temporarily Unavailable
      ' +
+        '
      We are performing scheduled maintenance. Please try again later.
      ' +
+        '
      ' + 'Maintenance info. ' * 500 + '
      ' +
+        ''),
+    True, "503")
+
+# --- 200 with short but real content ---
+check("Short thank you page (200, 120 bytes)",
+    is_blocked(200, '
      Thank You!
      Your order has been placed. Confirmation email sent.
      '),
+    False)
+
+
+# =========================================================================
+# EDGE CASES
+# =========================================================================
+print("\n=== EDGE CASES ===\n")
+
+check("None status code + empty html",
+    is_blocked(None, ''),
+    True, "no ")
+
+check("None status code + block content",
+    is_blocked(None, 'Reference #18.2d351ab8.1557333295.a4e16ab'),
+    True, "Akamai")
+
+check("200 + tier1 pattern (Imperva deceptive 200)",
+    is_blocked(200, 'Request unsuccessful. Incapsula incident ID: 555-999'),
+    True, "Incapsula")
+
+check("403 + 4999 bytes (just under threshold)",
+    is_blocked(403, 'Access Denied' + 'x' * 4950 + ''),
+    True, "Access Denied")
+
+check("403 + 5001 bytes (over old threshold, now blocked)",
+    is_blocked(403, 'Some error page' + 'x' * 4960 + ''),
+    True, "403")
+
+check("403 + 9999 bytes with generic block text",
+    is_blocked(403, 'blocked by security' + 'x' * 9950 + ''),
+    True, "Blocked by security")
+
+check("403 + 10001 bytes with generic block text (now detected regardless of size)",
+    is_blocked(403, 'blocked by security' + 'x' * 9970 + ''),
+    True, "Blocked by security")
+
+check("200 + whitespace-padded but 89 bytes content (above threshold for meaningful)",
+    is_blocked(200, ' ' * 10 + 'x' * 89 + ' ' * 10),
+    True, "empty")
+
+check("200 + exactly 100 bytes stripped (at threshold, no body = structural fail)",
+    is_blocked(200, 'x' * 100),
+    True, "no ")
+
+
+# =========================================================================
+# SUMMARY
+# =========================================================================
+print(f"\n{'=' * 60}")
+print(f"RESULTS: {PASS} passed, {FAIL} failed out of {PASS + FAIL} tests")
+print(f"{'=' * 60}")
+if FAIL > 0:
+    print("SOME TESTS FAILED!")
+    sys.exit(1)
+else:
+    print("ALL TESTS PASSED!")
diff --git a/tests/proxy/test_chanel_cdp_proxy.py b/tests/proxy/test_chanel_cdp_proxy.py
new file mode 100644
index 0000000..3fc90bb
--- /dev/null
+++ b/tests/proxy/test_chanel_cdp_proxy.py
@@ -0,0 +1,112 @@
+"""
+Test: Chanel.com anti-bot bypass via crawl4ai
+
+Requires env vars:
+  MASSIVE_USERNAME  — Massive residential proxy username
+  MASSIVE_PASSWORD  — Massive residential proxy password
+
+Optional:
+  --cdp URL       Connect to external browser via CDP (e.g. http://localhost:9223)
+  --attempts N    Number of attempts per test (default 3)
+
+Usage:
+  export MASSIVE_USERNAME="your_user"
+  export MASSIVE_PASSWORD="your_pass"
+  .venv/bin/python tests/proxy/test_chanel_cdp_proxy.py
+  .venv/bin/python tests/proxy/test_chanel_cdp_proxy.py --cdp http://localhost:9223
+"""
+
+import asyncio
+import os
+import sys
+import re
+import tempfile
+import shutil
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_configs import ProxyConfig
+
+URL = "https://www.chanel.com/us/fashion/handbags/c/1x1x1/"
+
+MASSIVE_USERNAME = os.environ.get("MASSIVE_USERNAME", "")
+MASSIVE_PASSWORD = os.environ.get("MASSIVE_PASSWORD", "")
+MASSIVE_SERVER = "https://network.joinmassive.com:65535"
+
+
+def get_proxy_config():
+    if not MASSIVE_USERNAME or not MASSIVE_PASSWORD:
+        print("ERROR: Set MASSIVE_USERNAME and MASSIVE_PASSWORD env vars")
+        sys.exit(1)
+    return ProxyConfig(
+        server=MASSIVE_SERVER,
+        username=MASSIVE_USERNAME,
+        password=MASSIVE_PASSWORD,
+    )
+
+
+async def test_isolated_context(cdp_url: str = None, attempts: int = 3):
+    """Test with isolated context (works with both Playwright and CDP)."""
+    mode = f"CDP ({cdp_url})" if cdp_url else "Playwright Chromium"
+    print(f"\n{'='*60}")
+    print(f"Mode: Isolated context — {mode}")
+    print(f"{'='*60}\n")
+
+    kwargs = dict(
+        enable_stealth=True,
+        create_isolated_context=True,
+        viewport_width=1920,
+        viewport_height=1080,
+    )
+    if cdp_url:
+        kwargs["cdp_url"] = cdp_url
+    else:
+        kwargs["headless"] = True
+
+    config = BrowserConfig(**kwargs)
+    run_config = CrawlerRunConfig(
+        magic=True,
+        simulate_user=True,
+        override_navigator=True,
+        proxy_config=get_proxy_config(),
+        page_timeout=120000,
+        wait_until="load",
+        delay_before_return_html=15.0,
+    )
+
+    passed = 0
+    async with AsyncWebCrawler(config=config) as crawler:
+        for i in range(attempts):
+            result = await crawler.arun(URL, config=run_config)
+            ok = result.status_code == 200 and len(result.html) > 10000
+            title = ""
+            if ok:
+                passed += 1
+                m = re.search(r"(.*?)", result.html)
+                title = f"  title={m.group(1)}" if m else ""
+            print(f"  Attempt {i+1}: status={result.status_code}  html={len(result.html):>10,} bytes  {'PASS' if ok else 'FAIL'}{title}")
+
+    print(f"\nResult: {passed}/{attempts} passed")
+    return passed > 0
+
+
+async def main():
+    cdp_url = None
+    attempts = 3
+
+    args = sys.argv[1:]
+    for j, arg in enumerate(args):
+        if arg == "--cdp" and j + 1 < len(args):
+            cdp_url = args[j + 1]
+        if arg == "--attempts" and j + 1 < len(args):
+            attempts = int(args[j + 1])
+
+    ok = await test_isolated_context(cdp_url=cdp_url, attempts=attempts)
+
+    print(f"\n{'='*60}")
+    print(f"Result: {'PASS' if ok else 'FAIL'}")
+    print(f"{'='*60}")
+    return ok
+
+
+if __name__ == "__main__":
+    ok = asyncio.run(main())
+    sys.exit(0 if ok else 1)
diff --git a/tests/proxy/test_persistent_proxy.py b/tests/proxy/test_persistent_proxy.py
new file mode 100644
index 0000000..b700825
--- /dev/null
+++ b/tests/proxy/test_persistent_proxy.py
@@ -0,0 +1,68 @@
+import asyncio
+import os
+import shutil
+import uuid
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_configs import ProxyConfig
+
+
+async def crawl_chanel(url: str):
+    profile_dir = os.path.expanduser(f"~/.crawl4ai/chanel_{uuid.uuid4().hex[:8]}")
+    os.makedirs(profile_dir, exist_ok=True)
+
+    browser_config = BrowserConfig(
+        headless=True,
+        enable_stealth=True,
+        use_persistent_context=True,
+        user_data_dir=profile_dir,
+        viewport_width=1920,
+        viewport_height=1080,
+        user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+        headers={
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+        },
+        proxy_config=ProxyConfig(
+            server="https://network.joinmassive.com:65535",
+            username="mpuQHs4sWZ-country-US",
+            password="D0yWxVQo8wQ05RWqz1Bn",
+        ),
+    )
+
+    run_config = CrawlerRunConfig(
+        magic=True,
+        simulate_user=True,
+        override_navigator=True,
+        page_timeout=120000,
+        wait_until="load",
+        delay_before_return_html=10.0,
+    )
+
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url, config=run_config)
+            return result
+    finally:
+        shutil.rmtree(profile_dir, ignore_errors=True)
+
+
+async def main():
+    url = "https://www.chanel.com/us/fashion/handbags/c/1x1x1/"
+    result = await crawl_chanel(url)
+    print(f"Status: {result.status_code}")
+    print(f"Success: {result.success}")
+    print(f"HTML: {len(result.html):,} bytes")
+    if result.markdown:
+        md_len = len(result.markdown.raw_markdown)
+        print(f"Markdown: {md_len:,} chars")
+        if md_len > 500:
+            print(f"\nFirst 500 chars of markdown:\n{result.markdown.raw_markdown[:500]}")
+    if result.error_message:
+        print(f"Error: {result.error_message}")
+
+
+asyncio.run(main())
diff --git a/tests/proxy/test_proxy_config.py b/tests/proxy/test_proxy_config.py
new file mode 100644
index 0000000..ebea567
--- /dev/null
+++ b/tests/proxy/test_proxy_config.py
@@ -0,0 +1,582 @@
+"""
+Comprehensive test suite for ProxyConfig in different forms:
+1. String form (ip:port:username:password)
+2. Dict form (dictionary with keys)
+3. Object form (ProxyConfig instance)
+4. Environment variable form (from env vars)
+
+Tests cover all possible scenarios and edge cases using pytest.
+"""
+
+import asyncio
+import os
+import pytest
+import tempfile
+from unittest.mock import patch
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+from crawl4ai.async_configs import CrawlerRunConfig, ProxyConfig
+from crawl4ai.cache_context import CacheMode
+
+
+class TestProxyConfig:
+    """Comprehensive test suite for ProxyConfig functionality."""
+    
+    # Test data for different scenarios
+    # get free proxy server from from webshare.io https://www.webshare.io/?referral_code=3sqog0y1fvsl
+    TEST_PROXY_DATA = {
+        "server": "",
+        "username": "", 
+        "password": "",
+        "ip": ""
+    }
+    
+    def setup_method(self):
+        """Setup for each test method."""
+        self.test_url = "https://httpbin.org/ip"  # Use httpbin for testing
+        
+    # ==================== OBJECT FORM TESTS ====================
+    
+    def test_proxy_config_object_creation_basic(self):
+        """Test basic ProxyConfig object creation."""
+        proxy = ProxyConfig(server="127.0.0.1:8080")
+        assert proxy.server == "127.0.0.1:8080"
+        assert proxy.username is None
+        assert proxy.password is None
+        assert proxy.ip == "127.0.0.1"  # Should auto-extract IP
+        
+    def test_proxy_config_object_creation_full(self):
+        """Test ProxyConfig object creation with all parameters."""
+        proxy = ProxyConfig(
+            server=f"http://{self.TEST_PROXY_DATA['server']}",
+            username=self.TEST_PROXY_DATA['username'],
+            password=self.TEST_PROXY_DATA['password'],
+            ip=self.TEST_PROXY_DATA['ip']
+        )
+        assert proxy.server == f"http://{self.TEST_PROXY_DATA['server']}"
+        assert proxy.username == self.TEST_PROXY_DATA['username']
+        assert proxy.password == self.TEST_PROXY_DATA['password']
+        assert proxy.ip == self.TEST_PROXY_DATA['ip']
+        
+    def test_proxy_config_object_ip_extraction(self):
+        """Test automatic IP extraction from server URL."""
+        test_cases = [
+            ("http://192.168.1.1:8080", "192.168.1.1"),
+            ("https://10.0.0.1:3128", "10.0.0.1"),
+            ("192.168.1.100:8080", "192.168.1.100"),
+            ("proxy.example.com:8080", "proxy.example.com"),
+        ]
+        
+        for server, expected_ip in test_cases:
+            proxy = ProxyConfig(server=server)
+            assert proxy.ip == expected_ip, f"Failed for server: {server}"
+            
+    def test_proxy_config_object_invalid_server(self):
+        """Test ProxyConfig with invalid server formats."""
+        # Should not raise exception but may not extract IP properly
+        proxy = ProxyConfig(server="invalid-format")
+        assert proxy.server == "invalid-format"
+        # IP extraction might fail but object should still be created
+        
+    # ==================== DICT FORM TESTS ====================
+    
+    def test_proxy_config_from_dict_basic(self):
+        """Test creating ProxyConfig from basic dictionary."""
+        proxy_dict = {"server": "127.0.0.1:8080"}
+        proxy = ProxyConfig.from_dict(proxy_dict)
+        assert proxy.server == "127.0.0.1:8080"
+        assert proxy.username is None
+        assert proxy.password is None
+        
+    def test_proxy_config_from_dict_full(self):
+        """Test creating ProxyConfig from complete dictionary."""
+        proxy_dict = {
+            "server": f"http://{self.TEST_PROXY_DATA['server']}",
+            "username": self.TEST_PROXY_DATA['username'],
+            "password": self.TEST_PROXY_DATA['password'],
+            "ip": self.TEST_PROXY_DATA['ip']
+        }
+        proxy = ProxyConfig.from_dict(proxy_dict)
+        assert proxy.server == proxy_dict["server"]
+        assert proxy.username == proxy_dict["username"]
+        assert proxy.password == proxy_dict["password"]
+        assert proxy.ip == proxy_dict["ip"]
+        
+    def test_proxy_config_from_dict_missing_keys(self):
+        """Test creating ProxyConfig from dictionary with missing keys."""
+        proxy_dict = {"server": "127.0.0.1:8080", "username": "user"}
+        proxy = ProxyConfig.from_dict(proxy_dict)
+        assert proxy.server == "127.0.0.1:8080"
+        assert proxy.username == "user"
+        assert proxy.password is None
+        assert proxy.ip == "127.0.0.1"  # Should auto-extract
+        
+    def test_proxy_config_from_dict_empty(self):
+        """Test creating ProxyConfig from empty dictionary."""
+        proxy_dict = {}
+        proxy = ProxyConfig.from_dict(proxy_dict)
+        assert proxy.server is None
+        assert proxy.username is None
+        assert proxy.password is None
+        assert proxy.ip is None
+        
+    def test_proxy_config_from_dict_none_values(self):
+        """Test creating ProxyConfig from dictionary with None values."""
+        proxy_dict = {
+            "server": "127.0.0.1:8080",
+            "username": None,
+            "password": None,
+            "ip": None
+        }
+        proxy = ProxyConfig.from_dict(proxy_dict)
+        assert proxy.server == "127.0.0.1:8080"
+        assert proxy.username is None
+        assert proxy.password is None
+        assert proxy.ip == "127.0.0.1"  # Should auto-extract despite None
+        
+    # ==================== STRING FORM TESTS ====================
+    
+    def test_proxy_config_from_string_full_format(self):
+        """Test creating ProxyConfig from full string format (ip:port:username:password)."""
+        proxy_str = f"{self.TEST_PROXY_DATA['ip']}:6114:{self.TEST_PROXY_DATA['username']}:{self.TEST_PROXY_DATA['password']}"
+        proxy = ProxyConfig.from_string(proxy_str)
+        assert proxy.server == f"http://{self.TEST_PROXY_DATA['ip']}:6114"
+        assert proxy.username == self.TEST_PROXY_DATA['username']
+        assert proxy.password == self.TEST_PROXY_DATA['password']
+        assert proxy.ip == self.TEST_PROXY_DATA['ip']
+        
+    def test_proxy_config_from_string_ip_port_only(self):
+        """Test creating ProxyConfig from string with only ip:port."""
+        proxy_str = "192.168.1.1:8080"
+        proxy = ProxyConfig.from_string(proxy_str)
+        assert proxy.server == "http://192.168.1.1:8080"
+        assert proxy.username is None
+        assert proxy.password is None
+        assert proxy.ip == "192.168.1.1"
+        
+    def test_proxy_config_from_string_invalid_format(self):
+        """Test creating ProxyConfig from invalid string formats."""
+        invalid_formats = [
+            "invalid",
+            "ip:port:user",  # Missing password (3 parts)
+            "ip:port:user:pass:extra",  # Too many parts (5 parts)
+            "",
+            "::",  # Empty parts but 3 total (invalid)
+            "::::",  # Empty parts but 5 total (invalid)
+        ]
+        
+        for proxy_str in invalid_formats:
+            with pytest.raises(ValueError, match="Invalid proxy string format"):
+                ProxyConfig.from_string(proxy_str)
+                
+    def test_proxy_config_from_string_edge_cases_that_work(self):
+        """Test string formats that should work but might be edge cases."""
+        # These cases actually work as valid formats
+        edge_cases = [
+            (":", "http://:", ""),  # ip:port format with empty values
+            (":::", "http://:", ""),  # ip:port:user:pass format with empty values
+        ]
+        
+        for proxy_str, expected_server, expected_ip in edge_cases:
+            proxy = ProxyConfig.from_string(proxy_str)
+            assert proxy.server == expected_server
+            assert proxy.ip == expected_ip
+                
+    def test_proxy_config_from_string_edge_cases(self):
+        """Test string parsing edge cases."""
+        # Test with different port numbers
+        proxy_str = "10.0.0.1:3128:user:pass"
+        proxy = ProxyConfig.from_string(proxy_str)
+        assert proxy.server == "http://10.0.0.1:3128"
+        
+        # Test with special characters in credentials
+        proxy_str = "10.0.0.1:8080:user@domain:pass:word"
+        with pytest.raises(ValueError):  # Should fail due to extra colon in password
+            ProxyConfig.from_string(proxy_str)
+            
+    # ==================== ENVIRONMENT VARIABLE TESTS ====================
+    
+    def test_proxy_config_from_env_single_proxy(self):
+        """Test loading single proxy from environment variable."""
+        proxy_str = f"{self.TEST_PROXY_DATA['ip']}:6114:{self.TEST_PROXY_DATA['username']}:{self.TEST_PROXY_DATA['password']}"
+        
+        with patch.dict(os.environ, {'TEST_PROXIES': proxy_str}):
+            proxies = ProxyConfig.from_env('TEST_PROXIES')
+            assert len(proxies) == 1
+            proxy = proxies[0]
+            assert proxy.ip == self.TEST_PROXY_DATA['ip']
+            assert proxy.username == self.TEST_PROXY_DATA['username']
+            assert proxy.password == self.TEST_PROXY_DATA['password']
+            
+    def test_proxy_config_from_env_multiple_proxies(self):
+        """Test loading multiple proxies from environment variable."""
+        proxy_list = [
+            "192.168.1.1:8080:user1:pass1",
+            "192.168.1.2:8080:user2:pass2",
+            "10.0.0.1:3128"  # No auth
+        ]
+        proxy_str = ",".join(proxy_list)
+        
+        with patch.dict(os.environ, {'TEST_PROXIES': proxy_str}):
+            proxies = ProxyConfig.from_env('TEST_PROXIES')
+            assert len(proxies) == 3
+            
+            # Check first proxy
+            assert proxies[0].ip == "192.168.1.1"
+            assert proxies[0].username == "user1"
+            assert proxies[0].password == "pass1"
+            
+            # Check second proxy
+            assert proxies[1].ip == "192.168.1.2"
+            assert proxies[1].username == "user2"
+            assert proxies[1].password == "pass2"
+            
+            # Check third proxy (no auth)
+            assert proxies[2].ip == "10.0.0.1"
+            assert proxies[2].username is None
+            assert proxies[2].password is None
+            
+    def test_proxy_config_from_env_empty_var(self):
+        """Test loading from empty environment variable."""
+        with patch.dict(os.environ, {'TEST_PROXIES': ''}):
+            proxies = ProxyConfig.from_env('TEST_PROXIES')
+            assert len(proxies) == 0
+            
+    def test_proxy_config_from_env_missing_var(self):
+        """Test loading from missing environment variable."""
+        # Ensure the env var doesn't exist
+        with patch.dict(os.environ, {}, clear=True):
+            proxies = ProxyConfig.from_env('NON_EXISTENT_VAR')
+            assert len(proxies) == 0
+            
+    def test_proxy_config_from_env_with_empty_entries(self):
+        """Test loading proxies with empty entries in the list."""
+        proxy_str = "192.168.1.1:8080:user:pass,,10.0.0.1:3128,"
+        
+        with patch.dict(os.environ, {'TEST_PROXIES': proxy_str}):
+            proxies = ProxyConfig.from_env('TEST_PROXIES')
+            assert len(proxies) == 2  # Empty entries should be skipped
+            assert proxies[0].ip == "192.168.1.1"
+            assert proxies[1].ip == "10.0.0.1"
+            
+    def test_proxy_config_from_env_with_invalid_entries(self):
+        """Test loading proxies with some invalid entries."""
+        proxy_str = "192.168.1.1:8080:user:pass,invalid_proxy,10.0.0.1:3128"
+        
+        with patch.dict(os.environ, {'TEST_PROXIES': proxy_str}):
+            # Should handle errors gracefully and return valid proxies
+            proxies = ProxyConfig.from_env('TEST_PROXIES')
+            # Depending on implementation, might return partial list or empty
+            # This tests error handling
+            assert isinstance(proxies, list)
+            
+    # ==================== SERIALIZATION TESTS ====================
+    
+    def test_proxy_config_to_dict(self):
+        """Test converting ProxyConfig to dictionary."""
+        proxy = ProxyConfig(
+            server=f"http://{self.TEST_PROXY_DATA['server']}",
+            username=self.TEST_PROXY_DATA['username'],
+            password=self.TEST_PROXY_DATA['password'],
+            ip=self.TEST_PROXY_DATA['ip']
+        )
+        
+        result_dict = proxy.to_dict()
+        expected = {
+            "server": f"http://{self.TEST_PROXY_DATA['server']}",
+            "username": self.TEST_PROXY_DATA['username'],
+            "password": self.TEST_PROXY_DATA['password'],
+            "ip": self.TEST_PROXY_DATA['ip']
+        }
+        assert result_dict == expected
+        
+    def test_proxy_config_clone(self):
+        """Test cloning ProxyConfig with modifications."""
+        original = ProxyConfig(
+            server="http://127.0.0.1:8080",
+            username="user",
+            password="pass"
+        )
+        
+        # Clone with modifications
+        cloned = original.clone(username="new_user", password="new_pass")
+        
+        # Original should be unchanged
+        assert original.username == "user"
+        assert original.password == "pass"
+        
+        # Clone should have new values
+        assert cloned.username == "new_user"
+        assert cloned.password == "new_pass"
+        assert cloned.server == original.server  # Unchanged value
+        
+    def test_proxy_config_roundtrip_serialization(self):
+        """Test that ProxyConfig can be serialized and deserialized without loss."""
+        original = ProxyConfig(
+            server=f"http://{self.TEST_PROXY_DATA['server']}",
+            username=self.TEST_PROXY_DATA['username'],
+            password=self.TEST_PROXY_DATA['password'],
+            ip=self.TEST_PROXY_DATA['ip']
+        )
+        
+        # Serialize to dict and back
+        serialized = original.to_dict()
+        deserialized = ProxyConfig.from_dict(serialized)
+        
+        assert deserialized.server == original.server
+        assert deserialized.username == original.username
+        assert deserialized.password == original.password
+        assert deserialized.ip == original.ip
+        
+    # ==================== INTEGRATION TESTS ====================
+    
+    @pytest.mark.asyncio
+    async def test_crawler_with_proxy_config_object(self):
+        """Test AsyncWebCrawler with ProxyConfig object."""
+        proxy_config = ProxyConfig(
+            server=f"http://{self.TEST_PROXY_DATA['server']}",
+            username=self.TEST_PROXY_DATA['username'],
+            password=self.TEST_PROXY_DATA['password']
+        )
+        
+        browser_config = BrowserConfig(headless=True)
+        
+        # Test that the crawler accepts the ProxyConfig object without errors
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            try:
+                # Note: This might fail due to actual proxy connection, but should not fail due to config issues
+                result = await crawler.arun(
+                    url=self.test_url,
+                    config=CrawlerRunConfig(
+                        cache_mode=CacheMode.BYPASS,
+                        proxy_config=proxy_config,
+                        page_timeout=10000  # Short timeout for testing
+                    )
+                )
+                # If we get here, proxy config was accepted
+                assert result is not None
+            except Exception as e:
+                # We expect connection errors with test proxies, but not config errors
+                error_msg = str(e).lower()
+                assert "attribute" not in error_msg, f"Config error: {e}"
+                assert "proxy_config" not in error_msg, f"Proxy config error: {e}"
+                
+    @pytest.mark.asyncio
+    async def test_crawler_with_proxy_config_dict(self):
+        """Test AsyncWebCrawler with ProxyConfig from dictionary."""
+        proxy_dict = {
+            "server": f"http://{self.TEST_PROXY_DATA['server']}",
+            "username": self.TEST_PROXY_DATA['username'],
+            "password": self.TEST_PROXY_DATA['password']
+        }
+        proxy_config = ProxyConfig.from_dict(proxy_dict)
+        
+        browser_config = BrowserConfig(headless=True)
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            try:
+                result = await crawler.arun(
+                    url=self.test_url,
+                    config=CrawlerRunConfig(
+                        cache_mode=CacheMode.BYPASS,
+                        proxy_config=proxy_config,
+                        page_timeout=10000
+                    )
+                )
+                assert result is not None
+            except Exception as e:
+                error_msg = str(e).lower()
+                assert "attribute" not in error_msg, f"Config error: {e}"
+                
+    @pytest.mark.asyncio
+    async def test_crawler_with_proxy_config_from_string(self):
+        """Test AsyncWebCrawler with ProxyConfig from string."""
+        proxy_str = f"{self.TEST_PROXY_DATA['ip']}:6114:{self.TEST_PROXY_DATA['username']}:{self.TEST_PROXY_DATA['password']}"
+        proxy_config = ProxyConfig.from_string(proxy_str)
+        
+        browser_config = BrowserConfig(headless=True)
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            try:
+                result = await crawler.arun(
+                    url=self.test_url,
+                    config=CrawlerRunConfig(
+                        cache_mode=CacheMode.BYPASS,
+                        proxy_config=proxy_config,
+                        page_timeout=10000
+                    )
+                )
+                assert result is not None
+            except Exception as e:
+                error_msg = str(e).lower()
+                assert "attribute" not in error_msg, f"Config error: {e}"
+                
+    # ==================== EDGE CASES AND ERROR HANDLING ====================
+    
+    def test_proxy_config_with_none_server(self):
+        """Test ProxyConfig behavior with None server."""
+        proxy = ProxyConfig(server=None)
+        assert proxy.server is None
+        assert proxy.ip is None  # Should not crash
+        
+    def test_proxy_config_with_empty_string_server(self):
+        """Test ProxyConfig behavior with empty string server."""
+        proxy = ProxyConfig(server="")
+        assert proxy.server == ""
+        assert proxy.ip is None or proxy.ip == ""
+        
+    def test_proxy_config_special_characters_in_credentials(self):
+        """Test ProxyConfig with special characters in username/password."""
+        special_chars_tests = [
+            ("user@domain.com", "pass!@#$%"),
+            ("user_123", "p@ssw0rd"),
+            ("user-test", "pass-word"),
+        ]
+        
+        for username, password in special_chars_tests:
+            proxy = ProxyConfig(
+                server="http://127.0.0.1:8080",
+                username=username,
+                password=password
+            )
+            assert proxy.username == username
+            assert proxy.password == password
+            
+    def test_proxy_config_unicode_handling(self):
+        """Test ProxyConfig with unicode characters."""
+        proxy = ProxyConfig(
+            server="http://127.0.0.1:8080",
+            username="ユーザー",  # Japanese characters
+            password="пароль"    # Cyrillic characters
+        )
+        assert proxy.username == "ユーザー"
+        assert proxy.password == "пароль"
+        
+    # ==================== PERFORMANCE TESTS ====================
+    
+    def test_proxy_config_creation_performance(self):
+        """Test that ProxyConfig creation is reasonably fast."""
+        import time
+        
+        start_time = time.time()
+        for i in range(1000):
+            proxy = ProxyConfig(
+                server=f"http://192.168.1.{i % 255}:8080",
+                username=f"user{i}",
+                password=f"pass{i}"
+            )
+        end_time = time.time()
+        
+        # Should be able to create 1000 configs in less than 1 second
+        assert (end_time - start_time) < 1.0
+        
+    def test_proxy_config_from_env_performance(self):
+        """Test that loading many proxies from env is reasonably fast."""
+        import time
+        
+        # Create a large list of proxy strings
+        proxy_list = [f"192.168.1.{i}:8080:user{i}:pass{i}" for i in range(100)]
+        proxy_str = ",".join(proxy_list)
+        
+        with patch.dict(os.environ, {'PERF_TEST_PROXIES': proxy_str}):
+            start_time = time.time()
+            proxies = ProxyConfig.from_env('PERF_TEST_PROXIES')
+            end_time = time.time()
+            
+            assert len(proxies) == 100
+            # Should be able to parse 100 proxies in less than 1 second
+            assert (end_time - start_time) < 1.0
+
+
+# ==================== STANDALONE TEST FUNCTIONS ====================
+
+@pytest.mark.asyncio
+async def test_dict_proxy():
+    """Original test function for dict proxy - kept for backward compatibility."""
+    proxy_config = {
+        "server": "23.95.150.145:6114", 
+        "username": "cfyswbwn",
+        "password": "1gs266hoqysi"
+    }
+    proxy_config_obj = ProxyConfig.from_dict(proxy_config)
+    
+    browser_config = BrowserConfig(headless=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        try:
+            result = await crawler.arun(url="https://httpbin.org/ip", config=CrawlerRunConfig(
+                stream=False,
+                cache_mode=CacheMode.BYPASS,
+                proxy_config=proxy_config_obj,
+                page_timeout=10000
+            ))
+            print("Dict proxy test passed!")
+            print(result.markdown[:200] if result and result.markdown else "No result")
+        except Exception as e:
+            print(f"Dict proxy test error (expected): {e}")
+
+
+@pytest.mark.asyncio
+async def test_string_proxy():
+    """Test function for string proxy format."""
+    proxy_str = "23.95.150.145:6114:cfyswbwn:1gs266hoqysi"
+    proxy_config_obj = ProxyConfig.from_string(proxy_str)
+    
+    browser_config = BrowserConfig(headless=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        try:
+            result = await crawler.arun(url="https://httpbin.org/ip", config=CrawlerRunConfig(
+                stream=False,
+                cache_mode=CacheMode.BYPASS,
+                proxy_config=proxy_config_obj,
+                page_timeout=10000
+            ))
+            print("String proxy test passed!")
+            print(result.markdown[:200] if result and result.markdown else "No result")
+        except Exception as e:
+            print(f"String proxy test error (expected): {e}")
+
+
+@pytest.mark.asyncio
+async def test_env_proxy():
+    """Test function for environment variable proxy."""
+    # Set environment variable
+    os.environ['TEST_PROXIES'] = "23.95.150.145:6114:cfyswbwn:1gs266hoqysi"
+    
+    proxies = ProxyConfig.from_env('TEST_PROXIES')
+    if proxies:
+        proxy_config_obj = proxies[0]  # Use first proxy
+        
+        browser_config = BrowserConfig(headless=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            try:
+                result = await crawler.arun(url="https://httpbin.org/ip", config=CrawlerRunConfig(
+                    stream=False,
+                    cache_mode=CacheMode.BYPASS,
+                    proxy_config=proxy_config_obj,
+                    page_timeout=10000
+                ))
+                print("Environment proxy test passed!")
+                print(result.markdown[:200] if result and result.markdown else "No result")
+            except Exception as e:
+                print(f"Environment proxy test error (expected): {e}")
+    else:
+        print("No proxies loaded from environment")
+
+
+if __name__ == "__main__":
+    print("Running comprehensive ProxyConfig tests...")
+    print("=" * 50)
+    
+    # Run the standalone test functions
+    print("\n1. Testing dict proxy format...")
+    asyncio.run(test_dict_proxy())
+    
+    print("\n2. Testing string proxy format...")
+    asyncio.run(test_string_proxy())
+    
+    print("\n3. Testing environment variable proxy format...")
+    asyncio.run(test_env_proxy())
+    
+    print("\n" + "=" * 50)
+    print("To run the full pytest suite, use: pytest " + __file__)
+    print("=" * 50)
\ No newline at end of file
diff --git a/tests/proxy/test_proxy_deprecation.py b/tests/proxy/test_proxy_deprecation.py
new file mode 100644
index 0000000..95ccfc7
--- /dev/null
+++ b/tests/proxy/test_proxy_deprecation.py
@@ -0,0 +1,42 @@
+import warnings
+
+import pytest
+
+from crawl4ai.async_configs import BrowserConfig, ProxyConfig
+
+
+def test_browser_config_proxy_string_emits_deprecation_and_autoconverts():
+    warnings.simplefilter("always", DeprecationWarning)
+
+    proxy_str = "23.95.150.145:6114:username:password"
+    with warnings.catch_warnings(record=True) as caught:
+        cfg = BrowserConfig(proxy=proxy_str, headless=True)
+
+    dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)]
+    assert dep_warnings, "Expected DeprecationWarning when using BrowserConfig(proxy=...)"
+
+    assert cfg.proxy is None, "cfg.proxy should be None after auto-conversion"
+    assert isinstance(cfg.proxy_config, ProxyConfig), "cfg.proxy_config should be ProxyConfig instance"
+    assert cfg.proxy_config.username == "username"
+    assert cfg.proxy_config.password == "password"
+    assert cfg.proxy_config.server.startswith("http://")
+    assert cfg.proxy_config.server.endswith(":6114")
+
+
+def test_browser_config_with_proxy_config_emits_no_deprecation():
+    warnings.simplefilter("always", DeprecationWarning)
+
+    with warnings.catch_warnings(record=True) as caught:
+        cfg = BrowserConfig(
+            headless=True,
+            proxy_config={
+                "server": "http://127.0.0.1:8080",
+                "username": "u",
+                "password": "p",
+            },
+        )
+
+    dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)]
+    assert not dep_warnings, "Did not expect DeprecationWarning when using proxy_config"
+    assert cfg.proxy is None
+    assert isinstance(cfg.proxy_config, ProxyConfig)
diff --git a/tests/proxy/test_proxy_regression.py b/tests/proxy/test_proxy_regression.py
new file mode 100644
index 0000000..764322d
--- /dev/null
+++ b/tests/proxy/test_proxy_regression.py
@@ -0,0 +1,96 @@
+"""Regression tests for proxy fix:
+1. Persistent context + proxy (new path via launch_persistent_context)
+2. Persistent context WITHOUT proxy (should still use launch_persistent_context)
+3. Non-persistent + proxy on CrawlerRunConfig (existing path, must not break)
+4. Non-persistent, no proxy (basic crawl, must not break)
+"""
+import asyncio
+import os
+import shutil
+import uuid
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_configs import ProxyConfig
+
+TEST_URL = "https://httpbin.org/ip"  # Simple endpoint, returns IP
+
+
+async def test(label, browser_config, run_config=None):
+    print(f"\n{'='*60}")
+    print(f"Test: {label}")
+    print(f"{'='*60}")
+    run_config = run_config or CrawlerRunConfig()
+    try:
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(TEST_URL, config=run_config)
+            print(f"  Status: {result.status_code}")
+            print(f"  HTML bytes: {len(result.html)}")
+            if result.markdown:
+                # httpbin.org/ip returns JSON with "origin" key
+                md = result.markdown.raw_markdown.strip()
+                print(f"  Content: {md[:200]}")
+            if result.error_message:
+                print(f"  ERROR: {result.error_message}")
+            return result
+    except Exception as e:
+        print(f"  EXCEPTION: {e}")
+        return None
+
+
+async def main():
+    proxy = ProxyConfig(
+        server="https://network.joinmassive.com:65535",
+        username="mpuQHs4sWZ-country-US",
+        password="D0yWxVQo8wQ05RWqz1Bn",
+    )
+
+    # 1. Persistent context + proxy (the fixed path)
+    pd = os.path.expanduser(f"~/.crawl4ai/test_{uuid.uuid4().hex[:8]}")
+    os.makedirs(pd, exist_ok=True)
+    try:
+        await test(
+            "Persistent + proxy (launch_persistent_context)",
+            BrowserConfig(
+                headless=True,
+                use_persistent_context=True,
+                user_data_dir=pd,
+                proxy_config=proxy,
+            ),
+        )
+    finally:
+        shutil.rmtree(pd, ignore_errors=True)
+
+    # 2. Persistent context WITHOUT proxy
+    pd2 = os.path.expanduser(f"~/.crawl4ai/test_{uuid.uuid4().hex[:8]}")
+    os.makedirs(pd2, exist_ok=True)
+    try:
+        await test(
+            "Persistent, no proxy (launch_persistent_context)",
+            BrowserConfig(
+                headless=True,
+                use_persistent_context=True,
+                user_data_dir=pd2,
+            ),
+        )
+    finally:
+        shutil.rmtree(pd2, ignore_errors=True)
+
+    # 3. Non-persistent + proxy on CrawlerRunConfig
+    await test(
+        "Non-persistent + proxy on RunConfig",
+        BrowserConfig(headless=True),
+        CrawlerRunConfig(
+            proxy_config=proxy,
+        ),
+    )
+
+    # 4. Basic crawl - no proxy, no persistent
+    await test(
+        "Basic crawl (no proxy, no persistent)",
+        BrowserConfig(headless=True),
+    )
+
+    print("\n" + "="*60)
+    print("All regression tests complete.")
+
+
+asyncio.run(main())
diff --git a/tests/proxy/test_proxy_verify.py b/tests/proxy/test_proxy_verify.py
new file mode 100644
index 0000000..bf9b4f4
--- /dev/null
+++ b/tests/proxy/test_proxy_verify.py
@@ -0,0 +1,109 @@
+"""
+Verify proxies are working and check what IPs they resolve to.
+Then test Chanel through NST proxy (different provider).
+"""
+import requests
+
+# Check our real IP
+def check_ip(label, proxy=None):
+    print(f"\n--- {label} ---")
+    try:
+        kwargs = {"url": "https://httpbin.org/ip", "timeout": 15}
+        if proxy:
+            kwargs["proxies"] = {"https": proxy, "http": proxy}
+        resp = requests.get(**kwargs)
+        print(f"  IP: {resp.json()}")
+    except Exception as e:
+        print(f"  ERROR: {e}")
+
+# Get NST proxy credentials
+def get_nst_proxy(channel_id):
+    token = "NSTPROXY-DA9C7A614946EA8FCEFDA9FD3B3F4A9D"
+    api_url = f"https://api.nstproxy.com/api/v1/generate/apiproxies?count=1&country=US&protocol=http&sessionDuration=0&channelId={channel_id}&token={token}"
+    print(f"\nFetching NST proxy ({channel_id[:8]}...):")
+    print(f"  URL: {api_url}")
+    try:
+        resp = requests.get(api_url, timeout=15)
+        print(f"  HTTP {resp.status_code}")
+        print(f"  Body: {resp.text[:500]}")
+        data = resp.json()
+        if data.get("code") == 200 and data.get("data"):
+            proxy_str = data["data"][0]
+            parts = proxy_str.split(":")
+            if len(parts) == 4:
+                ip, port, user, pwd = parts
+                proxy_url = f"http://{user}:{pwd}@{ip}:{port}"
+                print(f"  Proxy URL: http://{user[:10]}...@{ip}:{port}")
+                return proxy_url
+    except Exception as e:
+        print(f"  ERROR: {e}")
+    return None
+
+# Test Chanel
+def test_chanel(label, proxy=None, use_cffi=False):
+    url = "https://www.chanel.com/us/fashion/handbags/c/1x1x1/"
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+    print(f"\n{'='*60}")
+    print(f"TEST: {label}")
+    try:
+        if use_cffi:
+            from curl_cffi import requests as cffi_requests
+            kwargs = {"url": url, "headers": headers, "impersonate": "chrome", "timeout": 30, "allow_redirects": True}
+            if proxy:
+                kwargs["proxies"] = {"https": proxy, "http": proxy}
+            resp = cffi_requests.get(**kwargs)
+        else:
+            kwargs = {"url": url, "headers": headers, "timeout": 30, "allow_redirects": True}
+            if proxy:
+                kwargs["proxies"] = {"https": proxy, "http": proxy}
+            resp = requests.get(**kwargs)
+
+        blocked = "Access Denied" in resp.text
+        print(f"  Status: {resp.status_code}")
+        print(f"  Size: {len(resp.text):,} bytes")
+        print(f"  Result: {'BLOCKED' if blocked else 'SUCCESS' if resp.status_code == 200 and len(resp.text) > 10000 else 'UNCLEAR'}")
+        if not blocked and resp.status_code == 200:
+            print(f"  First 300 chars: {resp.text[:300]}")
+    except Exception as e:
+        print(f"  ERROR: {e}")
+
+
+if __name__ == "__main__":
+    MASSIVE_RES = "https://mpuQHs4sWZ-country-US:D0yWxVQo8wQ05RWqz1Bn@network.joinmassive.com:65535"
+    MASSIVE_DC = "http://mpuQHs4sWZ-country-US:D0yWxVQo8wQ05RWqz1Bn@isp.joinmassive.com:8000"
+
+    # Step 1: Verify IPs
+    print("="*60)
+    print("STEP 1: Verify proxy IPs")
+    check_ip("Direct (Hetzner)")
+    check_ip("Massive Residential", MASSIVE_RES)
+    check_ip("Massive Datacenter/ISP", MASSIVE_DC)
+
+    # Step 2: Get NST proxies
+    print("\n" + "="*60)
+    print("STEP 2: Get NST proxy credentials")
+    nst_res = get_nst_proxy("7864DDA266D5899C")  # residential
+    nst_dc = get_nst_proxy("AE0C3B5547F8A021")   # datacenter
+
+    if nst_res:
+        check_ip("NST Residential", nst_res)
+    if nst_dc:
+        check_ip("NST Datacenter", nst_dc)
+
+    # Step 3: Test Chanel with all available proxies
+    print("\n" + "="*60)
+    print("STEP 3: Test Chanel.com")
+
+    if nst_res:
+        test_chanel("curl_cffi + NST residential", proxy=nst_res, use_cffi=True)
+        test_chanel("plain requests + NST residential", proxy=nst_res, use_cffi=False)
+
+    if nst_dc:
+        test_chanel("curl_cffi + NST datacenter", proxy=nst_dc, use_cffi=True)
+
+    # Also try Massive ISP/datacenter (different from residential)
+    test_chanel("curl_cffi + Massive ISP", proxy=MASSIVE_DC, use_cffi=True)
diff --git a/tests/proxy/test_sticky_sessions.py b/tests/proxy/test_sticky_sessions.py
new file mode 100644
index 0000000..738240b
--- /dev/null
+++ b/tests/proxy/test_sticky_sessions.py
@@ -0,0 +1,569 @@
+"""
+Comprehensive test suite for Sticky Proxy Sessions functionality.
+
+Tests cover:
+1. Basic sticky session - same proxy for same session_id
+2. Different sessions get different proxies
+3. Session release
+4. TTL expiration
+5. Thread safety / concurrent access
+6. Integration tests with AsyncWebCrawler
+"""
+
+import asyncio
+import os
+import time
+import pytest
+from unittest.mock import patch
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+from crawl4ai.async_configs import CrawlerRunConfig, ProxyConfig
+from crawl4ai.proxy_strategy import RoundRobinProxyStrategy
+from crawl4ai.cache_context import CacheMode
+
+
+class TestRoundRobinProxyStrategySession:
+    """Test suite for RoundRobinProxyStrategy session methods."""
+
+    def setup_method(self):
+        """Setup for each test method."""
+        self.proxies = [
+            ProxyConfig(server=f"http://proxy{i}.test:8080")
+            for i in range(5)
+        ]
+
+    # ==================== BASIC STICKY SESSION TESTS ====================
+
+    @pytest.mark.asyncio
+    async def test_sticky_session_same_proxy(self):
+        """Verify same proxy is returned for same session_id."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        # First call - acquires proxy
+        proxy1 = await strategy.get_proxy_for_session("session-1")
+
+        # Second call - should return same proxy
+        proxy2 = await strategy.get_proxy_for_session("session-1")
+
+        # Third call - should return same proxy
+        proxy3 = await strategy.get_proxy_for_session("session-1")
+
+        assert proxy1 is not None
+        assert proxy1.server == proxy2.server == proxy3.server
+
+    @pytest.mark.asyncio
+    async def test_different_sessions_different_proxies(self):
+        """Verify different session_ids can get different proxies."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        proxy_a = await strategy.get_proxy_for_session("session-a")
+        proxy_b = await strategy.get_proxy_for_session("session-b")
+        proxy_c = await strategy.get_proxy_for_session("session-c")
+
+        # All should be different (round-robin)
+        servers = {proxy_a.server, proxy_b.server, proxy_c.server}
+        assert len(servers) == 3
+
+    @pytest.mark.asyncio
+    async def test_sticky_session_with_regular_rotation(self):
+        """Verify sticky sessions don't interfere with regular rotation."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        # Acquire a sticky session
+        session_proxy = await strategy.get_proxy_for_session("sticky-session")
+
+        # Regular rotation should continue independently
+        regular_proxy1 = await strategy.get_next_proxy()
+        regular_proxy2 = await strategy.get_next_proxy()
+
+        # Sticky session should still return same proxy
+        session_proxy_again = await strategy.get_proxy_for_session("sticky-session")
+
+        assert session_proxy.server == session_proxy_again.server
+        # Regular proxies should rotate
+        assert regular_proxy1.server != regular_proxy2.server
+
+    # ==================== SESSION RELEASE TESTS ====================
+
+    @pytest.mark.asyncio
+    async def test_session_release(self):
+        """Verify session can be released and reacquired."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        # Acquire session
+        proxy1 = await strategy.get_proxy_for_session("session-1")
+        assert strategy.get_session_proxy("session-1") is not None
+
+        # Release session
+        await strategy.release_session("session-1")
+        assert strategy.get_session_proxy("session-1") is None
+
+        # Reacquire - should get a new proxy (next in round-robin)
+        proxy2 = await strategy.get_proxy_for_session("session-1")
+        assert proxy2 is not None
+        # After release, next call gets the next proxy in rotation
+        # (not necessarily the same as before)
+
+    @pytest.mark.asyncio
+    async def test_release_nonexistent_session(self):
+        """Verify releasing non-existent session doesn't raise error."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        # Should not raise
+        await strategy.release_session("nonexistent-session")
+
+    @pytest.mark.asyncio
+    async def test_release_twice(self):
+        """Verify releasing session twice doesn't raise error."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        await strategy.get_proxy_for_session("session-1")
+        await strategy.release_session("session-1")
+        await strategy.release_session("session-1")  # Should not raise
+
+    # ==================== GET SESSION PROXY TESTS ====================
+
+    @pytest.mark.asyncio
+    async def test_get_session_proxy_existing(self):
+        """Verify get_session_proxy returns proxy for existing session."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        acquired = await strategy.get_proxy_for_session("session-1")
+        retrieved = strategy.get_session_proxy("session-1")
+
+        assert retrieved is not None
+        assert acquired.server == retrieved.server
+
+    def test_get_session_proxy_nonexistent(self):
+        """Verify get_session_proxy returns None for non-existent session."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        result = strategy.get_session_proxy("nonexistent-session")
+        assert result is None
+
+    # ==================== TTL EXPIRATION TESTS ====================
+
+    @pytest.mark.asyncio
+    async def test_session_ttl_not_expired(self):
+        """Verify session returns same proxy when TTL not expired."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        # Acquire with 10 second TTL
+        proxy1 = await strategy.get_proxy_for_session("session-1", ttl=10)
+
+        # Immediately request again - should return same proxy
+        proxy2 = await strategy.get_proxy_for_session("session-1", ttl=10)
+
+        assert proxy1.server == proxy2.server
+
+    @pytest.mark.asyncio
+    async def test_session_ttl_expired(self):
+        """Verify new proxy acquired after TTL expires."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        # Acquire with 1 second TTL
+        proxy1 = await strategy.get_proxy_for_session("session-1", ttl=1)
+
+        # Wait for TTL to expire
+        await asyncio.sleep(1.1)
+
+        # Request again - should get new proxy due to expiration
+        proxy2 = await strategy.get_proxy_for_session("session-1", ttl=1)
+
+        # May or may not be same server depending on round-robin state,
+        # but session should have been recreated
+        assert proxy2 is not None
+
+    @pytest.mark.asyncio
+    async def test_get_session_proxy_ttl_expired(self):
+        """Verify get_session_proxy returns None after TTL expires."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        await strategy.get_proxy_for_session("session-1", ttl=1)
+
+        # Wait for expiration
+        await asyncio.sleep(1.1)
+
+        # Should return None for expired session
+        result = strategy.get_session_proxy("session-1")
+        assert result is None
+
+    @pytest.mark.asyncio
+    async def test_cleanup_expired_sessions(self):
+        """Verify cleanup_expired_sessions removes expired sessions."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        # Create sessions with short TTL
+        await strategy.get_proxy_for_session("short-ttl-1", ttl=1)
+        await strategy.get_proxy_for_session("short-ttl-2", ttl=1)
+        # Create session without TTL (should not be cleaned up)
+        await strategy.get_proxy_for_session("no-ttl")
+
+        # Wait for TTL to expire
+        await asyncio.sleep(1.1)
+
+        # Cleanup
+        removed = await strategy.cleanup_expired_sessions()
+
+        assert removed == 2
+        assert strategy.get_session_proxy("short-ttl-1") is None
+        assert strategy.get_session_proxy("short-ttl-2") is None
+        assert strategy.get_session_proxy("no-ttl") is not None
+
+    # ==================== GET ACTIVE SESSIONS TESTS ====================
+
+    @pytest.mark.asyncio
+    async def test_get_active_sessions(self):
+        """Verify get_active_sessions returns all active sessions."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        await strategy.get_proxy_for_session("session-a")
+        await strategy.get_proxy_for_session("session-b")
+        await strategy.get_proxy_for_session("session-c")
+
+        active = strategy.get_active_sessions()
+
+        assert len(active) == 3
+        assert "session-a" in active
+        assert "session-b" in active
+        assert "session-c" in active
+
+    @pytest.mark.asyncio
+    async def test_get_active_sessions_excludes_expired(self):
+        """Verify get_active_sessions excludes expired sessions."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        await strategy.get_proxy_for_session("short-ttl", ttl=1)
+        await strategy.get_proxy_for_session("no-ttl")
+
+        # Before expiration
+        active = strategy.get_active_sessions()
+        assert len(active) == 2
+
+        # Wait for TTL to expire
+        await asyncio.sleep(1.1)
+
+        # After expiration
+        active = strategy.get_active_sessions()
+        assert len(active) == 1
+        assert "no-ttl" in active
+        assert "short-ttl" not in active
+
+    # ==================== THREAD SAFETY TESTS ====================
+
+    @pytest.mark.asyncio
+    async def test_concurrent_session_access(self):
+        """Verify thread-safe access to sessions."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        async def acquire_session(session_id: str):
+            proxy = await strategy.get_proxy_for_session(session_id)
+            await asyncio.sleep(0.01)  # Simulate work
+            return proxy.server
+
+        # Acquire same session from multiple coroutines
+        results = await asyncio.gather(*[
+            acquire_session("shared-session") for _ in range(10)
+        ])
+
+        # All should get same proxy
+        assert len(set(results)) == 1
+
+    @pytest.mark.asyncio
+    async def test_concurrent_different_sessions(self):
+        """Verify concurrent acquisition of different sessions works correctly."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        async def acquire_session(session_id: str):
+            proxy = await strategy.get_proxy_for_session(session_id)
+            await asyncio.sleep(0.01)
+            return (session_id, proxy.server)
+
+        # Acquire different sessions concurrently
+        results = await asyncio.gather(*[
+            acquire_session(f"session-{i}") for i in range(5)
+        ])
+
+        # Each session should have a consistent proxy
+        session_proxies = dict(results)
+        assert len(session_proxies) == 5
+
+        # Verify each session still returns same proxy
+        for session_id, expected_server in session_proxies.items():
+            actual = await strategy.get_proxy_for_session(session_id)
+            assert actual.server == expected_server
+
+    @pytest.mark.asyncio
+    async def test_concurrent_session_acquire_and_release(self):
+        """Verify concurrent acquire and release operations work correctly."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        async def acquire_and_release(session_id: str):
+            proxy = await strategy.get_proxy_for_session(session_id)
+            await asyncio.sleep(0.01)
+            await strategy.release_session(session_id)
+            return proxy.server
+
+        # Run multiple acquire/release cycles concurrently
+        await asyncio.gather(*[
+            acquire_and_release(f"session-{i}") for i in range(10)
+        ])
+
+        # All sessions should be released
+        active = strategy.get_active_sessions()
+        assert len(active) == 0
+
+    # ==================== EMPTY PROXY POOL TESTS ====================
+
+    @pytest.mark.asyncio
+    async def test_empty_proxy_pool_session(self):
+        """Verify behavior with empty proxy pool."""
+        strategy = RoundRobinProxyStrategy()  # No proxies
+
+        result = await strategy.get_proxy_for_session("session-1")
+        assert result is None
+
+    @pytest.mark.asyncio
+    async def test_add_proxies_after_session(self):
+        """Verify adding proxies after session creation works."""
+        strategy = RoundRobinProxyStrategy()
+
+        # No proxies initially
+        result1 = await strategy.get_proxy_for_session("session-1")
+        assert result1 is None
+
+        # Add proxies
+        strategy.add_proxies(self.proxies)
+
+        # Now should work
+        result2 = await strategy.get_proxy_for_session("session-2")
+        assert result2 is not None
+
+
+class TestCrawlerRunConfigSession:
+    """Test CrawlerRunConfig with sticky session parameters."""
+
+    def test_config_has_session_fields(self):
+        """Verify CrawlerRunConfig has sticky session fields."""
+        config = CrawlerRunConfig(
+            proxy_session_id="test-session",
+            proxy_session_ttl=300,
+            proxy_session_auto_release=True
+        )
+
+        assert config.proxy_session_id == "test-session"
+        assert config.proxy_session_ttl == 300
+        assert config.proxy_session_auto_release is True
+
+    def test_config_session_defaults(self):
+        """Verify default values for session fields."""
+        config = CrawlerRunConfig()
+
+        assert config.proxy_session_id is None
+        assert config.proxy_session_ttl is None
+        assert config.proxy_session_auto_release is False
+
+
+class TestCrawlerStickySessionIntegration:
+    """Integration tests for AsyncWebCrawler with sticky sessions."""
+
+    def setup_method(self):
+        """Setup for each test method."""
+        self.proxies = [
+            ProxyConfig(server=f"http://proxy{i}.test:8080")
+            for i in range(3)
+        ]
+        self.test_url = "https://httpbin.org/ip"
+
+    @pytest.mark.asyncio
+    async def test_crawler_sticky_session_without_proxy(self):
+        """Test that crawler works when proxy_session_id set but no strategy."""
+        browser_config = BrowserConfig(headless=True)
+
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            proxy_session_id="test-session",
+            page_timeout=15000
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=self.test_url, config=config)
+            # Should work without errors (no proxy strategy means no proxy)
+            assert result is not None
+
+    @pytest.mark.asyncio
+    async def test_crawler_sticky_session_basic(self):
+        """Test basic sticky session with crawler."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            proxy_rotation_strategy=strategy,
+            proxy_session_id="integration-test",
+            page_timeout=10000
+        )
+
+        browser_config = BrowserConfig(headless=True)
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            # First request
+            try:
+                result1 = await crawler.arun(url=self.test_url, config=config)
+            except Exception:
+                pass  # Proxy connection may fail, but session should be tracked
+
+            # Verify session was created
+            session_proxy = strategy.get_session_proxy("integration-test")
+            assert session_proxy is not None
+
+            # Cleanup
+            await strategy.release_session("integration-test")
+
+    @pytest.mark.asyncio
+    async def test_crawler_rotating_vs_sticky(self):
+        """Compare rotating behavior vs sticky session behavior."""
+        strategy = RoundRobinProxyStrategy(self.proxies)
+
+        # Config WITHOUT sticky session - should rotate
+        rotating_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            proxy_rotation_strategy=strategy,
+            page_timeout=5000
+        )
+
+        # Config WITH sticky session - should use same proxy
+        sticky_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            proxy_rotation_strategy=strategy,
+            proxy_session_id="sticky-test",
+            page_timeout=5000
+        )
+
+        browser_config = BrowserConfig(headless=True)
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            # Track proxy configs used
+            rotating_proxies = []
+            sticky_proxies = []
+
+            # Try rotating requests (may fail due to test proxies, but config should be set)
+            for _ in range(3):
+                try:
+                    await crawler.arun(url=self.test_url, config=rotating_config)
+                except Exception:
+                    pass
+                rotating_proxies.append(rotating_config.proxy_config.server if rotating_config.proxy_config else None)
+
+            # Try sticky requests
+            for _ in range(3):
+                try:
+                    await crawler.arun(url=self.test_url, config=sticky_config)
+                except Exception:
+                    pass
+                sticky_proxies.append(sticky_config.proxy_config.server if sticky_config.proxy_config else None)
+
+            # Rotating should have different proxies (or cycle through them)
+            # Sticky should have same proxy for all requests
+            if all(sticky_proxies):
+                assert len(set(sticky_proxies)) == 1, "Sticky session should use same proxy"
+
+            await strategy.release_session("sticky-test")
+
+
+class TestStickySessionRealWorld:
+    """Real-world scenario tests for sticky sessions.
+
+    Note: These tests require actual proxy servers to verify IP consistency.
+    They are marked to be skipped if no proxy is configured.
+    """
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.environ.get('TEST_PROXY_1'),
+        reason="Requires TEST_PROXY_1 environment variable"
+    )
+    async def test_verify_ip_consistency(self):
+        """Verify that sticky session actually uses same IP.
+
+        This test requires real proxies set in environment variables:
+        TEST_PROXY_1=ip:port:user:pass
+        TEST_PROXY_2=ip:port:user:pass
+        """
+        import re
+
+        # Load proxies from environment
+        proxy_strs = [
+            os.environ.get('TEST_PROXY_1', ''),
+            os.environ.get('TEST_PROXY_2', '')
+        ]
+        proxies = [ProxyConfig.from_string(p) for p in proxy_strs if p]
+
+        if len(proxies) < 2:
+            pytest.skip("Need at least 2 proxies for this test")
+
+        strategy = RoundRobinProxyStrategy(proxies)
+
+        # Config WITH sticky session
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            proxy_rotation_strategy=strategy,
+            proxy_session_id="ip-verify-session",
+            page_timeout=30000
+        )
+
+        browser_config = BrowserConfig(headless=True)
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            ips = []
+
+            for i in range(3):
+                result = await crawler.arun(
+                    url="https://httpbin.org/ip",
+                    config=config
+                )
+
+                if result and result.success and result.html:
+                    # Extract IP from response
+                    ip_match = re.search(r'"origin":\s*"([^"]+)"', result.html)
+                    if ip_match:
+                        ips.append(ip_match.group(1))
+
+            await strategy.release_session("ip-verify-session")
+
+            # All IPs should be same for sticky session
+            if len(ips) >= 2:
+                assert len(set(ips)) == 1, f"Expected same IP, got: {ips}"
+
+
+# ==================== STANDALONE TEST FUNCTIONS ====================
+
+@pytest.mark.asyncio
+async def test_sticky_session_simple():
+    """Simple test for sticky session functionality."""
+    proxies = [
+        ProxyConfig(server=f"http://proxy{i}.test:8080")
+        for i in range(3)
+    ]
+    strategy = RoundRobinProxyStrategy(proxies)
+
+    # Same session should return same proxy
+    p1 = await strategy.get_proxy_for_session("test")
+    p2 = await strategy.get_proxy_for_session("test")
+    p3 = await strategy.get_proxy_for_session("test")
+
+    assert p1.server == p2.server == p3.server
+    print(f"Sticky session works! All requests use: {p1.server}")
+
+    # Cleanup
+    await strategy.release_session("test")
+
+
+if __name__ == "__main__":
+    print("Running Sticky Session tests...")
+    print("=" * 50)
+
+    asyncio.run(test_sticky_session_simple())
+
+    print("\n" + "=" * 50)
+    print("To run the full pytest suite, use: pytest " + __file__)
+    print("=" * 50)
diff --git a/tests/regression/__init__.py b/tests/regression/__init__.py
new file mode 100644
index 0000000..5360a15
--- /dev/null
+++ b/tests/regression/__init__.py
@@ -0,0 +1 @@
+# Crawl4AI Regression Test Suite (crawl4ai-check)
diff --git a/tests/regression/conftest.py b/tests/regression/conftest.py
new file mode 100644
index 0000000..19f195e
--- /dev/null
+++ b/tests/regression/conftest.py
@@ -0,0 +1,628 @@
+"""
+Crawl4AI Regression Test Suite - Shared Fixtures
+
+Provides a local HTTP test server with crafted pages for deterministic testing,
+plus markers for network-dependent tests against real URLs.
+
+Usage:
+    pytest tests/regression/ -v                    # all tests
+    pytest tests/regression/ -v -m "not network"   # skip real URL tests
+    pytest tests/regression/ -v -k "core"          # only core tests
+"""
+
+import pytest
+import socket
+import threading
+import asyncio
+import time
+from aiohttp import web
+
+
+# ---------------------------------------------------------------------------
+# Pytest configuration
+# ---------------------------------------------------------------------------
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "network: tests requiring real network access")
+
+
+# ---------------------------------------------------------------------------
+# Test HTML Pages
+# ---------------------------------------------------------------------------
+
+HOME_HTML = """\
+
+
+
+    
+    Crawl4AI Test Home
+    
+    
+    
+    
+    
+    
+    
+    
+
+
+    
+    
      
+        
      Welcome to the Crawl4AI Test Site
      
+        
      This is a comprehensive test page designed for regression testing of the
+        Crawl4AI web crawling library. It contains various HTML elements to verify
+        content extraction, markdown generation, and link discovery work correctly.
      
+
+        
      Features Overview
      
+        
      The test suite covers multiple aspects of web crawling including content
+        extraction, JavaScript execution, screenshot capture, and deep crawling
+        capabilities. Each feature is tested both with local pages and real URLs.
      
+
+        
        
+            
      • Content extraction and markdown generation
        • 
+            
      • Link discovery and classification
        • 
+            
      • Image extraction and scoring
        • 
+            
      • Table extraction and validation
        • 
+        
      
+
+        
      Code Example
      
+        
      from crawl4ai import AsyncWebCrawler
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun("https://example.com")
+    print(result.markdown)
      
+
+        
      Contact us at test@example.com for more info.
      
+
+        
      Internal Links
      
+        Alpha Page
+        Beta Page
+
+        
      External Links
      
+        Example.com
+        Crawl4AI GitHub
+
+        Hero image for testing
+        
+    
      
+    
      
+        
      Footer content - should be excluded with excluded_tags
      
+    
      
+
+"""
+
+PRODUCTS_HTML = """\
+
+
+
+    Product Listing
+    
+
+
+    
      Products
      
+    
      
+        
      
+            
      Wireless Mouse
      
+            $29.99
+            
      4.5 stars
      
+            
      Ergonomic wireless mouse with precision tracking
      
+            Electronics
+            View Details
+        
      
+        
      
+            
      Mechanical Keyboard
      
+            $89.99
+            
      4.8 stars
      
+            
      Cherry MX switches with RGB backlighting
      
+            Electronics
+            View Details
+        
      
+        
      
+            
      USB-C Hub
      
+            $45.50
+            
      4.2 stars
      
+            
      7-in-1 hub with HDMI, USB-A, SD card reader
      
+            Accessories
+            View Details
+        
      
+        
      
+            
      Monitor Stand
      
+            $34.99
+            
      3.9 stars
      
+            
      Adjustable aluminum monitor riser with storage
      
+            Furniture
+            View Details
+        
      
+        
      
+            
      Webcam HD
      
+            $59.00
+            
      4.6 stars
      
+            
      1080p webcam with built-in microphone and privacy cover
      
+            Electronics
+            View Details
+        
      
+    
      
+
+"""
+
+TABLES_HTML = """\
+
+
+Tables Test
+
+    
      Data Tables
      
+
+    
      Sales Report
      
+    
+        
+            
+        
+        
+            
+            
+            
+            
+        
+    
      QuarterRevenueGrowth
      Q1 2025$1,234,56712.5%
      Q2 2025$1,456,78918.0%
      Q3 2025$1,678,90115.2%
      Q4 2025$1,890,12312.6%
      
+
+    
      Layout Table (should be filtered)
      
+    
+        
+    
      Left columnRight column
      
+
+    
      Employee Directory
      
+    
+        
+            
+        
+        
+            
+            
+            
+        
+    
      NameEmailDepartmentPhone
      Alice Johnsonalice@example.comEngineering+1-555-0101
      Bob Smithbob@example.comMarketing+1-555-0102
      Carol Whitecarol@example.comSales+1-555-0103
      
+
+"""
+
+JS_DYNAMIC_HTML = """\
+
+
+JS Dynamic Content
+
+    
      
+        
      Static Section
      
+        
      This content is immediately available in the HTML.
      
+    
      
+    
      
+    
      0
      
+    
+
+"""
+
+LINKS_HTML = """\
+
+
+Links Collection
+
+    
      Link Collection Page
      
+    
+    
      
+        
      External Resources
      
+        Example Domain
+        GitHub
+        Python
+        Python Docs
+    
      
+    
      
+        
      Social Media
      
+        Twitter
+        Facebook
+        LinkedIn
+    
      
+    
      
+        
      Duplicate Links
      
+        Home Again
+        Example Again
+    
      
+
+"""
+
+IMAGES_HTML = """\
+
+
+Images Gallery
+
+    
      Image Gallery
      
+
+    
+    
      
+        Beautiful mountain landscape at sunset
+        
      A stunning landscape photograph showcasing the beauty of mountain scenery
+        at golden hour. This image demonstrates proper extraction of high-quality
+        photographs with descriptive alt text and surrounding context.
      
+    
      
+
+    
+    Product photograph
+
+    
+    
+
+    
+    Lazy loaded image
+
+    
+    Responsive image with srcset
+
+    
+    
+
+    
+    Company Logo
+
+"""
+
+STRUCTURED_DATA_HTML = """\
+
+
+
+    Article with Structured Data
+    
+    
+    
+    
+    
+    
+    
+    
+    
+
+
+    
      
+        
      Web Crawling Best Practices
      
+        
      By Test Author | Published June 15, 2025
      
+        
      Web crawling is the process of systematically browsing the web to extract
+        information. Modern crawlers like Crawl4AI provide sophisticated tools for
+        content extraction, including markdown generation, structured data extraction,
+        and intelligent link following.
      
+        
      Key Techniques
      
+        
      Understanding how to properly configure a web crawler is essential for
+        efficient data collection. This includes setting appropriate delays, respecting
+        robots.txt, and using proper user agents.
      
+    
      
+
+"""
+
+EMPTY_HTML = """\
+
+Empty Page
+
+"""
+
+MALFORMED_HTML = """\
+
+Malformed Page</head>
+<body>
+<div>
+<p>Unclosed paragraph
+<p>Another paragraph without closing
+<img src="/test.jpg" alt="no closing bracket"
+<a href="/broken>Broken link</a>
+<div><span>Nested but unclosed
+<table><tr><td>Cell without closing tags
+</body>
+</html>"""
+
+REGEX_TEST_HTML = """\
+<!DOCTYPE html>
+<html>
+<head><title>Regex Test Content
+
+    
      Contact Information
      
+    
      Email us at support@crawl4ai.com or sales@example.org for inquiries.
      
+    
      Call us: +1-555-123-4567 or (800) 555-0199
      
+    
      Visit https://crawl4ai.com or https://docs.crawl4ai.com/api/v2
      
+    
      Server IP: 192.168.1.100
      
+    
      Request ID: 550e8400-e29b-41d4-a716-446655440000
      
+    
      Price: $199.99 or EUR 175.50
      
+    
      Completion rate: 95.7%
      
+    
      Published: 2025-03-15
      
+    
      Updated: 03/15/2025
      
+    
      Meeting at 14:30 or 09:00
      
+    
      Zip code: 94105 or 94105-1234
      
+    
      Follow @crawl4ai on social media
      
+    
      Tags: #WebCrawling #DataExtraction #Python
      
+    
      Color theme: #FF5733
      
+
+"""
+
+
+def _generate_large_html(num_sections=50):
+    """Generate a large HTML page with many sections."""
+    sections = []
+    for i in range(num_sections):
+        sections.append(f"""
+        
      
+            
      Section {i}: Important Topic Number {i}
      
+            
      This is paragraph one of section {i}. It contains enough text to be
+            meaningful for content extraction and markdown generation testing purposes.
+            The crawler should properly handle large pages with many sections.
      
+            
      This is paragraph two of section {i}. It provides additional context
+            and detail about topic {i}, ensuring that the content extraction pipeline
+            can handle substantial amounts of text without issues.
      
+            Read more about topic {i}
+        
      """)
+    return f"""\
+
+
+Large Page with Many Sections
+
+    
      Comprehensive Document
      
+    {"".join(sections)}
+
+"""
+
+LARGE_HTML = _generate_large_html(50)
+
+
+# Deep crawl pages: hub -> sub1,sub2,sub3 -> leaf pages
+DEEP_HUB_HTML = """\
+
+
+Deep Crawl Hub
+
+    
      Hub Page
      
+    
      This is the starting point for deep crawl testing.
      
+    
+
+"""
+
+DEEP_SUB_TEMPLATE = """\
+
+
+Deep Crawl - {title}
+
+    
      {title}
      
+    
      Content about {title}. This sub-page contains links to deeper content.
      
+    Leaf A under {title}
+    Leaf B under {title}
+    Back to Hub
+
+"""
+
+DEEP_LEAF_TEMPLATE = """\
+
+
+Deep Crawl - {title}
+
+    
      {title}
      
+    
      This is a leaf page in the deep crawl hierarchy. It contains substantial
+    content about {title} to ensure proper extraction at all crawl depths.
+    The adaptive crawler should find and process this content correctly.
      
+    Back to Hub
+
+"""
+
+IFRAME_HTML = """\
+
+
+Page with Iframes
+
+    
      Main Page Content
      
+    
      This page contains embedded iframes for testing iframe processing.
      
+    
+    
+
+"""
+
+
+# ---------------------------------------------------------------------------
+# Server Handlers
+# ---------------------------------------------------------------------------
+
+async def _serve_html(html, content_type="text/html"):
+    return web.Response(text=html, content_type=content_type)
+
+
+async def _home_handler(request):
+    return await _serve_html(HOME_HTML)
+
+async def _products_handler(request):
+    return await _serve_html(PRODUCTS_HTML)
+
+async def _tables_handler(request):
+    return await _serve_html(TABLES_HTML)
+
+async def _js_dynamic_handler(request):
+    return await _serve_html(JS_DYNAMIC_HTML)
+
+async def _links_handler(request):
+    return await _serve_html(LINKS_HTML)
+
+async def _images_handler(request):
+    return await _serve_html(IMAGES_HTML)
+
+async def _structured_handler(request):
+    return await _serve_html(STRUCTURED_DATA_HTML)
+
+async def _empty_handler(request):
+    return await _serve_html(EMPTY_HTML)
+
+async def _malformed_handler(request):
+    return await _serve_html(MALFORMED_HTML)
+
+async def _regex_test_handler(request):
+    return await _serve_html(REGEX_TEST_HTML)
+
+async def _large_handler(request):
+    return await _serve_html(LARGE_HTML)
+
+async def _iframe_handler(request):
+    return await _serve_html(IFRAME_HTML)
+
+async def _redirect_handler(request):
+    raise web.HTTPFound("/")
+
+async def _not_found_handler(request):
+    return web.Response(
+        text="404 Not Found"
+             "
      Page Not Found
      The requested page does not exist.
      ",
+        status=404, content_type="text/html",
+    )
+
+async def _slow_handler(request):
+    await asyncio.sleep(2)
+    return await _serve_html(
+        "Slow Page"
+        "
      Slow Response
      This page had a 2-second delay.
      "
+    )
+
+async def _deep_hub_handler(request):
+    return await _serve_html(DEEP_HUB_HTML)
+
+async def _deep_sub_handler(request):
+    sub_id = request.match_info["sub_id"]
+    titles = {"sub1": "Technology", "sub2": "Science", "sub3": "Arts"}
+    title = titles.get(sub_id, f"Sub {sub_id}")
+    html = DEEP_SUB_TEMPLATE.format(title=title, prefix=sub_id)
+    return await _serve_html(html)
+
+async def _deep_leaf_handler(request):
+    sub_id = request.match_info["sub_id"]
+    leaf_id = request.match_info["leaf_id"]
+    title = f"Leaf {leaf_id} under {sub_id}"
+    html = DEEP_LEAF_TEMPLATE.format(title=title)
+    return await _serve_html(html)
+
+async def _catch_all_handler(request):
+    """Serve a simple page for any unmatched path (useful for link targets)."""
+    path = request.path
+    return await _serve_html(
+        f"Page: {path}"
+        f"
      Page at {path}
      "
+        f"
      Auto-generated page for path: {path}
      "
+        f'Back to Home'
+    )
+
+
+# ---------------------------------------------------------------------------
+# Server Setup
+# ---------------------------------------------------------------------------
+
+def _find_free_port():
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("", 0))
+        return s.getsockname()[1]
+
+
+def _create_app():
+    app = web.Application()
+    app.router.add_get("/", _home_handler)
+    app.router.add_get("/products", _products_handler)
+    app.router.add_get("/tables", _tables_handler)
+    app.router.add_get("/js-dynamic", _js_dynamic_handler)
+    app.router.add_get("/links-page", _links_handler)
+    app.router.add_get("/images-page", _images_handler)
+    app.router.add_get("/structured-data", _structured_handler)
+    app.router.add_get("/empty", _empty_handler)
+    app.router.add_get("/malformed", _malformed_handler)
+    app.router.add_get("/regex-test", _regex_test_handler)
+    app.router.add_get("/large", _large_handler)
+    app.router.add_get("/iframe-page", _iframe_handler)
+    app.router.add_get("/redirect", _redirect_handler)
+    app.router.add_get("/not-found", _not_found_handler)
+    app.router.add_get("/slow", _slow_handler)
+    app.router.add_get("/deep/hub", _deep_hub_handler)
+    app.router.add_get("/deep/{sub_id}", _deep_sub_handler)
+    app.router.add_get("/deep/{sub_id}/{leaf_id}", _deep_leaf_handler)
+    # Catch-all for auto-generated pages (internal link targets, etc.)
+    app.router.add_get("/{path:.*}", _catch_all_handler)
+    return app
+
+
+def _run_server(app, host, port, ready_event):
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    runner = web.AppRunner(app)
+    loop.run_until_complete(runner.setup())
+    site = web.TCPSite(runner, host, port)
+    loop.run_until_complete(site.start())
+    ready_event.set()
+    try:
+        loop.run_forever()
+    finally:
+        loop.run_until_complete(runner.cleanup())
+        loop.close()
+
+
+@pytest.fixture(scope="session")
+def local_server():
+    """Start a local HTTP test server. Returns base URL like 'http://localhost:PORT'."""
+    port = _find_free_port()
+    app = _create_app()
+    ready = threading.Event()
+    thread = threading.Thread(
+        target=_run_server,
+        args=(app, "localhost", port, ready),
+        daemon=True,
+    )
+    thread.start()
+    assert ready.wait(timeout=10), "Test server failed to start within 10 seconds"
+    # Small delay to ensure server is fully ready
+    time.sleep(0.2)
+    yield f"http://localhost:{port}"
+    # Daemon thread cleans up automatically
+
+
+# ---------------------------------------------------------------------------
+# Common test constants
+# ---------------------------------------------------------------------------
+
+# Stable real URLs for network tests
+REAL_URL_SIMPLE = "https://example.com"
+REAL_URL_QUOTES = "https://quotes.toscrape.com"
+REAL_URL_BOOKS = "https://books.toscrape.com"
diff --git a/tests/regression/test_reg_browser.py b/tests/regression/test_reg_browser.py
new file mode 100644
index 0000000..ba90117
--- /dev/null
+++ b/tests/regression/test_reg_browser.py
@@ -0,0 +1,561 @@
+"""
+Crawl4AI Regression Tests - Browser Management and Features
+
+Tests browser lifecycle, viewport configuration, wait_for conditions, JavaScript
+execution, page interaction, screenshots, iframe processing, overlay removal,
+stealth mode, session management, network capture, and anti-bot features using
+real browser crawling with no mocking.
+"""
+
+import base64
+import time
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.cache_context import CacheMode
+
+
+# ---------------------------------------------------------------------------
+# Browser lifecycle
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_browser_lifecycle(local_server):
+    """Create crawler, start, crawl, and close explicitly without context manager."""
+    crawler = AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False))
+    await crawler.start()
+    try:
+        result = await crawler.arun(
+            url=local_server + "/",
+            config=CrawlerRunConfig(verbose=False),
+        )
+        assert result.success, f"Crawl failed: {result.error_message}"
+        assert len(result.html) > 0, "HTML should be non-empty"
+    finally:
+        await crawler.close()
+
+
+@pytest.mark.asyncio
+async def test_browser_context_manager(local_server):
+    """Verify async with pattern works and cleanup happens without error."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url=local_server + "/",
+            config=CrawlerRunConfig(verbose=False),
+        )
+        assert result.success, f"Context manager crawl failed: {result.error_message}"
+    # If we get here without exception, cleanup succeeded
+
+
+# ---------------------------------------------------------------------------
+# Viewport configuration
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_custom_viewport(local_server):
+    """Create BrowserConfig with 1920x1080 viewport and verify crawl succeeds."""
+    browser_config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        viewport_width=1920,
+        viewport_height=1080,
+    )
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=local_server + "/",
+            config=CrawlerRunConfig(verbose=False),
+        )
+        assert result.success, f"Custom viewport crawl failed: {result.error_message}"
+
+
+@pytest.mark.asyncio
+async def test_small_viewport(local_server):
+    """Mobile-like viewport (375x667) should still produce a successful crawl."""
+    browser_config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        viewport_width=375,
+        viewport_height=667,
+    )
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=local_server + "/",
+            config=CrawlerRunConfig(verbose=False),
+        )
+        assert result.success, f"Small viewport crawl failed: {result.error_message}"
+
+
+# ---------------------------------------------------------------------------
+# wait_for conditions
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_wait_for_css_selector(local_server):
+    """Wait for a CSS selector on /js-dynamic and verify dynamic content loaded."""
+    config = CrawlerRunConfig(wait_for="css:.js-loaded", verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/js-dynamic", config=config)
+        assert result.success, f"wait_for CSS crawl failed: {result.error_message}"
+        assert "Dynamic content successfully loaded" in (result.markdown or ""), (
+            "Dynamic JS content should appear after waiting for .js-loaded"
+        )
+
+
+@pytest.mark.asyncio
+async def test_wait_for_js_function(local_server):
+    """Wait for a JS condition on /js-dynamic and verify the counter value."""
+    config = CrawlerRunConfig(
+        wait_for="js:() => document.getElementById('counter').textContent === '42'",
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/js-dynamic", config=config)
+        assert result.success, f"wait_for JS crawl failed: {result.error_message}"
+        assert "42" in (result.html or ""), (
+            "Counter should be set to 42 after JS wait condition is met"
+        )
+
+
+@pytest.mark.asyncio
+async def test_wait_for_timeout(local_server):
+    """Wait for a non-existent selector with short timeout should not hang forever."""
+    config = CrawlerRunConfig(
+        wait_for="css:.nonexistent-class",
+        wait_for_timeout=500,
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        # This may succeed (with timeout warning) or fail, but should not hang
+        result = await crawler.arun(url=local_server + "/js-dynamic", config=config)
+        # We just verify it returned without hanging; success or failure is acceptable
+        assert result is not None, "Should return a result even if wait_for times out"
+
+
+# ---------------------------------------------------------------------------
+# JavaScript execution
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_js_code_modifies_dom(local_server):
+    """Execute JS that adds a DOM element and verify it appears in the result."""
+    config = CrawlerRunConfig(
+        js_code='document.body.innerHTML += \'
      Injected by JS
      \';',
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, f"JS DOM modification crawl failed: {result.error_message}"
+        combined = (result.html or "") + (result.markdown or "")
+        assert "Injected by JS" in combined, (
+            "Injected content should appear in HTML or markdown"
+        )
+
+
+@pytest.mark.asyncio
+async def test_js_code_returns_value(local_server):
+    """Execute JS that returns document.title and check js_execution_result."""
+    config = CrawlerRunConfig(
+        js_code="return document.title;",
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, f"JS return value crawl failed: {result.error_message}"
+        # js_execution_result should contain the returned value
+        if result.js_execution_result is not None:
+            # The result might be stored under a key or directly
+            result_str = str(result.js_execution_result)
+            assert "Crawl4AI Test Home" in result_str or len(result_str) > 0, (
+                "js_execution_result should contain the document title"
+            )
+
+
+@pytest.mark.asyncio
+async def test_multiple_js_scripts(local_server):
+    """Execute multiple JS scripts sequentially; last one sets title to 'B'."""
+    config = CrawlerRunConfig(
+        js_code=["document.title='A';", "document.title='B';"],
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, f"Multiple JS scripts crawl failed: {result.error_message}"
+        # Both scripts should have executed; title should end up as 'B'
+        # We can check via the HTML title tag or via another JS execution
+        # The HTML might still have the original title in source, but the page state changed
+
+
+# ---------------------------------------------------------------------------
+# Page interaction
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_scan_full_page(local_server):
+    """Crawl /large with scan_full_page=True and verify bottom sections appear."""
+    config = CrawlerRunConfig(
+        scan_full_page=True,
+        scroll_delay=0.05,
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/large", config=config)
+        assert result.success, f"Full page scan crawl failed: {result.error_message}"
+        # The large page has 50 sections; verify some from near the bottom
+        combined = (result.html or "") + (result.markdown or "")
+        assert "Section 49" in combined, (
+            "Scanning the full page should reveal the last section (Section 49)"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Screenshot features
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_screenshot_basic(local_server):
+    """Crawl with screenshot=True, decode base64, and verify PNG header."""
+    config = CrawlerRunConfig(screenshot=True, verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, f"Screenshot crawl failed: {result.error_message}"
+        assert result.screenshot, "Screenshot should be a non-empty base64 string"
+        raw_bytes = base64.b64decode(result.screenshot)
+        assert raw_bytes[:4] == b"\x89PNG", (
+            "Screenshot should be in PNG format"
+        )
+
+
+@pytest.mark.asyncio
+async def test_force_viewport_screenshot(local_server):
+    """Crawl /large with force_viewport_screenshot=True; should capture viewport only."""
+    config = CrawlerRunConfig(
+        screenshot=True,
+        force_viewport_screenshot=True,
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/large", config=config)
+        assert result.success, f"Force viewport screenshot crawl failed: {result.error_message}"
+        assert result.screenshot, "Screenshot should be captured"
+        raw_bytes = base64.b64decode(result.screenshot)
+        assert raw_bytes[:4] == b"\x89PNG", "Viewport screenshot should be PNG"
+
+
+# ---------------------------------------------------------------------------
+# Process iframes
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_process_iframes(local_server):
+    """Crawl /iframe-page with process_iframes=True and verify iframe content appears."""
+    config = CrawlerRunConfig(process_iframes=True, verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/iframe-page", config=config)
+        assert result.success, f"Iframe processing crawl failed: {result.error_message}"
+        combined = (result.html or "") + (result.markdown or "")
+        # At least one iframe's content should appear
+        has_iframe_content = (
+            "Iframe 1 content" in combined
+            or "Iframe 2 heading" in combined
+            or "embedded" in combined.lower()
+        )
+        assert has_iframe_content, (
+            "Iframe content should appear in the result when process_iframes=True"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Overlay and popup removal
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_remove_overlay_elements(local_server):
+    """Crawl with remove_overlay_elements=True; verify it does not break crawling."""
+    config = CrawlerRunConfig(remove_overlay_elements=True, verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, (
+            f"Overlay removal should not break crawling: {result.error_message}"
+        )
+        assert len(result.html) > 0, "HTML should still be present after overlay removal"
+
+
+# ---------------------------------------------------------------------------
+# Stealth mode
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_stealth_mode_no_crash(local_server):
+    """Stealth mode should not break basic local crawling."""
+    browser_config = BrowserConfig(
+        headless=True,
+        verbose=False,
+        enable_stealth=True,
+    )
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=local_server + "/",
+            config=CrawlerRunConfig(verbose=False),
+        )
+        assert result.success, f"Stealth mode crawl failed: {result.error_message}"
+        assert "Crawl4AI Test Home" in (result.html or ""), (
+            "Stealth mode should still extract content correctly"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Session management
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_session_persistence(local_server):
+    """Session state should persist between crawls with the same session_id."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        # First crawl: set a JS variable
+        config1 = CrawlerRunConfig(
+            session_id="persist-test",
+            js_code="window.__testVar = 'hello';",
+            verbose=False,
+        )
+        result1 = await crawler.arun(url=local_server + "/", config=config1)
+        assert result1.success, f"First session crawl failed: {result1.error_message}"
+
+        # Second crawl: read the JS variable using js_only mode
+        config2 = CrawlerRunConfig(
+            session_id="persist-test",
+            js_only=True,
+            js_code="return window.__testVar;",
+            verbose=False,
+        )
+        result2 = await crawler.arun(url=local_server + "/", config=config2)
+        assert result2.success, f"Second session crawl failed: {result2.error_message}"
+
+        # Check if testVar persisted
+        if result2.js_execution_result is not None:
+            result_str = str(result2.js_execution_result)
+            assert "hello" in result_str, (
+                f"Session variable should persist; got: {result_str}"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Delay before return HTML
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_delay_before_return(local_server):
+    """Crawl with delay_before_return_html=0.5 should succeed and take reasonable time."""
+    config = CrawlerRunConfig(delay_before_return_html=0.5, verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        start_time = time.monotonic()
+        result = await crawler.arun(url=local_server + "/", config=config)
+        elapsed = time.monotonic() - start_time
+
+        assert result.success, f"Delayed crawl failed: {result.error_message}"
+        assert elapsed >= 0.4, (
+            f"Crawl with 0.5s delay should take at least 0.4s, took {elapsed:.2f}s"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Network features
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_capture_network_requests(local_server):
+    """Crawl /js-dynamic with capture_network_requests=True and verify list returned."""
+    config = CrawlerRunConfig(
+        capture_network_requests=True,
+        cache_mode=CacheMode.BYPASS,
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/js-dynamic", config=config)
+        assert result.success, f"Network capture crawl failed: {result.error_message}"
+        assert result.network_requests is not None, "network_requests should not be None"
+        assert isinstance(result.network_requests, list), (
+            "network_requests should be a list"
+        )
+        assert len(result.network_requests) >= 1, (
+            "Should capture at least 1 network request (the page itself)"
+        )
+
+
+@pytest.mark.asyncio
+async def test_capture_console_messages(local_server):
+    """Crawl with capture_console_messages=True and verify the attribute is a list."""
+    config = CrawlerRunConfig(
+        capture_console_messages=True,
+        cache_mode=CacheMode.BYPASS,
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, f"Console capture crawl failed: {result.error_message}"
+        assert result.console_messages is not None, (
+            "console_messages should not be None when capture is enabled"
+        )
+        assert isinstance(result.console_messages, list), (
+            "console_messages should be a list"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Real URL browser tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_real_url_with_wait():
+    """Crawl https://quotes.toscrape.com with wait_until='load' and verify content."""
+    config = CrawlerRunConfig(wait_until="load", verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url="https://quotes.toscrape.com", config=config)
+        assert result.success, f"Real URL crawl failed: {result.error_message}"
+        assert len(result.html) > 100, "Real page should have substantial HTML"
+        combined = (result.markdown or "") + (result.html or "")
+        assert "quote" in combined.lower() or "quotes" in combined.lower(), (
+            "Quotes page should contain the word 'quote'"
+        )
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_real_url_screenshot():
+    """Crawl https://example.com with screenshot=True and verify PNG captured."""
+    config = CrawlerRunConfig(screenshot=True, verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url="https://example.com", config=config)
+        assert result.success, f"Real URL screenshot crawl failed: {result.error_message}"
+        assert result.screenshot, "Screenshot should be non-empty"
+        raw_bytes = base64.b64decode(result.screenshot)
+        assert raw_bytes[:4] == b"\x89PNG", "Real URL screenshot should be PNG format"
+
+
+# ---------------------------------------------------------------------------
+# Anti-bot basic check
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_magic_mode_no_crash(local_server):
+    """Magic mode should not break normal local crawling."""
+    config = CrawlerRunConfig(magic=True, verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, (
+            f"Magic mode should not break crawling: {result.error_message}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Edge cases
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_crawl_empty_page(local_server):
+    """Crawling a page with empty body should not crash, even if anti-bot flags it."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url=local_server + "/empty",
+            config=CrawlerRunConfig(verbose=False),
+        )
+        # Anti-bot detection may flag near-empty pages as blocked, which is expected
+        # behavior. The key assertion is that it returns a result without crashing.
+        assert result is not None, "Should return a result even for empty page"
+        assert result.html is not None, "HTML should not be None for empty page"
+        if not result.success:
+            assert "empty" in (result.error_message or "").lower() or "blocked" in (result.error_message or "").lower(), (
+                f"Empty page failure should mention empty/blocked content: {result.error_message}"
+            )
+
+
+@pytest.mark.asyncio
+async def test_crawl_malformed_html(local_server):
+    """Crawling malformed HTML should not crash, even if anti-bot flags it."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url=local_server + "/malformed",
+            config=CrawlerRunConfig(verbose=False),
+        )
+        # Anti-bot may flag malformed HTML as blocked due to minimal visible text.
+        # The key assertion is that it returns a result without crashing.
+        assert result is not None, "Should return a result for malformed HTML"
+        assert result.html is not None, "HTML should not be None even for malformed input"
+        # The content is present in the HTML even if the crawl is marked as not successful
+        assert "Unclosed paragraph" in (result.html or "") or "Malformed" in (result.html or ""), (
+            "Some original content should appear in the HTML"
+        )
+
+
+@pytest.mark.asyncio
+async def test_multiple_crawls_same_crawler(local_server):
+    """A single crawler instance should handle multiple sequential crawls."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        urls = [
+            local_server + "/",
+            local_server + "/products",
+            local_server + "/js-dynamic",
+        ]
+        for url in urls:
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(verbose=False),
+            )
+            assert result.success, f"Sequential crawl of {url} failed: {result.error_message}"
+
+
+@pytest.mark.asyncio
+async def test_screenshot_not_captured_by_default(local_server):
+    """Without screenshot=True, result.screenshot should be None or empty."""
+    config = CrawlerRunConfig(screenshot=False, verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, f"No-screenshot crawl failed: {result.error_message}"
+        assert not result.screenshot, (
+            "Screenshot should be None or empty when not requested"
+        )
+
+
+@pytest.mark.asyncio
+async def test_js_code_empty_string(local_server):
+    """Empty js_code string should not cause errors."""
+    config = CrawlerRunConfig(js_code="", verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, (
+            f"Empty js_code should not break crawling: {result.error_message}"
+        )
+
+
+@pytest.mark.asyncio
+async def test_wait_until_load(local_server):
+    """wait_until='load' should wait for full page load including resources."""
+    config = CrawlerRunConfig(wait_until="load", verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, f"wait_until=load crawl failed: {result.error_message}"
+
+
+@pytest.mark.asyncio
+async def test_wait_until_networkidle(local_server):
+    """wait_until='networkidle' should wait until network is idle."""
+    config = CrawlerRunConfig(wait_until="networkidle", verbose=False)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=local_server + "/", config=config)
+        assert result.success, f"wait_until=networkidle crawl failed: {result.error_message}"
diff --git a/tests/regression/test_reg_config.py b/tests/regression/test_reg_config.py
new file mode 100644
index 0000000..9e47b47
--- /dev/null
+++ b/tests/regression/test_reg_config.py
@@ -0,0 +1,776 @@
+"""
+Regression tests for Crawl4AI configuration objects.
+
+Covers BrowserConfig, CrawlerRunConfig, ProxyConfig, GeolocationConfig,
+deep_merge logic, and serialization roundtrips.
+"""
+
+import copy
+import pytest
+
+from crawl4ai import (
+    BrowserConfig,
+    CrawlerRunConfig,
+    ProxyConfig,
+    GeolocationConfig,
+    CacheMode,
+)
+from crawl4ai.async_configs import to_serializable_dict, from_serializable_dict
+
+
+# ---------------------------------------------------------------------------
+# Helper: deep_merge (copied from deploy/docker/utils.py to avoid dns dep)
+# ---------------------------------------------------------------------------
+
+def _deep_merge(base, override):
+    """Recursively merge override into base dict."""
+    result = base.copy()
+    for key, value in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+            result[key] = _deep_merge(result[key], value)
+        else:
+            result[key] = value
+    return result
+
+
+# ===================================================================
+# BrowserConfig
+# ===================================================================
+
+class TestBrowserConfigDefaults:
+    """Verify BrowserConfig default values are sensible."""
+
+    def test_headless_default(self):
+        """Default headless should be True."""
+        cfg = BrowserConfig()
+        assert cfg.headless is True
+
+    def test_browser_type_default(self):
+        """Default browser_type should be 'chromium'."""
+        cfg = BrowserConfig()
+        assert cfg.browser_type == "chromium"
+
+    def test_viewport_defaults(self):
+        """Default viewport should be 1080x600."""
+        cfg = BrowserConfig()
+        assert cfg.viewport_width == 1080
+        assert cfg.viewport_height == 600
+
+    def test_javascript_enabled_default(self):
+        """JavaScript should be enabled by default."""
+        cfg = BrowserConfig()
+        assert cfg.java_script_enabled is True
+
+    def test_ignore_https_errors_default(self):
+        """HTTPS errors should be ignored by default."""
+        cfg = BrowserConfig()
+        assert cfg.ignore_https_errors is True
+
+    def test_stealth_disabled_default(self):
+        """Stealth should be disabled by default."""
+        cfg = BrowserConfig()
+        assert cfg.enable_stealth is False
+
+    def test_browser_mode_default(self):
+        """Default browser_mode should be 'dedicated'."""
+        cfg = BrowserConfig()
+        assert cfg.browser_mode == "dedicated"
+
+
+class TestBrowserConfigRoundtrip:
+    """Verify to_dict -> from_kwargs roundtrip preserves fields."""
+
+    def test_basic_roundtrip(self):
+        """to_dict -> from_kwargs should preserve basic scalar fields."""
+        original = BrowserConfig(
+            headless=False,
+            viewport_width=1920,
+            viewport_height=1080,
+            browser_type="firefox",
+            text_mode=True,
+        )
+        d = original.to_dict()
+        restored = BrowserConfig.from_kwargs(d)
+
+        assert restored.headless is False
+        assert restored.viewport_width == 1920
+        assert restored.viewport_height == 1080
+        assert restored.browser_type == "firefox"
+        assert restored.text_mode is True
+
+    def test_roundtrip_preserves_extra_args(self):
+        """Extra args list should survive roundtrip."""
+        original = BrowserConfig(extra_args=["--no-sandbox", "--disable-dev-shm-usage"])
+        d = original.to_dict()
+        restored = BrowserConfig.from_kwargs(d)
+        assert restored.extra_args == ["--no-sandbox", "--disable-dev-shm-usage"]
+
+    def test_roundtrip_preserves_headers(self):
+        """Custom headers dict should survive roundtrip."""
+        headers = {"X-Custom": "test-value", "Accept-Language": "en-US"}
+        original = BrowserConfig(headers=headers)
+        d = original.to_dict()
+        restored = BrowserConfig.from_kwargs(d)
+        assert restored.headers["X-Custom"] == "test-value"
+        assert restored.headers["Accept-Language"] == "en-US"
+
+    def test_roundtrip_preserves_cookies(self):
+        """Cookies list should survive roundtrip."""
+        cookies = [{"name": "session", "value": "abc123", "url": "http://example.com"}]
+        original = BrowserConfig(cookies=cookies)
+        d = original.to_dict()
+        restored = BrowserConfig.from_kwargs(d)
+        assert len(restored.cookies) == 1
+        assert restored.cookies[0]["name"] == "session"
+
+
+class TestBrowserConfigClone:
+    """Verify clone() creates independent copy with overrides."""
+
+    def test_clone_with_override(self):
+        """Clone should apply overrides while keeping other fields."""
+        original = BrowserConfig(headless=True, viewport_width=1080)
+        cloned = original.clone(headless=False, viewport_width=1920)
+
+        assert cloned.headless is False
+        assert cloned.viewport_width == 1920
+        # Original unchanged
+        assert original.headless is True
+        assert original.viewport_width == 1080
+
+    def test_clone_independence(self):
+        """Clone should produce a distinct object with same scalar values."""
+        original = BrowserConfig(headless=True, viewport_width=1080)
+        cloned = original.clone()
+        cloned.headless = False
+        cloned.viewport_width = 1920
+        # Scalar mutations on clone should not affect original
+        assert original.headless is True
+        assert original.viewport_width == 1080
+
+    def test_clone_preserves_unmodified(self):
+        """Fields not in overrides should be preserved."""
+        original = BrowserConfig(
+            browser_type="firefox",
+            text_mode=True,
+            verbose=False,
+        )
+        cloned = original.clone(verbose=True)
+        assert cloned.browser_type == "firefox"
+        assert cloned.text_mode is True
+        assert cloned.verbose is True
+
+
+class TestBrowserConfigClassDefaults:
+    """Verify set_defaults / get_defaults / reset_defaults class-level defaults."""
+
+    def test_set_defaults_affects_new_instances(self):
+        """set_defaults(headless=False) should make new instances headless=False."""
+        try:
+            BrowserConfig.set_defaults(headless=False)
+            cfg = BrowserConfig()
+            assert cfg.headless is False
+        finally:
+            BrowserConfig.reset_defaults()
+
+    def test_explicit_arg_overrides_class_default(self):
+        """Explicit constructor arg should override class-level default."""
+        try:
+            BrowserConfig.set_defaults(headless=False)
+            cfg = BrowserConfig(headless=True)
+            assert cfg.headless is True
+        finally:
+            BrowserConfig.reset_defaults()
+
+    def test_get_defaults_returns_copy(self):
+        """get_defaults() should return the current overrides."""
+        try:
+            BrowserConfig.set_defaults(viewport_width=1920)
+            defaults = BrowserConfig.get_defaults()
+            assert defaults["viewport_width"] == 1920
+        finally:
+            BrowserConfig.reset_defaults()
+
+    def test_reset_defaults_clears_all(self):
+        """reset_defaults() should clear all overrides."""
+        try:
+            BrowserConfig.set_defaults(headless=False, viewport_width=1920)
+            BrowserConfig.reset_defaults()
+            defaults = BrowserConfig.get_defaults()
+            assert len(defaults) == 0
+            cfg = BrowserConfig()
+            assert cfg.headless is True
+            assert cfg.viewport_width == 1080
+        finally:
+            BrowserConfig.reset_defaults()
+
+    def test_reset_defaults_selective(self):
+        """reset_defaults('headless') should only clear that one override."""
+        try:
+            BrowserConfig.set_defaults(headless=False, viewport_width=1920)
+            BrowserConfig.reset_defaults("headless")
+            cfg = BrowserConfig()
+            assert cfg.headless is True  # reset to hardcoded default
+            assert cfg.viewport_width == 1920  # still overridden
+        finally:
+            BrowserConfig.reset_defaults()
+
+    def test_set_defaults_invalid_param_raises(self):
+        """set_defaults with invalid parameter name should raise ValueError."""
+        try:
+            with pytest.raises(ValueError):
+                BrowserConfig.set_defaults(nonexistent_param=42)
+        finally:
+            BrowserConfig.reset_defaults()
+
+
+class TestBrowserConfigDumpLoad:
+    """Verify dump() and load() serialization includes type info."""
+
+    def test_dump_includes_type(self):
+        """dump() should produce a dict with 'type' key."""
+        cfg = BrowserConfig(headless=False)
+        dumped = cfg.dump()
+        assert isinstance(dumped, dict)
+        assert dumped.get("type") == "BrowserConfig"
+        assert "params" in dumped
+
+    def test_dump_load_roundtrip(self):
+        """dump() -> load() should reproduce equivalent config."""
+        original = BrowserConfig(
+            headless=False,
+            viewport_width=1920,
+            text_mode=True,
+        )
+        dumped = original.dump()
+        restored = BrowserConfig.load(dumped)
+
+        assert isinstance(restored, BrowserConfig)
+        assert restored.headless is False
+        assert restored.viewport_width == 1920
+        assert restored.text_mode is True
+
+
+# ===================================================================
+# CrawlerRunConfig
+# ===================================================================
+
+class TestCrawlerRunConfigDefaults:
+    """Verify CrawlerRunConfig default values."""
+
+    def test_cache_mode_default(self):
+        """Default cache_mode should be CacheMode.BYPASS."""
+        cfg = CrawlerRunConfig()
+        assert cfg.cache_mode == CacheMode.BYPASS
+
+    def test_word_count_threshold_default(self):
+        """Default word_count_threshold should match MIN_WORD_THRESHOLD (1)."""
+        from crawl4ai.config import MIN_WORD_THRESHOLD
+        cfg = CrawlerRunConfig()
+        assert cfg.word_count_threshold == MIN_WORD_THRESHOLD
+
+    def test_wait_until_default(self):
+        """Default wait_until should be 'domcontentloaded'."""
+        cfg = CrawlerRunConfig()
+        assert cfg.wait_until == "domcontentloaded"
+
+    def test_page_timeout_default(self):
+        """Default page_timeout should be 60000 ms."""
+        cfg = CrawlerRunConfig()
+        assert cfg.page_timeout == 60000
+
+    def test_delay_before_return_html_default(self):
+        """Default delay_before_return_html should be 0.1."""
+        cfg = CrawlerRunConfig()
+        assert cfg.delay_before_return_html == 0.1
+
+    def test_magic_default_false(self):
+        """Magic mode should be off by default."""
+        cfg = CrawlerRunConfig()
+        assert cfg.magic is False
+
+    def test_screenshot_default_false(self):
+        """Screenshot should be off by default."""
+        cfg = CrawlerRunConfig()
+        assert cfg.screenshot is False
+
+    def test_verbose_default_true(self):
+        """Verbose should be on by default."""
+        cfg = CrawlerRunConfig()
+        assert cfg.verbose is True
+
+
+class TestCrawlerRunConfigRoundtrip:
+    """Verify to_dict -> from_kwargs roundtrip."""
+
+    def test_basic_roundtrip(self):
+        """Scalar fields should survive roundtrip."""
+        original = CrawlerRunConfig(
+            word_count_threshold=500,
+            wait_until="load",
+            page_timeout=30000,
+            magic=True,
+        )
+        d = original.to_dict()
+        restored = CrawlerRunConfig.from_kwargs(d)
+
+        assert restored.word_count_threshold == 500
+        assert restored.wait_until == "load"
+        assert restored.page_timeout == 30000
+        assert restored.magic is True
+
+    def test_roundtrip_preserves_js_code(self):
+        """js_code should survive roundtrip."""
+        original = CrawlerRunConfig(js_code=["document.title", "console.log('hi')"])
+        d = original.to_dict()
+        restored = CrawlerRunConfig.from_kwargs(d)
+        assert restored.js_code == ["document.title", "console.log('hi')"]
+
+    def test_roundtrip_preserves_excluded_tags(self):
+        """excluded_tags should survive roundtrip."""
+        original = CrawlerRunConfig(excluded_tags=["nav", "footer", "aside"])
+        d = original.to_dict()
+        restored = CrawlerRunConfig.from_kwargs(d)
+        assert "nav" in restored.excluded_tags
+        assert "footer" in restored.excluded_tags
+
+
+class TestCrawlerRunConfigClone:
+    """Verify clone() with overrides."""
+
+    def test_clone_with_override(self):
+        """Clone should apply overrides while keeping other fields."""
+        original = CrawlerRunConfig(magic=False, verbose=True)
+        cloned = original.clone(magic=True)
+
+        assert cloned.magic is True
+        assert cloned.verbose is True
+        # Original unchanged
+        assert original.magic is False
+
+    def test_clone_cache_mode_override(self):
+        """Clone should be able to change cache_mode."""
+        original = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+        cloned = original.clone(cache_mode=CacheMode.ENABLED)
+        assert cloned.cache_mode == CacheMode.ENABLED
+        assert original.cache_mode == CacheMode.BYPASS
+
+
+class TestCrawlerRunConfigClassDefaults:
+    """Verify set_defaults / reset_defaults for CrawlerRunConfig."""
+
+    def test_set_defaults_affects_new_instances(self):
+        """set_defaults(verbose=False) should make new instances verbose=False."""
+        try:
+            CrawlerRunConfig.set_defaults(verbose=False)
+            cfg = CrawlerRunConfig()
+            assert cfg.verbose is False
+        finally:
+            CrawlerRunConfig.reset_defaults()
+
+    def test_reset_defaults_restores_original(self):
+        """reset_defaults should restore hardcoded defaults."""
+        try:
+            CrawlerRunConfig.set_defaults(page_timeout=5000)
+            CrawlerRunConfig.reset_defaults()
+            cfg = CrawlerRunConfig()
+            assert cfg.page_timeout == 60000
+        finally:
+            CrawlerRunConfig.reset_defaults()
+
+    def test_set_defaults_invalid_param_raises(self):
+        """set_defaults with invalid parameter name should raise ValueError."""
+        try:
+            with pytest.raises(ValueError):
+                CrawlerRunConfig.set_defaults(totally_bogus=42)
+        finally:
+            CrawlerRunConfig.reset_defaults()
+
+
+class TestCrawlerRunConfigSerialization:
+    """Verify extraction_strategy and deep_crawl_strategy serialize correctly."""
+
+    def test_dump_load_basic(self):
+        """dump -> load roundtrip for basic CrawlerRunConfig."""
+        original = CrawlerRunConfig(
+            word_count_threshold=300,
+            magic=True,
+            wait_until="load",
+        )
+        dumped = original.dump()
+        assert dumped["type"] == "CrawlerRunConfig"
+        restored = CrawlerRunConfig.load(dumped)
+        assert isinstance(restored, CrawlerRunConfig)
+        assert restored.magic is True
+
+    def test_dump_with_extraction_strategy(self):
+        """CrawlerRunConfig with extraction_strategy should serialize."""
+        try:
+            from crawl4ai import JsonCssExtractionStrategy
+            schema = {
+                "name": "test",
+                "baseSelector": "div.item",
+                "fields": [{"name": "title", "selector": "h2", "type": "text"}],
+            }
+            strategy = JsonCssExtractionStrategy(schema)
+            cfg = CrawlerRunConfig(extraction_strategy=strategy)
+            dumped = cfg.dump()
+            assert dumped["type"] == "CrawlerRunConfig"
+            # extraction_strategy should be serialized with type info
+            es_data = dumped["params"].get("extraction_strategy", {})
+            assert es_data.get("type") == "JsonCssExtractionStrategy"
+        except ImportError:
+            pytest.skip("JsonCssExtractionStrategy not available")
+
+    def test_dump_with_deep_crawl_strategy(self):
+        """CrawlerRunConfig with deep_crawl_strategy should serialize."""
+        try:
+            from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+            strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+            cfg = CrawlerRunConfig(deep_crawl_strategy=strategy)
+            dumped = cfg.dump()
+            ds_data = dumped["params"].get("deep_crawl_strategy", {})
+            assert ds_data.get("type") == "BFSDeepCrawlStrategy"
+        except ImportError:
+            pytest.skip("BFSDeepCrawlStrategy not available")
+
+
+# ===================================================================
+# ProxyConfig
+# ===================================================================
+
+class TestProxyConfigFromString:
+    """Verify ProxyConfig.from_string() parsing."""
+
+    def test_simple_http_url(self):
+        """from_string('http://proxy:8080') should parse server correctly."""
+        pc = ProxyConfig.from_string("http://proxy:8080")
+        assert pc.server == "http://proxy:8080"
+        assert pc.username is None
+        assert pc.password is None
+
+    def test_http_url_with_credentials(self):
+        """from_string('http://user:pass@proxy:8080') should parse credentials."""
+        pc = ProxyConfig.from_string("http://user:pass@proxy:8080")
+        assert pc.server == "http://proxy:8080"
+        assert pc.username == "user"
+        assert pc.password == "pass"
+
+    def test_ip_port_user_pass_format(self):
+        """from_string('1.2.3.4:8080:user:pass') should parse ip:port:user:pass."""
+        pc = ProxyConfig.from_string("1.2.3.4:8080:user:pass")
+        assert pc.server == "http://1.2.3.4:8080"
+        assert pc.username == "user"
+        assert pc.password == "pass"
+
+    def test_ip_port_format(self):
+        """from_string('1.2.3.4:8080') should parse ip:port without credentials."""
+        pc = ProxyConfig.from_string("1.2.3.4:8080")
+        assert pc.server == "http://1.2.3.4:8080"
+        assert pc.username is None
+        assert pc.password is None
+
+    def test_socks5_url(self):
+        """from_string('socks5://proxy:1080') should preserve socks5 scheme."""
+        pc = ProxyConfig.from_string("socks5://proxy:1080")
+        assert pc.server == "socks5://proxy:1080"
+
+    def test_invalid_format_raises(self):
+        """from_string with invalid format should raise ValueError."""
+        with pytest.raises(ValueError):
+            ProxyConfig.from_string("invalid")
+
+    def test_password_with_colon(self):
+        """Password containing a colon should be preserved via split(':', 1)."""
+        # Format: http://user:complex:pass@proxy:8080
+        # The @ split gives auth="http://user:complex:pass", server="proxy:8080"
+        # Then protocol split gives credentials="user:complex:pass"
+        # Then credentials.split(":", 1) gives user="user", password="complex:pass"
+        pc = ProxyConfig.from_string("http://user:complex:pass@proxy:8080")
+        assert pc.username == "user"
+        assert pc.password == "complex:pass"
+        assert pc.server == "http://proxy:8080"
+
+
+class TestProxyConfigRoundtrip:
+    """Verify to_dict -> from_dict roundtrip."""
+
+    def test_basic_roundtrip(self):
+        """to_dict -> from_dict should preserve all fields."""
+        original = ProxyConfig(
+            server="http://proxy:8080",
+            username="user",
+            password="secret",
+        )
+        d = original.to_dict()
+        restored = ProxyConfig.from_dict(d)
+        assert restored.server == original.server
+        assert restored.username == original.username
+        assert restored.password == original.password
+
+    def test_roundtrip_without_credentials(self):
+        """Roundtrip should work without username/password."""
+        original = ProxyConfig(server="http://proxy:3128")
+        d = original.to_dict()
+        restored = ProxyConfig.from_dict(d)
+        assert restored.server == "http://proxy:3128"
+        assert restored.username is None
+        assert restored.password is None
+
+
+class TestProxyConfigClone:
+    """Verify clone() with override."""
+
+    def test_clone_with_server_override(self):
+        """Clone should apply server override."""
+        original = ProxyConfig(server="http://proxy1:8080", username="user1")
+        cloned = original.clone(server="http://proxy2:9090")
+        assert cloned.server == "http://proxy2:9090"
+        assert cloned.username == "user1"
+        # Original unchanged
+        assert original.server == "http://proxy1:8080"
+
+    def test_clone_with_credentials_override(self):
+        """Clone should be able to override credentials."""
+        original = ProxyConfig(server="http://proxy:8080", username="old", password="old")
+        cloned = original.clone(username="new", password="new")
+        assert cloned.username == "new"
+        assert cloned.password == "new"
+        assert original.username == "old"
+
+
+class TestProxyConfigSentinel:
+    """Verify ProxyConfig.DIRECT sentinel."""
+
+    def test_direct_sentinel_exists(self):
+        """ProxyConfig.DIRECT should exist and be 'direct'."""
+        assert ProxyConfig.DIRECT == "direct"
+
+    def test_direct_is_string(self):
+        """DIRECT sentinel should be a string."""
+        assert isinstance(ProxyConfig.DIRECT, str)
+
+
+# ===================================================================
+# GeolocationConfig
+# ===================================================================
+
+class TestGeolocationConfig:
+    """Verify GeolocationConfig construction and roundtrip."""
+
+    def test_constructor(self):
+        """Constructor should set lat/lon/accuracy."""
+        geo = GeolocationConfig(latitude=37.7749, longitude=-122.4194, accuracy=10.0)
+        assert geo.latitude == 37.7749
+        assert geo.longitude == -122.4194
+        assert geo.accuracy == 10.0
+
+    def test_default_accuracy(self):
+        """Default accuracy should be 0.0."""
+        geo = GeolocationConfig(latitude=0.0, longitude=0.0)
+        assert geo.accuracy == 0.0
+
+    def test_to_dict_from_dict_roundtrip(self):
+        """to_dict -> from_dict should preserve all fields."""
+        original = GeolocationConfig(latitude=48.8566, longitude=2.3522, accuracy=50.0)
+        d = original.to_dict()
+        restored = GeolocationConfig.from_dict(d)
+        assert restored.latitude == original.latitude
+        assert restored.longitude == original.longitude
+        assert restored.accuracy == original.accuracy
+
+    def test_clone_with_overrides(self):
+        """Clone should apply overrides while preserving other fields."""
+        original = GeolocationConfig(latitude=40.7128, longitude=-74.0060, accuracy=5.0)
+        cloned = original.clone(accuracy=100.0)
+        assert cloned.latitude == 40.7128
+        assert cloned.longitude == -74.0060
+        assert cloned.accuracy == 100.0
+        # Original unchanged
+        assert original.accuracy == 5.0
+
+    def test_clone_independence(self):
+        """Clone should be a fully independent object."""
+        original = GeolocationConfig(latitude=0.0, longitude=0.0)
+        cloned = original.clone(latitude=1.0)
+        assert original.latitude == 0.0
+        assert cloned.latitude == 1.0
+
+    def test_negative_coordinates(self):
+        """Negative lat/lon (southern/western hemisphere) should work."""
+        geo = GeolocationConfig(latitude=-33.8688, longitude=151.2093)
+        assert geo.latitude == -33.8688
+        assert geo.longitude == 151.2093
+
+
+# ===================================================================
+# Deep merge tests
+# ===================================================================
+
+class TestDeepMerge:
+    """Verify _deep_merge helper for server config merging."""
+
+    def test_empty_override_returns_base(self):
+        """Empty override should return base unchanged."""
+        base = {"a": 1, "b": 2}
+        result = _deep_merge(base, {})
+        assert result == {"a": 1, "b": 2}
+
+    def test_flat_key_override(self):
+        """Flat key in override should replace base value."""
+        base = {"a": 1, "b": 2}
+        result = _deep_merge(base, {"b": 99})
+        assert result == {"a": 1, "b": 99}
+
+    def test_nested_dict_merge_preserves_siblings(self):
+        """Nested dict merge should preserve sibling keys."""
+        base = {"server": {"host": "localhost", "port": 8080}}
+        override = {"server": {"port": 9090}}
+        result = _deep_merge(base, override)
+        assert result["server"]["host"] == "localhost"
+        assert result["server"]["port"] == 9090
+
+    def test_override_with_non_dict_replaces_dict(self):
+        """Non-dict override should replace entire dict value."""
+        base = {"server": {"host": "localhost", "port": 8080}}
+        override = {"server": "http://remote:9090"}
+        result = _deep_merge(base, override)
+        assert result["server"] == "http://remote:9090"
+
+    def test_deep_nesting_three_levels(self):
+        """3+ levels of nesting should merge correctly."""
+        base = {"a": {"b": {"c": 1, "d": 2}, "e": 3}}
+        override = {"a": {"b": {"c": 99}}}
+        result = _deep_merge(base, override)
+        assert result["a"]["b"]["c"] == 99
+        assert result["a"]["b"]["d"] == 2
+        assert result["a"]["e"] == 3
+
+    def test_new_key_in_override(self):
+        """Override can add entirely new keys."""
+        base = {"a": 1}
+        result = _deep_merge(base, {"b": 2})
+        assert result == {"a": 1, "b": 2}
+
+    def test_base_not_mutated(self):
+        """Original base dict should not be mutated."""
+        base = {"a": {"b": 1}}
+        override = {"a": {"b": 2}}
+        _deep_merge(base, override)
+        assert base["a"]["b"] == 1
+
+    def test_empty_base(self):
+        """Empty base should return override contents."""
+        result = _deep_merge({}, {"a": 1, "b": {"c": 2}})
+        assert result == {"a": 1, "b": {"c": 2}}
+
+
+# ===================================================================
+# Serialization: to_serializable_dict / from_serializable_dict
+# ===================================================================
+
+class TestSerializableDict:
+    """Verify to_serializable_dict / from_serializable_dict roundtrips."""
+
+    def test_browser_config_roundtrip(self):
+        """BrowserConfig should survive serialization roundtrip."""
+        original = BrowserConfig(
+            headless=False,
+            viewport_width=1920,
+            browser_type="firefox",
+        )
+        serialized = to_serializable_dict(original)
+        assert serialized["type"] == "BrowserConfig"
+        restored = from_serializable_dict(serialized)
+        assert isinstance(restored, BrowserConfig)
+        assert restored.headless is False
+        assert restored.viewport_width == 1920
+
+    def test_crawler_run_config_roundtrip(self):
+        """CrawlerRunConfig should survive serialization roundtrip."""
+        original = CrawlerRunConfig(
+            word_count_threshold=500,
+            magic=True,
+            wait_until="load",
+        )
+        serialized = to_serializable_dict(original)
+        assert serialized["type"] == "CrawlerRunConfig"
+        restored = from_serializable_dict(serialized)
+        assert isinstance(restored, CrawlerRunConfig)
+        assert restored.magic is True
+
+    def test_crawler_run_config_with_extraction_strategy(self):
+        """CrawlerRunConfig with extraction strategy should roundtrip."""
+        try:
+            from crawl4ai import JsonCssExtractionStrategy
+            schema = {
+                "name": "products",
+                "baseSelector": "div.product",
+                "fields": [
+                    {"name": "title", "selector": "h2", "type": "text"},
+                    {"name": "price", "selector": ".price", "type": "text"},
+                ],
+            }
+            strategy = JsonCssExtractionStrategy(schema)
+            original = CrawlerRunConfig(extraction_strategy=strategy)
+            serialized = to_serializable_dict(original)
+            restored = from_serializable_dict(serialized)
+            assert isinstance(restored, CrawlerRunConfig)
+            assert isinstance(restored.extraction_strategy, JsonCssExtractionStrategy)
+        except ImportError:
+            pytest.skip("JsonCssExtractionStrategy not available")
+
+    def test_none_value(self):
+        """None should serialize to None."""
+        assert to_serializable_dict(None) is None
+
+    def test_basic_types_passthrough(self):
+        """Strings, ints, floats, bools should pass through unchanged."""
+        assert to_serializable_dict("hello") == "hello"
+        assert to_serializable_dict(42) == 42
+        assert to_serializable_dict(3.14) == 3.14
+        assert to_serializable_dict(True) is True
+
+    def test_enum_serialization(self):
+        """CacheMode enum should serialize with type info."""
+        serialized = to_serializable_dict(CacheMode.ENABLED)
+        assert serialized["type"] == "CacheMode"
+        assert serialized["params"] == "enabled"
+        restored = from_serializable_dict(serialized)
+        assert restored == CacheMode.ENABLED
+
+    def test_list_serialization(self):
+        """Lists should serialize element-by-element."""
+        result = to_serializable_dict([1, "two", 3.0])
+        assert result == [1, "two", 3.0]
+
+    def test_dict_serialization(self):
+        """Plain dicts should be wrapped with type='dict'."""
+        result = to_serializable_dict({"key": "value"})
+        assert result["type"] == "dict"
+        assert result["value"]["key"] == "value"
+
+    def test_disallowed_type_returns_none(self):
+        """Deserializing a non-allowlisted type should return None (not instantiate it)."""
+        bad_data = {"type": "os.system", "params": {"command": "rm -rf /"}}
+        result = from_serializable_dict(bad_data)
+        assert result is None
+
+    def test_geolocation_config_roundtrip(self):
+        """GeolocationConfig should survive serialization roundtrip."""
+        original = GeolocationConfig(latitude=37.7749, longitude=-122.4194, accuracy=10.0)
+        serialized = to_serializable_dict(original)
+        assert serialized["type"] == "GeolocationConfig"
+        restored = from_serializable_dict(serialized)
+        assert isinstance(restored, GeolocationConfig)
+        assert restored.latitude == 37.7749
+
+    def test_proxy_config_roundtrip(self):
+        """ProxyConfig should survive serialization roundtrip."""
+        original = ProxyConfig(server="http://proxy:8080", username="user", password="pass")
+        serialized = to_serializable_dict(original)
+        assert serialized["type"] == "ProxyConfig"
+        restored = from_serializable_dict(serialized)
+        assert isinstance(restored, ProxyConfig)
+        assert restored.server == "http://proxy:8080"
+        assert restored.username == "user"
diff --git a/tests/regression/test_reg_content.py b/tests/regression/test_reg_content.py
new file mode 100644
index 0000000..4390c41
--- /dev/null
+++ b/tests/regression/test_reg_content.py
@@ -0,0 +1,512 @@
+"""
+Regression tests for Crawl4AI content processing pipeline.
+
+Covers markdown generation, content filtering (BM25, Pruning),
+link/image/table extraction, metadata extraction, tag exclusion,
+CSS selector targeting, and real-URL content quality.
+
+Run:
+    pytest tests/regression/test_reg_content.py -v
+    pytest tests/regression/test_reg_content.py -v -m "not network"
+"""
+
+import pytest
+import json
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import BM25ContentFilter, PruningContentFilter
+
+
+# ---------------------------------------------------------------------------
+# Markdown generation
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_markdown_raw(local_server):
+    """Crawl the home page and verify raw markdown is a non-empty string
+    containing the expected heading text and heading markers."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=CrawlerRunConfig())
+        assert result.success, f"Crawl failed: {result.error_message}"
+        md = result.markdown
+        assert md is not None
+        assert isinstance(md, str)
+        assert len(md) > 0
+        assert "Welcome to the Crawl4AI Test Site" in md
+        # Should have at least one markdown heading marker
+        assert "#" in md
+
+
+@pytest.mark.asyncio
+async def test_markdown_has_headings(local_server):
+    """Verify markdown contains the expected h1 and h2 headings."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=CrawlerRunConfig())
+        assert result.success
+        md = result.markdown
+        assert "# Welcome" in md or "# Welcome to the Crawl4AI Test Site" in md
+        # h2 heading for Features Overview
+        assert "## Features" in md or "## Features Overview" in md
+
+
+@pytest.mark.asyncio
+async def test_markdown_has_code_block(local_server):
+    """Verify markdown preserves the code block with triple backticks."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=CrawlerRunConfig())
+        assert result.success
+        md = result.markdown
+        assert "```" in md
+        assert "AsyncWebCrawler" in md
+
+
+@pytest.mark.asyncio
+async def test_markdown_has_list(local_server):
+    """Verify markdown contains list items from the home page features list."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=CrawlerRunConfig())
+        assert result.success
+        md = result.markdown
+        # Markdown list items should contain at least some of these
+        assert "Content extraction" in md or "content extraction" in md
+        assert "Link discovery" in md or "link discovery" in md
+
+
+@pytest.mark.asyncio
+async def test_markdown_citations(local_server):
+    """Access markdown_with_citations and verify it contains numbered citation references."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=CrawlerRunConfig())
+        assert result.success
+        citations_md = result.markdown.markdown_with_citations
+        assert isinstance(citations_md, str)
+        assert len(citations_md) > 0
+        # Should have at least one citation reference like [1] or similar
+        has_citation = any(f"[{i}]" in citations_md for i in range(1, 20))
+        # Some implementations use a different format
+        assert has_citation or "⟨" in citations_md or "[" in citations_md
+
+
+@pytest.mark.asyncio
+async def test_markdown_references(local_server):
+    """Access references_markdown and verify it contains URLs."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=CrawlerRunConfig())
+        assert result.success
+        refs = result.markdown.references_markdown
+        assert isinstance(refs, str)
+        # References should mention URLs or link targets
+        assert "http" in refs or "/" in refs
+
+
+@pytest.mark.asyncio
+async def test_markdown_string_compat(local_server):
+    """Verify StringCompatibleMarkdown behaves like a string:
+    str() works, equality with raw_markdown, and 'in' operator."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=CrawlerRunConfig())
+        assert result.success
+        md = result.markdown
+        raw = md.raw_markdown
+        # str(result.markdown) should equal raw_markdown
+        assert str(md) == raw
+        # 'in' operator should work on the string content
+        assert "Welcome" in md
+
+
+# ---------------------------------------------------------------------------
+# Content filtering - BM25
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_bm25_fit_markdown(local_server):
+    """Crawl with BM25ContentFilter and verify fit_markdown is shorter
+    than the full raw_markdown (content was filtered)."""
+    gen = DefaultMarkdownGenerator(
+        content_filter=BM25ContentFilter(user_query="features")
+    )
+    config = CrawlerRunConfig(markdown_generator=gen)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=config)
+        assert result.success
+        fit = result.markdown.fit_markdown
+        raw = result.markdown.raw_markdown
+        assert fit is not None
+        assert len(fit) > 0
+        assert len(fit) < len(raw), (
+            "fit_markdown should be shorter than raw_markdown after BM25 filtering"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Content filtering - Pruning
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_pruning_fit_markdown(local_server):
+    """Crawl with PruningContentFilter and verify fit_markdown exists
+    and is shorter than the full raw_markdown."""
+    gen = DefaultMarkdownGenerator(content_filter=PruningContentFilter())
+    config = CrawlerRunConfig(markdown_generator=gen)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=config)
+        assert result.success
+        fit = result.markdown.fit_markdown
+        raw = result.markdown.raw_markdown
+        assert fit is not None
+        assert len(fit) > 0
+        assert len(fit) <= len(raw), (
+            "fit_markdown should not be longer than raw_markdown"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Link extraction
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_links_internal(local_server):
+    """Crawl /links-page and verify internal links are extracted with href keys."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/links-page", config=CrawlerRunConfig())
+        assert result.success
+        internal = result.links.get("internal", [])
+        assert isinstance(internal, list)
+        assert len(internal) > 0, "Expected internal links to be found"
+        # Each link dict should have an href
+        for link in internal:
+            assert "href" in link, f"Link missing 'href' key: {link}"
+
+
+@pytest.mark.asyncio
+async def test_links_external(local_server):
+    """Verify external links include the expected domains."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/links-page", config=CrawlerRunConfig())
+        assert result.success
+        external = result.links.get("external", [])
+        assert len(external) > 0, "Expected external links to be found"
+        hrefs = [link["href"] for link in external]
+        all_hrefs = " ".join(hrefs)
+        assert "example.com" in all_hrefs
+        assert "github.com" in all_hrefs
+        assert "python.org" in all_hrefs
+
+
+@pytest.mark.asyncio
+async def test_links_exclude_external(local_server):
+    """Crawl with exclude_external_links=True and verify no external links remain."""
+    config = CrawlerRunConfig(exclude_external_links=True)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/links-page", config=config)
+        assert result.success
+        external = result.links.get("external", [])
+        assert len(external) == 0, f"Expected no external links, got {len(external)}"
+
+
+@pytest.mark.asyncio
+async def test_links_exclude_social(local_server):
+    """Crawl with exclude_social_media_links=True and verify no social media
+    links appear in the external links list."""
+    config = CrawlerRunConfig(exclude_social_media_links=True)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/links-page", config=config)
+        assert result.success
+        external = result.links.get("external", [])
+        social_domains = ["twitter.com", "facebook.com", "linkedin.com"]
+        for link in external:
+            href = link.get("href", "")
+            for domain in social_domains:
+                assert domain not in href, (
+                    f"Social media link should be excluded: {href}"
+                )
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_links_real_url():
+    """Crawl a real URL (quotes.toscrape.com) and verify internal links are found
+    (pagination links exist on the main page)."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url="https://quotes.toscrape.com",
+            config=CrawlerRunConfig(),
+        )
+        assert result.success
+        internal = result.links.get("internal", [])
+        assert len(internal) > 0, "Expected internal links on quotes.toscrape.com"
+
+
+# ---------------------------------------------------------------------------
+# Image extraction
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_images_extracted(local_server):
+    """Crawl /images-page and verify images are extracted."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/images-page", config=CrawlerRunConfig())
+        assert result.success
+        images = result.media.get("images", [])
+        assert isinstance(images, list)
+        assert len(images) > 0, "Expected images to be extracted"
+
+
+@pytest.mark.asyncio
+async def test_images_have_fields(local_server):
+    """Verify each extracted image dict has src, alt, and score keys."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/images-page", config=CrawlerRunConfig())
+        assert result.success
+        images = result.media.get("images", [])
+        assert len(images) > 0
+        for img in images:
+            assert "src" in img, f"Image missing 'src': {img}"
+            assert "alt" in img, f"Image missing 'alt': {img}"
+            assert "score" in img, f"Image missing 'score': {img}"
+
+
+@pytest.mark.asyncio
+async def test_images_scoring(local_server):
+    """High-quality images (large, with alt text) should score higher
+    than small icons without alt text."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/images-page", config=CrawlerRunConfig())
+        assert result.success
+        images = result.media.get("images", [])
+        assert len(images) >= 2
+
+        # Find the hero/landscape image and the small icon
+        hero = None
+        icon = None
+        for img in images:
+            src = img.get("src", "")
+            if "landscape" in src or "hero" in src:
+                hero = img
+            elif "icon" in src and img.get("alt", "") == "":
+                icon = img
+
+        if hero and icon:
+            assert hero["score"] > icon["score"], (
+                f"Hero score ({hero['score']}) should exceed icon score ({icon['score']})"
+            )
+
+
+@pytest.mark.asyncio
+async def test_images_exclude_all(local_server):
+    """Crawl with exclude_all_images=True and verify no images are returned."""
+    config = CrawlerRunConfig(exclude_all_images=True)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/images-page", config=config)
+        assert result.success
+        images = result.media.get("images", [])
+        assert len(images) == 0, f"Expected no images with exclude_all_images, got {len(images)}"
+
+
+# ---------------------------------------------------------------------------
+# Table extraction
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_tables_extracted(local_server):
+    """Crawl /tables and verify tables appear in the result (either in
+    result.media, result.tables, or markdown pipe formatting)."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/tables", config=CrawlerRunConfig())
+        assert result.success
+        # Tables may appear in result.tables, result.media, or markdown
+        has_tables = (
+            len(getattr(result, "tables", []) or []) > 0
+            or "tables" in result.media
+            or "|" in str(result.markdown)
+        )
+        assert has_tables, "Expected table data to be found in the result"
+
+
+@pytest.mark.asyncio
+async def test_tables_in_markdown(local_server):
+    """Verify the markdown output contains table formatting with pipes and dashes."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/tables", config=CrawlerRunConfig())
+        assert result.success
+        md = str(result.markdown)
+        assert "|" in md, "Expected pipe character in markdown tables"
+        assert "---" in md or "- -" in md, "Expected separator row in markdown tables"
+
+
+# ---------------------------------------------------------------------------
+# Metadata extraction
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_metadata_title(local_server):
+    """Crawl /structured-data and verify the page title is in metadata."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url=f"{local_server}/structured-data", config=CrawlerRunConfig()
+        )
+        assert result.success
+        assert result.metadata is not None
+        # Title should be "Article with Structured Data"
+        title = result.metadata.get("title", "")
+        assert "Article with Structured Data" in title or "Structured Data" in title
+
+
+@pytest.mark.asyncio
+async def test_metadata_og_tags(local_server):
+    """Verify og:title, og:description, og:image are present in metadata."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url=f"{local_server}/structured-data", config=CrawlerRunConfig()
+        )
+        assert result.success
+        meta = result.metadata
+        assert meta is not None
+
+        # Check for og tags -- they may be stored with different key formats
+        og_title = meta.get("og:title", meta.get("og_title", ""))
+        og_desc = meta.get("og:description", meta.get("og_description", ""))
+        og_image = meta.get("og:image", meta.get("og_image", ""))
+
+        assert og_title, f"Missing og:title in metadata: {meta}"
+        assert og_desc, f"Missing og:description in metadata: {meta}"
+        assert og_image, f"Missing og:image in metadata: {meta}"
+
+
+@pytest.mark.asyncio
+async def test_metadata_description(local_server):
+    """Verify meta description is present in metadata."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url=f"{local_server}/structured-data", config=CrawlerRunConfig()
+        )
+        assert result.success
+        meta = result.metadata
+        assert meta is not None
+        desc = meta.get("description", "")
+        assert desc, f"Missing description in metadata: {meta}"
+        assert "web crawling" in desc.lower()
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_metadata_real():
+    """Crawl https://example.com and verify title metadata exists."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url="https://example.com", config=CrawlerRunConfig()
+        )
+        assert result.success
+        assert result.metadata is not None
+        title = result.metadata.get("title", "")
+        assert title, "Expected title metadata from example.com"
+
+
+# ---------------------------------------------------------------------------
+# Excluded tags
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_excluded_tags_nav(local_server):
+    """Crawl / with excluded_tags=["nav"] and verify navigation links are
+    removed from cleaned_html."""
+    config = CrawlerRunConfig(excluded_tags=["nav"])
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=config)
+        assert result.success
+        cleaned = result.cleaned_html or ""
+        # The nav element contained links to Products, Links, Tables
+        # After exclusion these should be absent from cleaned_html
+        assert "
+        assert "Footer content" not in md
+
+
+@pytest.mark.asyncio
+async def test_css_selector_product(local_server):
+    """Crawl /products with css_selector targeting only product #1 and verify
+    only the first product is extracted."""
+    config = CrawlerRunConfig(css_selector=".product[data-id='1']")
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/products", config=config)
+        assert result.success
+        md = str(result.markdown)
+        assert "Wireless Mouse" in md
+        # Other products should not appear
+        assert "Mechanical Keyboard" not in md
+        assert "USB-C Hub" not in md
+
+
+# ---------------------------------------------------------------------------
+# Real URL content tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_real_url_markdown_quality():
+    """Crawl https://example.com and verify markdown has reasonable content
+    with more than 50 chars and contains 'Example Domain'."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url="https://example.com", config=CrawlerRunConfig()
+        )
+        assert result.success
+        md = str(result.markdown)
+        assert len(md) > 50, f"Markdown too short ({len(md)} chars)"
+        assert "Example Domain" in md
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_real_url_links():
+    """Crawl https://books.toscrape.com and verify internal links (product links)
+    and images (book covers) are found."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url="https://books.toscrape.com", config=CrawlerRunConfig()
+        )
+        assert result.success
+        internal = result.links.get("internal", [])
+        assert len(internal) > 0, "Expected product links on books.toscrape.com"
+        images = result.media.get("images", [])
+        assert len(images) > 0, "Expected book cover images on books.toscrape.com"
diff --git a/tests/regression/test_reg_core_crawl.py b/tests/regression/test_reg_core_crawl.py
new file mode 100644
index 0000000..6dc3209
--- /dev/null
+++ b/tests/regression/test_reg_core_crawl.py
@@ -0,0 +1,405 @@
+"""
+Crawl4AI Regression Tests - Core Crawling Functionality
+
+Tests core crawling features including basic crawls, raw HTML, multiple URLs,
+screenshots, JavaScript execution, caching, sessions, hooks, network capture,
+CSS selectors, excluded tags, timeouts, and status codes.
+
+All tests use real browser crawling with no mocking.
+"""
+
+import asyncio
+import base64
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.cache_context import CacheMode
+
+
+# ---------------------------------------------------------------------------
+# Basic crawl tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_basic_crawl(local_server):
+    """Crawl the local server home page and verify basic result fields."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/")
+        assert result.success, f"Crawl failed: {result.error_message}"
+        assert "
      " in result.html, "HTML should contain an 
       tag"
+        assert isinstance(result.markdown, str), "Markdown should be a string"
+        assert len(result.markdown) > 0, "Markdown should be non-empty"
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_basic_crawl_real_url():
+    """Crawl https://example.com and verify success with real content."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun("https://example.com")
+        assert result.success, f"Crawl failed: {result.error_message}"
+        assert len(result.html) > 100, "HTML should have substantial content"
+        assert len(result.markdown) > 10, "Markdown should have content"
+
+
+# ---------------------------------------------------------------------------
+# Raw HTML crawl tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_raw_html_crawl():
+    """Crawl raw HTML and verify markdown extraction."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun("raw:
      Test
      Hello world
      ")
+        assert result.success, f"Raw HTML crawl failed: {result.error_message}"
+        assert "Test" in result.markdown, "Markdown should contain 'Test'"
+        assert "Hello" in result.markdown, "Markdown should contain 'Hello'"
+
+
+@pytest.mark.asyncio
+async def test_raw_html_with_base_url():
+    """Raw HTML with relative links should resolve against base_url."""
+    raw_html = (
+        "raw:"
+        'Link 1'
+        'Link 2'
+        'Absolute'
+        ""
+    )
+    config = CrawlerRunConfig(base_url="http://example.com")
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(raw_html, config=config)
+        assert result.success, f"Raw HTML with base_url failed: {result.error_message}"
+        # Check that links were resolved (they should appear in the result's links or markdown)
+        md_lower = result.markdown.lower() if result.markdown else ""
+        html_lower = result.html.lower() if result.html else ""
+        combined = md_lower + html_lower
+        # At minimum, the link text should appear
+        assert "link 1" in combined, "Link text should be present"
+
+
+# ---------------------------------------------------------------------------
+# Multiple URL crawl tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_arun_many(local_server):
+    """Crawl 3 local server URLs with arun_many and verify all succeed."""
+    urls = [
+        local_server + "/",
+        local_server + "/products",
+        local_server + "/tables",
+    ]
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun_many(urls, config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS))
+        assert isinstance(results, list), "arun_many should return a list"
+        assert len(results) == 3, f"Expected 3 results, got {len(results)}"
+        for i, result in enumerate(results):
+            assert result.success, f"Result {i} failed: {result.error_message}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_arun_many_real():
+    """Crawl multiple real URLs together."""
+    urls = ["https://example.com", "https://quotes.toscrape.com"]
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun_many(urls, config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS))
+        assert len(results) == 2, f"Expected 2 results, got {len(results)}"
+        for result in results:
+            assert result.success, f"Real URL crawl failed: {result.error_message}"
+
+
+# ---------------------------------------------------------------------------
+# Screenshot tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_screenshot_capture(local_server):
+    """Crawl with screenshot=True and verify PNG format output."""
+    config = CrawlerRunConfig(screenshot=True)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/", config=config)
+        assert result.success, f"Screenshot crawl failed: {result.error_message}"
+        assert result.screenshot, "Screenshot should be a non-empty string"
+        assert isinstance(result.screenshot, str), "Screenshot should be a base64 string"
+        # Decode and verify PNG header
+        raw_bytes = base64.b64decode(result.screenshot)
+        assert raw_bytes[:4] == b"\x89PNG", "Screenshot should be in PNG format"
+
+
+@pytest.mark.asyncio
+async def test_screenshot_not_bmp(local_server):
+    """Verify screenshot is PNG format, NOT BMP (regression for #1758)."""
+    config = CrawlerRunConfig(screenshot=True)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/", config=config)
+        assert result.success
+        raw_bytes = base64.b64decode(result.screenshot)
+        # BMP files start with b'BM'
+        assert raw_bytes[:2] != b"BM", "Screenshot should NOT be BMP format"
+        assert raw_bytes[:4] == b"\x89PNG", "Screenshot should be PNG format"
+
+
+# ---------------------------------------------------------------------------
+# JavaScript execution tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_js_execution(local_server):
+    """Crawl /js-dynamic with wait_for to verify JS-generated content loads."""
+    config = CrawlerRunConfig(wait_for="css:.js-loaded")
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/js-dynamic", config=config)
+        assert result.success, f"JS dynamic crawl failed: {result.error_message}"
+        assert "Dynamic content successfully loaded" in result.markdown, (
+            "JS-generated content should appear in markdown"
+        )
+
+
+@pytest.mark.asyncio
+async def test_js_code_execution(local_server):
+    """Execute custom JS code during crawl and verify modification."""
+    config = CrawlerRunConfig(
+        js_code="document.title = 'Modified Title';",
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/", config=config)
+        assert result.success, f"JS code execution crawl failed: {result.error_message}"
+        # The JS ran after page load; verify it did not cause errors
+        # (title change may or may not be reflected in html depending on timing)
+
+
+@pytest.mark.asyncio
+async def test_js_code_before_wait(local_server):
+    """Use js_code_before_wait to inject content, then wait_for to verify it."""
+    js_inject = """
+    const div = document.createElement('div');
+    div.id = 'injected-marker';
+    div.className = 'injected';
+    div.textContent = 'Injected by js_code_before_wait';
+    document.body.appendChild(div);
+    """
+    config = CrawlerRunConfig(
+        js_code_before_wait=js_inject,
+        wait_for="css:#injected-marker",
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/", config=config)
+        assert result.success, f"js_code_before_wait crawl failed: {result.error_message}"
+        assert "Injected by js_code_before_wait" in result.markdown, (
+            "Injected content should appear in markdown"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Cache mode tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_cache_write_and_read(local_server):
+    """Crawl with ENABLED cache, then crawl again to verify cache hit."""
+    config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        # First crawl - writes to cache
+        result1 = await crawler.arun(local_server + "/", config=config)
+        assert result1.success, f"First crawl failed: {result1.error_message}"
+
+        # Second crawl - should read from cache
+        result2 = await crawler.arun(local_server + "/", config=config)
+        assert result2.success, f"Second crawl failed: {result2.error_message}"
+        if result2.cache_status:
+            assert "hit" in result2.cache_status.lower(), (
+                f"Second crawl should be a cache hit, got: {result2.cache_status}"
+            )
+
+
+@pytest.mark.asyncio
+async def test_cache_bypass(local_server):
+    """Crawl with BYPASS cache mode; result should still succeed."""
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/", config=config)
+        assert result.success, f"Bypass cache crawl failed: {result.error_message}"
+        assert len(result.html) > 0, "HTML should be non-empty even with bypass"
+
+
+@pytest.mark.asyncio
+async def test_cache_disabled(local_server):
+    """Crawl with DISABLED cache; second crawl should not be cached."""
+    config = CrawlerRunConfig(cache_mode=CacheMode.DISABLED)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result1 = await crawler.arun(local_server + "/", config=config)
+        assert result1.success
+        result2 = await crawler.arun(local_server + "/", config=config)
+        assert result2.success
+        # With DISABLED, there should be no cache hit
+        if result2.cache_status:
+            assert "hit" not in result2.cache_status.lower(), (
+                "DISABLED cache should not produce a cache hit"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Session reuse test
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_session_reuse(local_server):
+    """Crawl with a session_id, crawl again with same session_id; both succeed."""
+    config = CrawlerRunConfig(session_id="test-session", cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result1 = await crawler.arun(local_server + "/", config=config)
+        assert result1.success, f"First session crawl failed: {result1.error_message}"
+
+        result2 = await crawler.arun(local_server + "/", config=config)
+        assert result2.success, f"Second session crawl failed: {result2.error_message}"
+
+
+# ---------------------------------------------------------------------------
+# Hooks test
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_hooks_fire(local_server):
+    """Verify before_goto and after_goto hooks are called during crawl."""
+    calls = []
+
+    async def before_hook(page, context, url, **kwargs):
+        calls.append(("before_goto", url))
+        return page
+
+    async def after_hook(page, context, url, **kwargs):
+        calls.append(("after_goto", url))
+        return page
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        crawler.crawler_strategy.set_hook("before_goto", before_hook)
+        crawler.crawler_strategy.set_hook("after_goto", after_hook)
+
+        result = await crawler.arun(local_server + "/", config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS))
+        assert result.success, f"Hook crawl failed: {result.error_message}"
+        hook_types = [c[0] for c in calls]
+        assert "before_goto" in hook_types, "before_goto hook should have been called"
+        assert "after_goto" in hook_types, "after_goto hook should have been called"
+
+
+# ---------------------------------------------------------------------------
+# Network capture test
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_network_request_capture(local_server):
+    """Crawl with capture_network_requests=True and verify requests are captured."""
+    config = CrawlerRunConfig(capture_network_requests=True, cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/", config=config)
+        assert result.success, f"Network capture crawl failed: {result.error_message}"
+        assert result.network_requests is not None, "network_requests should not be None"
+        assert isinstance(result.network_requests, list), "network_requests should be a list"
+        assert len(result.network_requests) >= 1, "Should capture at least 1 network request"
+        # Each entry should have a url key
+        assert "url" in result.network_requests[0], (
+            "Network request entries should have a 'url' key"
+        )
+
+
+# ---------------------------------------------------------------------------
+# CSS selector test
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_css_selector(local_server):
+    """Crawl /products with css_selector to narrow content extraction."""
+    config = CrawlerRunConfig(css_selector=".product-list", cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/products", config=config)
+        assert result.success, f"CSS selector crawl failed: {result.error_message}"
+        # The product content should be present
+        assert "Wireless Mouse" in result.html, "Product content should be in HTML"
+        # The h1 "Products" is outside .product-list, should not be in the selected HTML
+        # css_selector filters the HTML sent to content extraction
+        assert "
      " not in result.html, (
+            "The h1 outside .product-list should not appear in result.html"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Excluded tags test
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_excluded_tags(local_server):
+    """Crawl with excluded_tags to remove nav and footer content."""
+    config = CrawlerRunConfig(excluded_tags=["nav", "footer"], cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/", config=config)
+        assert result.success, f"Excluded tags crawl failed: {result.error_message}"
+        cleaned = result.cleaned_html or ""
+        assert " str:
+    """Convert http://localhost:PORT to http://127.0.0.1:PORT.
+
+    Deep crawl strategies reject netlocs without a dot (e.g. 'localhost'),
+    so we use the IP form which contains dots and passes validation.
+    """
+    return local_server.replace("localhost", "127.0.0.1")
+
+
+# ---------------------------------------------------------------------------
+# BFS Deep Crawl
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_bfs_basic(local_server):
+    """BFS deep crawl of /deep/hub at depth 1 should return hub + sub pages."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=10)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        assert len(result_list) >= 1, "Should return at least the hub page"
+
+        # First result should be the hub
+        assert "/deep/hub" in result_list[0].url, "First result should be the hub page"
+
+        # Check sub pages are present
+        sub_urls = [r.url for r in result_list if "/deep/sub" in r.url]
+        assert len(sub_urls) >= 1, "Should discover at least one sub page"
+
+        # Verify metadata has depth key
+        for r in result_list:
+            assert r.metadata is not None, "Each result should have metadata"
+            assert "depth" in r.metadata, "Metadata should contain 'depth' key"
+
+        # Hub should be at depth 0
+        hub_result = result_list[0]
+        assert hub_result.metadata["depth"] == 0, "Hub should be at depth 0"
+
+        # Sub pages should be at depth 1
+        for r in result_list:
+            if "/deep/sub" in r.url:
+                assert r.metadata["depth"] == 1, f"Sub page {r.url} should be at depth 1"
+
+
+@pytest.mark.asyncio
+async def test_bfs_depth_enforcement(local_server):
+    """BFS with max_depth=1 must not include leaf pages at depth 2."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=20)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        leaf_urls = [r.url for r in result_list if "leaf" in r.url]
+        assert len(leaf_urls) == 0, (
+            f"No leaf pages should appear at max_depth=1, but found: {leaf_urls}"
+        )
+
+
+@pytest.mark.asyncio
+async def test_bfs_max_pages(local_server):
+    """BFS with max_pages=3 should return at most 3 results."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BFSDeepCrawlStrategy(max_depth=3, max_pages=3)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        assert len(result_list) <= 3, (
+            f"Expected at most 3 results, got {len(result_list)}"
+        )
+
+
+@pytest.mark.asyncio
+async def test_bfs_level_order(local_server):
+    """BFS should return results in level order: depth 0 before depth 1 before depth 2."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=20)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        depths = [r.metadata["depth"] for r in result_list]
+
+        # Verify ordering: once a higher depth appears, no lower depth should follow
+        max_depth_seen = -1
+        for i, d in enumerate(depths):
+            if d < max_depth_seen:
+                pytest.fail(
+                    f"BFS level order violated at index {i}: depth {d} appeared "
+                    f"after depth {max_depth_seen}. Full sequence: {depths}"
+                )
+            max_depth_seen = max(max_depth_seen, d)
+
+
+# ---------------------------------------------------------------------------
+# DFS Deep Crawl
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_dfs_basic(local_server):
+    """DFS deep crawl at depth 2 should find both sub pages and leaf pages."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = DFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        urls = [r.url for r in result_list]
+
+        sub_pages = [u for u in urls if "/deep/sub" in u and "leaf" not in u]
+        leaf_pages = [u for u in urls if "leaf" in u]
+
+        assert len(sub_pages) >= 1, "DFS should visit at least one sub page"
+        assert len(leaf_pages) >= 1, "DFS at depth 2 should visit at least one leaf page"
+
+
+@pytest.mark.asyncio
+async def test_dfs_depth_first_order(local_server):
+    """DFS should explore depth-first: some leaf page should appear before all sub pages are visited."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    # Give enough pages to see the DFS pattern
+    strategy = DFSDeepCrawlStrategy(max_depth=2, max_pages=15)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        urls = [r.url for r in result_list]
+
+        # Find indices of sub pages and leaf pages
+        sub_indices = [i for i, u in enumerate(urls) if "/deep/sub" in u and "leaf" not in u]
+        leaf_indices = [i for i, u in enumerate(urls) if "leaf" in u]
+
+        if sub_indices and leaf_indices:
+            # In DFS, at least one leaf should appear before the last sub page
+            earliest_leaf = min(leaf_indices)
+            latest_sub = max(sub_indices)
+            assert earliest_leaf < latest_sub, (
+                "DFS should explore a branch deeply before exhausting all sub pages. "
+                f"Earliest leaf at index {earliest_leaf}, latest sub at index {latest_sub}."
+            )
+
+
+@pytest.mark.asyncio
+async def test_dfs_max_depth(local_server):
+    """DFS with max_depth=1 should only visit hub and sub pages, no leaves."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = DFSDeepCrawlStrategy(max_depth=1, max_pages=20)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        leaf_urls = [r.url for r in result_list if "leaf" in r.url]
+        assert len(leaf_urls) == 0, (
+            f"DFS with max_depth=1 should not reach leaf pages, found: {leaf_urls}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# BestFirst Deep Crawl
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_bestfirst_basic(local_server):
+    """BestFirst deep crawl should return results from /deep/hub."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BestFirstCrawlingStrategy(max_depth=2, max_pages=10)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        assert len(result_list) >= 1, "BestFirst should return at least the start page"
+        assert result_list[0].success, "First result should be successful"
+
+
+# ---------------------------------------------------------------------------
+# Filters
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_url_pattern_filter_include(local_server):
+    """URLPatternFilter with sub1 pattern should only crawl the sub1 branch."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    url_filter = URLPatternFilter(patterns=["*/sub1*"])
+    chain = FilterChain(filters=[url_filter])
+    strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10, filter_chain=chain)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        # Hub (depth 0) bypasses filter; subsequent URLs should only match sub1
+        non_hub = [r for r in result_list if r.metadata.get("depth", 0) > 0]
+        for r in non_hub:
+            assert "sub1" in r.url, (
+                f"All non-hub results should be in sub1 branch, but found: {r.url}"
+            )
+
+
+@pytest.mark.asyncio
+async def test_url_pattern_filter_exclude(local_server):
+    """URLPatternFilter with reverse=True should exclude leaf pages."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    url_filter = URLPatternFilter(patterns=["*/leaf*"], reverse=True)
+    chain = FilterChain(filters=[url_filter])
+    strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=15, filter_chain=chain)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        leaf_urls = [r.url for r in result_list if "leaf" in r.url]
+        assert len(leaf_urls) == 0, (
+            f"Reverse pattern filter should exclude leaf pages, found: {leaf_urls}"
+        )
+
+
+@pytest.mark.asyncio
+async def test_domain_filter(local_server):
+    """DomainFilter allowing only 127.0.0.1 should keep local URLs only."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    domain_filter = DomainFilter(allowed_domains=["127.0.0.1"])
+    chain = FilterChain(filters=[domain_filter])
+    strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=10, filter_chain=chain)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        for r in result_list:
+            assert "127.0.0.1" in r.url, (
+                f"All results should be local, but found: {r.url}"
+            )
+
+
+@pytest.mark.asyncio
+async def test_filter_chain(local_server):
+    """FilterChain combining URLPatternFilter and DomainFilter should apply both."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    url_filter = URLPatternFilter(patterns=["*/sub1*"])
+    domain_filter = DomainFilter(allowed_domains=["127.0.0.1"])
+    chain = FilterChain(filters=[url_filter, domain_filter])
+    strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10, filter_chain=chain)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        non_hub = [r for r in result_list if r.metadata.get("depth", 0) > 0]
+        for r in non_hub:
+            assert "sub1" in r.url, (
+                f"URL pattern filter not applied: {r.url}"
+            )
+            assert "127.0.0.1" in r.url, (
+                f"Domain filter not applied: {r.url}"
+            )
+
+
+def test_content_type_filter():
+    """ContentTypeFilter should pass HTML URLs and reject image/pdf extensions."""
+    ct_filter = ContentTypeFilter(allowed_types=["text/html"])
+
+    assert ct_filter.apply("http://example.com/page") is True, (
+        "URL with no extension should pass (assumed HTML)"
+    )
+    assert ct_filter.apply("http://example.com/page.html") is True, (
+        ".html should pass text/html filter"
+    )
+    assert ct_filter.apply("http://example.com/photo.jpg") is False, (
+        ".jpg should be rejected by text/html filter"
+    )
+    assert ct_filter.apply("http://example.com/doc.pdf") is False, (
+        ".pdf should be rejected by text/html filter"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Scorers
+# ---------------------------------------------------------------------------
+
+
+def test_keyword_scorer():
+    """KeywordRelevanceScorer should rank URLs containing keywords higher."""
+    scorer = KeywordRelevanceScorer(keywords=["technology", "science"])
+
+    tech_score = scorer.score("http://example.com/technology/article")
+    generic_score = scorer.score("http://example.com/about/contact")
+
+    assert tech_score > generic_score, (
+        f"URL with keyword should score higher: tech={tech_score}, generic={generic_score}"
+    )
+
+    both_score = scorer.score("http://example.com/technology/science-report")
+    assert both_score >= tech_score, (
+        "URL matching both keywords should score at least as high as one keyword"
+    )
+
+
+def test_composite_scorer():
+    """CompositeScorer combining two scorers should produce scores without error."""
+    scorer1 = KeywordRelevanceScorer(keywords=["python"], weight=1.0)
+    scorer2 = KeywordRelevanceScorer(keywords=["crawl"], weight=0.5)
+    composite = CompositeScorer(scorers=[scorer1, scorer2])
+
+    score = composite.score("http://example.com/python-crawl-guide")
+    assert isinstance(score, float), "Composite score should be a float"
+    assert score > 0, "URL matching both scorers' keywords should have positive score"
+
+    zero_score = composite.score("http://example.com/unrelated-page")
+    assert zero_score == 0.0, "URL matching no keywords should score zero"
+
+
+# ---------------------------------------------------------------------------
+# URL normalization in deep crawl context
+# ---------------------------------------------------------------------------
+
+
+def test_deep_crawl_url_normalization():
+    """normalize_url_for_deep_crawl should resolve relative URLs against base."""
+    base = "http://example.com/deep/hub"
+
+    result = normalize_url_for_deep_crawl("/deep/sub1", base)
+    assert result == "http://example.com/deep/sub1", (
+        f"Relative URL not resolved correctly: {result}"
+    )
+
+    result2 = normalize_url_for_deep_crawl("sub2", base)
+    assert "example.com" in result2, "Relative path should resolve against base"
+    assert "sub2" in result2, "Relative path should include the target"
+
+
+def test_deep_crawl_trailing_slash():
+    """Trailing slashes should be preserved during normalization (fix #1520)."""
+    base = "http://example.com/"
+
+    with_slash = normalize_url_for_deep_crawl("/path/", base)
+    without_slash = normalize_url_for_deep_crawl("/path", base)
+
+    # The function uses `parsed.path or '/'` which preserves trailing slashes
+    assert with_slash.endswith("/path/"), (
+        f"Trailing slash should be preserved: {with_slash}"
+    )
+    assert not without_slash.endswith("/"), (
+        f"No trailing slash should be added: {without_slash}"
+    )
+
+
+def test_deep_crawl_deduplication():
+    """Same URL with different fragments should normalize to the same string."""
+    base = "http://example.com/"
+
+    url1 = normalize_url_for_deep_crawl("/page#section1", base)
+    url2 = normalize_url_for_deep_crawl("/page#section2", base)
+    url3 = normalize_url_for_deep_crawl("/page", base)
+
+    assert url1 == url2, (
+        f"Fragment-only difference should normalize to same URL: {url1} vs {url2}"
+    )
+    assert url1 == url3, (
+        f"URL with and without fragment should normalize the same: {url1} vs {url3}"
+    )
+
+
+def test_deep_crawl_efficient_normalization():
+    """efficient_normalize_url_for_deep_crawl should produce consistent results."""
+    base = "http://example.com/deep/hub"
+
+    result = efficient_normalize_url_for_deep_crawl("/deep/sub1", base)
+    assert result == "http://example.com/deep/sub1", (
+        f"Efficient normalization failed: {result}"
+    )
+
+    # Fragments should be removed
+    result_frag = efficient_normalize_url_for_deep_crawl("/page#anchor", base)
+    assert "#" not in result_frag, "Fragments should be stripped"
+
+
+def test_deep_crawl_normalization_none_input():
+    """Normalizing None or empty string should return None."""
+    result_none = normalize_url_for_deep_crawl(None, "http://example.com/")
+    assert result_none is None, "None input should return None"
+
+    result_empty = normalize_url_for_deep_crawl("", "http://example.com/")
+    assert result_empty is None, "Empty string should return None"
+
+
+def test_deep_crawl_normalization_case():
+    """Hostname normalization should be case-insensitive."""
+    base = "http://Example.COM/"
+
+    result = normalize_url_for_deep_crawl("/Page", base)
+    assert "example.com" in result, (
+        f"Hostname should be lowercased: {result}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Stream mode
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_deep_crawl_stream(local_server):
+    """Deep crawl with stream=True should yield results via async iteration."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=5)
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        stream=True,
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = []
+        async for result in await crawler.arun(url=hub_url, config=config):
+            results.append(result)
+
+        assert len(results) > 0, "Stream mode should yield at least one result"
+        assert results[0].success, "First streamed result should be successful"
+
+
+# ---------------------------------------------------------------------------
+# Real URL deep crawl
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_deep_crawl_real():
+    """Deep crawl https://quotes.toscrape.com with BFS to verify real-world usage."""
+    strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=3)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url="https://quotes.toscrape.com", config=config)
+
+        result_list = list(results)
+        assert len(result_list) >= 1, "Should crawl at least the start page"
+        assert result_list[0].success, "Start page should crawl successfully"
+        # The site has links; with max_depth=1 we should find some
+        if len(result_list) > 1:
+            assert result_list[1].metadata.get("depth") == 1, (
+                "Second-level pages should have depth 1"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Edge cases
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_bfs_max_pages_one(local_server):
+    """BFS with max_pages=1 should return exactly 1 result (the start page)."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BFSDeepCrawlStrategy(max_depth=5, max_pages=1)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        assert len(result_list) == 1, (
+            f"max_pages=1 should yield exactly 1 result, got {len(result_list)}"
+        )
+        assert "/deep/hub" in result_list[0].url, "The single result should be the hub"
+
+
+@pytest.mark.asyncio
+async def test_dfs_max_pages_one(local_server):
+    """DFS with max_pages=1 should return exactly 1 result."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = DFSDeepCrawlStrategy(max_depth=5, max_pages=1)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        assert len(result_list) == 1, (
+            f"max_pages=1 should yield exactly 1 result, got {len(result_list)}"
+        )
+
+
+@pytest.mark.asyncio
+async def test_bfs_depth_zero(local_server):
+    """BFS with max_depth=0 should only return the start page."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BFSDeepCrawlStrategy(max_depth=0, max_pages=100)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        assert len(result_list) == 1, (
+            f"max_depth=0 should yield exactly 1 result, got {len(result_list)}"
+        )
+        assert result_list[0].metadata["depth"] == 0, "Only depth-0 page should exist"
+
+
+@pytest.mark.asyncio
+async def test_bfs_results_have_parent_url(local_server):
+    """Each non-root result should have a parent_url in metadata."""
+    base = _to_ip_url(local_server)
+    hub_url = base + "/deep/hub"
+    strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=10)
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, verbose=False)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        results = await crawler.arun(url=hub_url, config=config)
+
+        result_list = list(results)
+        for r in result_list:
+            assert "parent_url" in r.metadata, (
+                f"Result for {r.url} should have 'parent_url' in metadata"
+            )
+            if r.metadata["depth"] == 0:
+                assert r.metadata["parent_url"] is None, (
+                    "Root page should have parent_url=None"
+                )
+            else:
+                assert r.metadata["parent_url"] is not None, (
+                    f"Non-root page {r.url} should have a parent_url"
+                )
+
+
+def test_url_pattern_filter_no_match():
+    """URLPatternFilter should reject URLs that match no patterns."""
+    f = URLPatternFilter(patterns=["*/special/*"])
+    assert f.apply("http://example.com/normal/page") is False
+    assert f.apply("http://example.com/special/page") is True
+
+
+def test_domain_filter_blocked():
+    """DomainFilter with blocked_domains should reject those domains."""
+    f = DomainFilter(blocked_domains=["evil.com"])
+    assert f.apply("http://evil.com/page") is False
+    assert f.apply("http://good.com/page") is True
+
+
+def test_domain_filter_subdomain():
+    """DomainFilter should handle subdomains of allowed domains."""
+    f = DomainFilter(allowed_domains=["example.com"])
+    assert f.apply("http://example.com/page") is True
+    assert f.apply("http://sub.example.com/page") is True
+    assert f.apply("http://other.com/page") is False
+
+
+def test_keyword_scorer_case_insensitive():
+    """KeywordRelevanceScorer should be case-insensitive by default."""
+    scorer = KeywordRelevanceScorer(keywords=["Python"])
+    score_lower = scorer.score("http://example.com/python-guide")
+    score_upper = scorer.score("http://example.com/PYTHON-GUIDE")
+    assert score_lower > 0, "Lowercase URL should match"
+    assert score_upper > 0, "Uppercase URL should match"
+
+
+def test_keyword_scorer_no_match():
+    """KeywordRelevanceScorer should return 0 for URLs with no keyword matches."""
+    scorer = KeywordRelevanceScorer(keywords=["quantum", "physics"])
+    score = scorer.score("http://example.com/cooking/recipes")
+    assert score == 0.0, "No keywords matched should give zero score"
diff --git a/tests/regression/test_reg_domain_mapper.py b/tests/regression/test_reg_domain_mapper.py
new file mode 100644
index 0000000..bc028a4
--- /dev/null
+++ b/tests/regression/test_reg_domain_mapper.py
@@ -0,0 +1,270 @@
+"""
+Crawl4AI Regression Tests - DomainMapper
+
+Tests DomainMapper functionality: host discovery, soft-404 detection,
+multi-source scanning, post-processing, and crawler integration.
+
+All network tests use real endpoints.
+"""
+
+import pytest
+import pytest_asyncio
+from crawl4ai import DomainMapper, DomainMapperConfig, AsyncWebCrawler
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+@pytest_asyncio.fixture
+async def mapper():
+    async with DomainMapper() as m:
+        yield m
+
+
+# ---------------------------------------------------------------------------
+# Basic scan tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_basic_scan(mapper):
+    """Scan a domain with sitemaps and get results."""
+    config = DomainMapperConfig(
+        source="sitemap",
+        extract_head=False,
+        verbose=False,
+    )
+    results = await mapper.scan("docs.crawl4ai.com", config)
+    assert len(results) >= 1, "Should find at least 1 URL from sitemap"
+    assert all("url" in r for r in results)
+    assert all("host" in r for r in results)
+    assert all("source" in r for r in results)
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_scan_with_head_extraction(mapper):
+    """Head extraction should populate title and meta."""
+    config = DomainMapperConfig(
+        source="sitemap",
+        extract_head=True,
+        max_urls=3,
+        verbose=False,
+    )
+    results = await mapper.scan("docs.crawl4ai.com", config)
+    assert len(results) >= 1
+    has_title = any(r.get("head_data", {}).get("title") for r in results)
+    assert has_title, "At least one result should have a title"
+
+
+# ---------------------------------------------------------------------------
+# Host discovery tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_crt_subdomain_discovery(mapper):
+    """crt.sh should discover subdomains."""
+    config = DomainMapperConfig(
+        source="crt+probe",
+        extract_head=False,
+        verbose=False,
+    )
+    results = await mapper.scan("superdesign.dev", config)
+    hosts = {r["host"] for r in results}
+    assert len(hosts) >= 3, f"Expected >=3 hosts via crt, got {len(hosts)}: {hosts}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_dns_subdomain_guessing(mapper):
+    """DNS guessing should find common subdomains."""
+    hosts = await mapper._guess_subdomains("crawl4ai.com", ["docs", "www", "api"], DomainMapperConfig())
+    # docs.crawl4ai.com should resolve
+    assert "docs.crawl4ai.com" in hosts
+
+
+# ---------------------------------------------------------------------------
+# Soft-404 detection tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_soft_404_detection(mapper):
+    """SPA site should be detected as soft-404."""
+    fp = await mapper._fingerprint_soft_404("app.superdesign.dev", DomainMapperConfig())
+    assert fp is not None
+    assert fp.status_code == 200, "SPA should return 200 for nonexistent paths"
+    assert fp.title is not None, "Should capture the SPA shell title"
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_soft_404_filters_probes(mapper):
+    """Probing an SPA with soft-404 enabled should filter all paths."""
+    config = DomainMapperConfig(
+        source="probe",
+        soft_404_detection=True,
+        extract_head=False,
+        verbose=False,
+    )
+    results = await mapper.scan("app.superdesign.dev", config)
+    probe_urls = [r for r in results if r["source"] == "probe"]
+    assert len(probe_urls) == 0, "All probe paths on SPA should be soft-404 filtered"
+
+
+# ---------------------------------------------------------------------------
+# Source isolation tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_sitemap_only_no_cross_contamination(mapper):
+    """source='sitemap' should only produce sitemap-sourced results."""
+    config = DomainMapperConfig(
+        source="sitemap",
+        extract_head=False,
+        verbose=False,
+    )
+    results = await mapper.scan("docs.crawl4ai.com", config)
+    for r in results:
+        for part in r["source"].split("+"):
+            assert part == "sitemap", f"Unexpected source: {part}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_probe_only(mapper):
+    """source='probe' should work standalone."""
+    config = DomainMapperConfig(
+        source="probe",
+        extract_head=False,
+        verbose=False,
+    )
+    results = await mapper.scan("docs.crawl4ai.com", config)
+    assert isinstance(results, list)
+    assert len(results) >= 1
+
+
+# ---------------------------------------------------------------------------
+# Post-processing tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_max_urls_respected(mapper):
+    """max_urls should cap results."""
+    config = DomainMapperConfig(
+        source="sitemap+probe",
+        extract_head=False,
+        max_urls=5,
+        verbose=False,
+    )
+    results = await mapper.scan("docs.crawl4ai.com", config)
+    assert len(results) <= 5
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_nonsense_filter_removes_assets(mapper):
+    """Nonsense filter should remove JS/CSS/image URLs."""
+    config = DomainMapperConfig(
+        source="sitemap+homepage",
+        extract_head=False,
+        filter_nonsense_urls=True,
+        verbose=False,
+    )
+    results = await mapper.scan("docs.crawl4ai.com", config)
+    for r in results:
+        url = r["url"].lower()
+        assert not url.endswith(".js"), f"JS file should be filtered: {url}"
+        assert not url.endswith(".css"), f"CSS file should be filtered: {url}"
+        assert not url.endswith(".png"), f"Image should be filtered: {url}"
+
+
+# ---------------------------------------------------------------------------
+# Error handling tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+async def test_invalid_source_raises(mapper):
+    """Invalid source should raise ValueError."""
+    with pytest.raises(ValueError, match="Invalid source"):
+        await mapper.scan("example.com", DomainMapperConfig(source="bogus"))
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_nonexistent_domain(mapper):
+    """Nonexistent domain should return empty list, not crash."""
+    config = DomainMapperConfig(
+        source="sitemap+probe",
+        extract_head=False,
+        verbose=False,
+    )
+    results = await mapper.scan("thiswillneverexist99999.dev", config)
+    assert results == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_domain_with_scheme_stripped(mapper):
+    """Domain with https:// prefix should work."""
+    config = DomainMapperConfig(
+        source="sitemap",
+        extract_head=False,
+        max_urls=3,
+        verbose=False,
+    )
+    results = await mapper.scan("https://docs.crawl4ai.com", config)
+    assert len(results) >= 1
+
+
+# ---------------------------------------------------------------------------
+# Crawler integration tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_amap_domain_on_crawler():
+    """AsyncWebCrawler.amap_domain() should work end-to-end."""
+    async with AsyncWebCrawler() as crawler:
+        results = await crawler.amap_domain(
+            "docs.crawl4ai.com",
+            DomainMapperConfig(
+                source="sitemap",
+                extract_head=False,
+                max_urls=5,
+                verbose=False,
+            ),
+        )
+        assert len(results) >= 1
+        assert all("url" in r for r in results)
+
+
+# ---------------------------------------------------------------------------
+# Config tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+async def test_config_clone():
+    """DomainMapperConfig.clone() should produce correct copies."""
+    config = DomainMapperConfig(source="sitemap", max_urls=10, verbose=True)
+    cloned = config.clone(max_urls=20, force=True)
+    assert cloned.max_urls == 20
+    assert cloned.force is True
+    assert cloned.source == "sitemap"
+    assert cloned.verbose is True
+
+
+@pytest.mark.asyncio
+async def test_config_from_kwargs():
+    """DomainMapperConfig.from_kwargs() should work."""
+    config = DomainMapperConfig.from_kwargs({
+        "source": "crt+probe",
+        "max_urls": 50,
+    })
+    assert config.source == "crt+probe"
+    assert config.max_urls == 50
+    assert config.extract_head is True  # default
diff --git a/tests/regression/test_reg_edge_cases.py b/tests/regression/test_reg_edge_cases.py
new file mode 100644
index 0000000..a5821a0
--- /dev/null
+++ b/tests/regression/test_reg_edge_cases.py
@@ -0,0 +1,359 @@
+"""
+Crawl4AI Regression Tests - Edge Cases and Error Handling
+
+Adversarial tests for empty pages, malformed HTML, large pages, unicode,
+concurrent crawls, error recovery, and other boundary conditions.
+
+All tests use real browser crawling with no mocking.
+"""
+
+import asyncio
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.cache_context import CacheMode
+
+
+# ---------------------------------------------------------------------------
+# Empty and minimal pages
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_empty_page(local_server):
+    """Crawl an empty page and verify no crash. Anti-bot may flag it as blocked."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/empty")
+        # An empty page may be flagged by the anti-bot detector as "near-empty content"
+        # so success may be False. The key thing is no unhandled exception and
+        # we get a result object back.
+        assert result.html is not None, "HTML should not be None for empty page"
+        # Markdown should be empty or minimal
+        md = result.markdown or ""
+        assert len(md.strip()) < 50, (
+            "Empty page should produce little to no markdown"
+        )
+
+
+@pytest.mark.asyncio
+async def test_empty_raw_html():
+    """Crawl raw HTML with empty body; should succeed without crash."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun("raw:")
+        assert result.success, f"Empty raw HTML crawl failed: {result.error_message}"
+
+
+# ---------------------------------------------------------------------------
+# Malformed HTML
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_malformed_html(local_server):
+    """Crawl intentionally broken HTML; should not crash, even if anti-bot flags it."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/malformed")
+        # The malformed HTML is so broken that the browser may put content into
+        # unexpected places (e.g., the title). The anti-bot detector may flag the
+        # result as blocked due to empty body. The key assertion is: no unhandled
+        # exception and we get a result object back with html content.
+        assert result.html is not None, "Should still return HTML even for malformed pages"
+        assert len(result.html) > 0, "HTML should be non-empty for malformed page"
+
+
+@pytest.mark.asyncio
+async def test_raw_html_no_doctype():
+    """Raw HTML without doctype or  wrapper should still parse."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun("raw:
      No doctype
      ")
+        assert result.success, f"No-doctype raw HTML failed: {result.error_message}"
+        assert "No doctype" in (result.markdown or ""), (
+            "Content should be extracted despite missing doctype"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Large pages
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_large_page(local_server):
+    """Crawl a page with 50 sections and verify content from beginning and end."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/large")
+        assert result.success, f"Large page crawl failed: {result.error_message}"
+        md = result.markdown or ""
+        assert "Section 0" in md, "Markdown should contain content from section 0"
+        assert "Section 49" in md, "Markdown should contain content from section 49"
+
+
+# ---------------------------------------------------------------------------
+# Unicode and special characters
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_unicode_content():
+    """Crawl raw HTML with unicode characters and verify they survive extraction."""
+    raw = "raw:
      Unicode: \u00e9\u00e8\u00ea \u4e16\u754c \U0001f600
      "
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(raw)
+        assert result.success, f"Unicode crawl failed: {result.error_message}"
+        md = result.markdown or ""
+        assert "\u00e9" in md, "French accented 'e' should be in markdown"
+        assert "\u4e16\u754c" in md, "Chinese characters should be in markdown"
+        # Emoji may or may not survive depending on markdown generator;
+        # at least the other unicode should be present
+
+
+@pytest.mark.asyncio
+async def test_html_entities():
+    """Crawl raw HTML with entities and verify they are decoded in markdown."""
+    raw = "raw:
      & < > " '
      "
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(raw)
+        assert result.success, f"HTML entities crawl failed: {result.error_message}"
+        md = result.markdown or ""
+        assert "&" in md, "Ampersand entity should be decoded"
+        assert "<" in md, "Less-than entity should be decoded"
+        assert ">" in md, "Greater-than entity should be decoded"
+
+
+# ---------------------------------------------------------------------------
+# Multiple crawls - no state leakage
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_sequential_crawls_no_leakage(local_server):
+    """Crawl 3 different pages sequentially; verify no content bleed."""
+    pages = [
+        (local_server + "/products", "Wireless Mouse"),
+        (local_server + "/tables", "Sales Report"),
+        (local_server + "/js-dynamic", "Static Section"),
+    ]
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        for url, expected_content in pages:
+            result = await crawler.arun(url, config=config)
+            assert result.success, f"Sequential crawl of {url} failed: {result.error_message}"
+            md = result.markdown or ""
+            assert expected_content in md, (
+                f"Expected '{expected_content}' in markdown for {url}, "
+                f"got: {md[:200]}..."
+            )
+
+
+# ---------------------------------------------------------------------------
+# Raw HTML edge cases
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_raw_html_only_whitespace():
+    """Raw HTML with only whitespace body should succeed with empty markdown."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun("raw:   \n\t  ")
+        assert result.success, f"Whitespace-only raw HTML failed: {result.error_message}"
+        md = result.markdown or ""
+        assert len(md.strip()) < 20, "Whitespace-only body should produce minimal markdown"
+
+
+@pytest.mark.asyncio
+async def test_raw_html_script_only():
+    """Raw HTML with only a script tag should produce empty markdown (scripts stripped)."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            "raw:"
+        )
+        assert result.success, f"Script-only raw HTML failed: {result.error_message}"
+        md = result.markdown or ""
+        assert "var x" not in md, "Script content should be stripped from markdown"
+
+
+# ---------------------------------------------------------------------------
+# Concurrent crawls
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_concurrent_crawls(local_server):
+    """Use asyncio.gather to crawl 5 pages concurrently with same crawler."""
+    urls = [
+        local_server + "/",
+        local_server + "/products",
+        local_server + "/tables",
+        local_server + "/links-page",
+        local_server + "/images-page",
+    ]
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        tasks = [crawler.arun(url, config=config) for url in urls]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        for i, result in enumerate(results):
+            assert not isinstance(result, Exception), (
+                f"Concurrent crawl {i} raised exception: {result}"
+            )
+            assert result.success, (
+                f"Concurrent crawl {i} ({urls[i]}) failed: {result.error_message}"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Very long URL
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_long_url(local_server):
+    """Crawl a URL with a very long path (200 chars); catch-all handler serves it."""
+    long_path = "/" + "a" * 200
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + long_path)
+        assert result.success, f"Long URL crawl failed: {result.error_message}"
+
+
+# ---------------------------------------------------------------------------
+# Special URL characters
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_url_with_query_params(local_server):
+    """Crawl a URL with query parameters and verify success."""
+    url = local_server + "/products?page=1&sort=name&filter=electronics"
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url)
+        assert result.success, f"Query params URL crawl failed: {result.error_message}"
+
+
+@pytest.mark.asyncio
+async def test_url_with_fragment(local_server):
+    """Crawl a URL with a fragment identifier and verify success."""
+    url = local_server + "/#section-5"
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url)
+        assert result.success, f"Fragment URL crawl failed: {result.error_message}"
+
+
+# ---------------------------------------------------------------------------
+# Error recovery
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_invalid_url_scheme():
+    """Try crawling an FTP URL; should handle gracefully without crash."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun("ftp://example.com")
+        # Either it fails gracefully with an error or succeeds with empty content
+        # The critical thing is no unhandled exception
+        if not result.success:
+            assert result.error_message is not None, (
+                "Invalid scheme should produce an error message"
+            )
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_nonexistent_domain():
+    """Try crawling a nonexistent domain; should fail gracefully."""
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            "https://this-domain-definitely-does-not-exist-xyz123.com",
+            config=CrawlerRunConfig(page_timeout=10000),
+        )
+        # Should fail but not crash
+        if not result.success:
+            assert result.error_message is not None, (
+                "Nonexistent domain should produce an error message"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Multiple identical crawls (idempotency)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_idempotent_crawl(local_server):
+    """Crawl same URL twice with BYPASS cache; both should succeed with similar content."""
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result1 = await crawler.arun(local_server + "/products", config=config)
+        result2 = await crawler.arun(local_server + "/products", config=config)
+        assert result1.success, f"First crawl failed: {result1.error_message}"
+        assert result2.success, f"Second crawl failed: {result2.error_message}"
+        # Both should have similar content length (within 20% tolerance)
+        len1 = len(result1.markdown or "")
+        len2 = len(result2.markdown or "")
+        if len1 > 0 and len2 > 0:
+            ratio = min(len1, len2) / max(len1, len2)
+            assert ratio > 0.8, (
+                f"Idempotent crawls should produce similar content "
+                f"(len1={len1}, len2={len2}, ratio={ratio:.2f})"
+            )
+
+
+# ---------------------------------------------------------------------------
+# PDF generation
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_pdf_capture(local_server):
+    """Crawl with pdf=True and verify PDF bytes output."""
+    config = CrawlerRunConfig(pdf=True, cache_mode=CacheMode.BYPASS)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/", config=config)
+        assert result.success, f"PDF capture crawl failed: {result.error_message}"
+        assert result.pdf is not None, "PDF should not be None"
+        assert isinstance(result.pdf, bytes), "PDF should be bytes"
+        assert len(result.pdf) > 0, "PDF should be non-empty"
+        # PDF files start with %PDF
+        assert result.pdf[:4] == b"%PDF", "PDF should start with %PDF header"
+
+
+# ---------------------------------------------------------------------------
+# Scan full page
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_scan_full_page(local_server):
+    """Crawl /large with scan_full_page=True to scroll through entire page."""
+    config = CrawlerRunConfig(
+        scan_full_page=True,
+        scroll_delay=0.1,
+        cache_mode=CacheMode.BYPASS,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/large", config=config)
+        assert result.success, f"Scan full page crawl failed: {result.error_message}"
+        md = result.markdown or ""
+        assert len(md) > 100, "Full page scan should produce substantial markdown"
+
+
+# ---------------------------------------------------------------------------
+# Console capture
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_console_capture(local_server):
+    """Crawl /js-dynamic with capture_console_messages=True; verify no error."""
+    config = CrawlerRunConfig(
+        capture_console_messages=True,
+        cache_mode=CacheMode.BYPASS,
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(local_server + "/js-dynamic", config=config)
+        assert result.success, f"Console capture crawl failed: {result.error_message}"
+        # console_messages should be a list (possibly empty)
+        assert result.console_messages is not None, (
+            "console_messages should not be None when capture_console_messages=True"
+        )
+        assert isinstance(result.console_messages, list), (
+            "console_messages should be a list"
+        )
diff --git a/tests/regression/test_reg_extraction.py b/tests/regression/test_reg_extraction.py
new file mode 100644
index 0000000..7d70098
--- /dev/null
+++ b/tests/regression/test_reg_extraction.py
@@ -0,0 +1,608 @@
+"""
+Regression tests for Crawl4AI extraction strategies.
+
+Covers JsonCssExtractionStrategy, JsonXPathExtractionStrategy,
+JsonLxmlExtractionStrategy, RegexExtractionStrategy, NoExtractionStrategy,
+and CosineStrategy (optional, requires sklearn).
+
+Run:
+    pytest tests/regression/test_reg_extraction.py -v
+    pytest tests/regression/test_reg_extraction.py -v -m "not network"
+"""
+
+import pytest
+import json
+import time
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.extraction_strategy import (
+    JsonCssExtractionStrategy,
+    JsonXPathExtractionStrategy,
+    JsonLxmlExtractionStrategy,
+    RegexExtractionStrategy,
+    NoExtractionStrategy,
+)
+
+try:
+    from crawl4ai.extraction_strategy import CosineStrategy
+    # CosineStrategy requires torch and sklearn at instantiation time;
+    # verify they are actually available before declaring it usable.
+    import torch  # noqa: F401
+    HAS_COSINE = True
+except (ImportError, ModuleNotFoundError):
+    HAS_COSINE = False
+
+
+# ---------------------------------------------------------------------------
+# JsonCssExtractionStrategy
+# ---------------------------------------------------------------------------
+
+PRODUCT_CSS_SCHEMA = {
+    "baseSelector": "div.product",
+    "fields": [
+        {"name": "name", "selector": "h2.name", "type": "text"},
+        {"name": "price", "selector": "span.price", "type": "text"},
+        {"name": "description", "selector": "p.description", "type": "text"},
+        {"name": "category", "selector": "span.category", "type": "text"},
+        {
+            "name": "link",
+            "selector": "a.details-link",
+            "type": "attribute",
+            "attribute": "href",
+        },
+    ],
+}
+
+PRODUCT_CSS_SCHEMA_WITH_ID = {
+    "baseSelector": "div.product",
+    "baseFields": [
+        {
+            "name": "product_id",
+            "type": "attribute",
+            "attribute": "data-id",
+        },
+    ],
+    "fields": [
+        {"name": "name", "selector": "h2.name", "type": "text"},
+        {"name": "price", "selector": "span.price", "type": "text"},
+        {"name": "description", "selector": "p.description", "type": "text"},
+        {"name": "category", "selector": "span.category", "type": "text"},
+        {
+            "name": "link",
+            "selector": "a.details-link",
+            "type": "attribute",
+            "attribute": "href",
+        },
+    ],
+}
+
+
+@pytest.mark.asyncio
+async def test_css_extract_products(local_server):
+    """Extract all 5 products from /products using JsonCssExtractionStrategy.
+    Verify count, first product name, price, and product_id."""
+    strategy = JsonCssExtractionStrategy(schema=PRODUCT_CSS_SCHEMA_WITH_ID)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/products", config=config)
+        assert result.success, f"Crawl failed: {result.error_message}"
+        extracted = json.loads(result.extracted_content)
+        assert isinstance(extracted, list)
+        assert len(extracted) == 5, f"Expected 5 products, got {len(extracted)}"
+
+        first = extracted[0]
+        assert first["name"] == "Wireless Mouse"
+        assert first["price"] == "$29.99"
+        assert first["product_id"] == "1"
+
+
+@pytest.mark.asyncio
+async def test_css_extract_with_default(local_server):
+    """Use a field with a non-existent selector and a default value.
+    Verify the default is used when no element matches."""
+    schema = {
+        "baseSelector": "div.product",
+        "fields": [
+            {"name": "name", "selector": "h2.name", "type": "text"},
+            {
+                "name": "sku",
+                "selector": "span.sku-number",
+                "type": "text",
+                "default": "N/A",
+            },
+        ],
+    }
+    strategy = JsonCssExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/products", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) > 0
+        for item in extracted:
+            assert item["sku"] == "N/A", (
+                f"Expected default 'N/A' for missing sku, got: {item.get('sku')}"
+            )
+
+
+@pytest.mark.asyncio
+async def test_css_extract_nested(local_server):
+    """Test nested type extraction using JsonCssExtractionStrategy.
+    Extract a nested object from within each product element."""
+    schema = {
+        "baseSelector": "div.product",
+        "fields": [
+            {"name": "name", "selector": "h2.name", "type": "text"},
+            {
+                "name": "details",
+                "selector": "div.rating",
+                "type": "nested",
+                "fields": [
+                    {
+                        "name": "stars",
+                        "type": "attribute",
+                        "attribute": "data-stars",
+                    },
+                ],
+            },
+        ],
+    }
+    strategy = JsonCssExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/products", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) == 5
+        first = extracted[0]
+        assert "details" in first
+        assert first["details"]["stars"] == "4.5"
+
+
+@pytest.mark.asyncio
+async def test_css_extract_empty_results(local_server):
+    """Use a baseSelector that matches nothing and verify an empty list is returned."""
+    schema = {
+        "baseSelector": "div.nonexistent-class-xyz",
+        "fields": [
+            {"name": "text", "selector": "p", "type": "text"},
+        ],
+    }
+    strategy = JsonCssExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/products", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert isinstance(extracted, list)
+        assert len(extracted) == 0
+
+
+@pytest.mark.asyncio
+async def test_css_extract_table(local_server):
+    """Extract table rows from /tables using CSS selectors.
+    Verify 4 quarterly rows with correct Q1 revenue."""
+    schema = {
+        "baseSelector": "#sales-table tbody tr",
+        "fields": [
+            {"name": "quarter", "selector": "td:nth-child(1)", "type": "text"},
+            {"name": "revenue", "selector": "td:nth-child(2)", "type": "text"},
+            {"name": "growth", "selector": "td:nth-child(3)", "type": "text"},
+        ],
+    }
+    strategy = JsonCssExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/tables", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) == 4, f"Expected 4 rows, got {len(extracted)}"
+        assert extracted[0]["quarter"] == "Q1 2025"
+        assert extracted[0]["revenue"] == "$1,234,567"
+        assert extracted[0]["growth"] == "12.5%"
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_css_real_quotes():
+    """Crawl quotes.toscrape.com and extract quotes with CSS selectors.
+    Verify multiple quotes are extracted with text and author."""
+    schema = {
+        "baseSelector": "div.quote",
+        "fields": [
+            {"name": "text", "selector": "span.text", "type": "text"},
+            {"name": "author", "selector": "small.author", "type": "text"},
+        ],
+    }
+    strategy = JsonCssExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url="https://quotes.toscrape.com", config=config
+        )
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) > 0, "Expected quotes to be extracted"
+        for quote in extracted:
+            assert "text" in quote and quote["text"], f"Quote missing text: {quote}"
+            assert "author" in quote and quote["author"], f"Quote missing author: {quote}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_css_real_books():
+    """Crawl books.toscrape.com and extract book titles and prices."""
+    schema = {
+        "baseSelector": "article.product_pod",
+        "fields": [
+            {"name": "title", "selector": "h3 a", "type": "attribute", "attribute": "title"},
+            {"name": "price", "selector": "p.price_color", "type": "text"},
+        ],
+    }
+    strategy = JsonCssExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url="https://books.toscrape.com", config=config
+        )
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) > 0, "Expected books to be extracted"
+        for book in extracted:
+            assert "title" in book and book["title"]
+            assert "price" in book and book["price"]
+            # Price should start with a currency symbol
+            assert book["price"][0] in ("£", "$", "€") or book["price"].startswith("£")
+
+
+# ---------------------------------------------------------------------------
+# JsonXPathExtractionStrategy
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_xpath_extract_products(local_server):
+    """Extract products using XPath selectors. Verify same results as CSS version."""
+    schema = {
+        # Use exact class match to avoid matching 'product-list' parent
+        "baseSelector": "//div[contains(concat(' ', normalize-space(@class), ' '), ' product ')]",
+        "fields": [
+            {
+                "name": "name",
+                "selector": ".//h2[contains(@class, 'name')]",
+                "type": "text",
+            },
+            {
+                "name": "price",
+                "selector": ".//span[contains(@class, 'price')]",
+                "type": "text",
+            },
+        ],
+    }
+    strategy = JsonXPathExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/products", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) == 5, f"Expected 5 products via XPath, got {len(extracted)}"
+        assert extracted[0]["name"] == "Wireless Mouse"
+        assert extracted[0]["price"] == "$29.99"
+
+
+# ---------------------------------------------------------------------------
+# JsonLxmlExtractionStrategy
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_lxml_extract_products(local_server):
+    """Extract products using JsonLxmlExtractionStrategy with the same
+    CSS-style schema. Verify same results as JsonCss."""
+    strategy = JsonLxmlExtractionStrategy(schema=PRODUCT_CSS_SCHEMA)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/products", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) == 5, f"Expected 5 products via lxml, got {len(extracted)}"
+        assert extracted[0]["name"] == "Wireless Mouse"
+        assert extracted[0]["price"] == "$29.99"
+
+
+@pytest.mark.asyncio
+async def test_lxml_caching_performance(local_server):
+    """Extract twice with the same JsonLxmlExtractionStrategy instance.
+    Second extraction should be faster or equal due to caching."""
+    strategy = JsonLxmlExtractionStrategy(schema=PRODUCT_CSS_SCHEMA)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        # First run
+        t0 = time.perf_counter()
+        result1 = await crawler.arun(url=f"{local_server}/products", config=config)
+        t1 = time.perf_counter()
+        first_time = t1 - t0
+
+        # Second run (caching should help)
+        t2 = time.perf_counter()
+        result2 = await crawler.arun(url=f"{local_server}/products", config=config)
+        t3 = time.perf_counter()
+        second_time = t3 - t2
+
+        assert result1.success and result2.success
+        data1 = json.loads(result1.extracted_content)
+        data2 = json.loads(result2.extracted_content)
+        assert len(data1) == len(data2) == 5
+
+        # Allow generous tolerance -- caching may not always be faster due to
+        # browser overhead, but it should certainly not be drastically slower
+        assert second_time < first_time * 3, (
+            f"Second run ({second_time:.3f}s) significantly slower than first ({first_time:.3f}s)"
+        )
+
+
+# ---------------------------------------------------------------------------
+# RegexExtractionStrategy
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_regex_email(local_server):
+    """Extract emails from /regex-test using the Email pattern.
+    Verify both expected addresses are found."""
+    strategy = RegexExtractionStrategy(pattern=RegexExtractionStrategy.Email)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/regex-test", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        values = [item["value"] for item in extracted]
+        assert any("support@crawl4ai.com" in v for v in values), (
+            f"Expected support@crawl4ai.com in {values}"
+        )
+        assert any("sales@example.org" in v for v in values), (
+            f"Expected sales@example.org in {values}"
+        )
+
+
+@pytest.mark.asyncio
+async def test_regex_phone(local_server):
+    """Extract US phone numbers from /regex-test."""
+    strategy = RegexExtractionStrategy(pattern=RegexExtractionStrategy.PhoneUS)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/regex-test", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        values = [item["value"] for item in extracted]
+        assert len(values) > 0, "Expected at least one phone number"
+        # At least one phone number should contain expected digits
+        all_vals = " ".join(values)
+        assert "555" in all_vals, f"Expected phone with 555 in {values}"
+
+
+@pytest.mark.asyncio
+async def test_regex_url(local_server):
+    """Extract URLs from /regex-test using the Url pattern."""
+    strategy = RegexExtractionStrategy(pattern=RegexExtractionStrategy.Url)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/regex-test", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        values = [item["value"] for item in extracted]
+        assert len(values) > 0, "Expected URLs to be extracted"
+        all_vals = " ".join(values)
+        assert "crawl4ai.com" in all_vals
+
+
+@pytest.mark.asyncio
+async def test_regex_all(local_server):
+    """Use RegexExtractionStrategy.All to extract all built-in patterns.
+    Verify it finds emails, phones, URLs, dates, and more."""
+    strategy = RegexExtractionStrategy(pattern=RegexExtractionStrategy.All)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/regex-test", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        labels = {item["label"] for item in extracted}
+        # Should find at least emails, URLs, and dates
+        assert "email" in labels, f"Expected 'email' in labels: {labels}"
+        assert "url" in labels, f"Expected 'url' in labels: {labels}"
+        assert "date_iso" in labels or "date_us" in labels, (
+            f"Expected date patterns in labels: {labels}"
+        )
+
+
+@pytest.mark.asyncio
+async def test_regex_custom(local_server):
+    """Use a custom regex pattern to extract IPv4 addresses.
+    Verify 192.168.1.100 is found."""
+    strategy = RegexExtractionStrategy(
+        custom={"ip_address": r"(?:\d{1,3}\.){3}\d{1,3}"}
+    )
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/regex-test", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        values = [item["value"] for item in extracted]
+        assert "192.168.1.100" in values, f"Expected 192.168.1.100 in {values}"
+
+
+@pytest.mark.asyncio
+async def test_regex_output_format(local_server):
+    """Verify each regex extraction result has the expected keys:
+    url, label, value, span."""
+    strategy = RegexExtractionStrategy(pattern=RegexExtractionStrategy.Email)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/regex-test", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) > 0
+        for item in extracted:
+            assert "url" in item, f"Missing 'url' key in {item}"
+            assert "label" in item, f"Missing 'label' key in {item}"
+            assert "value" in item, f"Missing 'value' key in {item}"
+            assert "span" in item, f"Missing 'span' key in {item}"
+            # Span should be a list/tuple of two ints
+            span = item["span"]
+            assert isinstance(span, (list, tuple)) and len(span) == 2
+
+
+@pytest.mark.asyncio
+async def test_regex_span_accuracy(local_server):
+    """Verify that span[0]:span[1] in the source content equals value.
+    This tests that span offsets are accurate relative to the input text."""
+    strategy = RegexExtractionStrategy(pattern=RegexExtractionStrategy.Email)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/regex-test", config=config)
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) > 0
+
+        # The regex runs on the content source (fit_html by default).
+        # We verify the span produces the correct value from that source.
+        # Since we cannot easily get the exact input text the regex ran on,
+        # we verify span[0] < span[1] and the value is non-empty.
+        for item in extracted:
+            span = item["span"]
+            assert span[0] < span[1], f"Invalid span: {span}"
+            assert len(item["value"]) > 0
+            assert span[1] - span[0] == len(item["value"]), (
+                f"Span length ({span[1] - span[0]}) != value length ({len(item['value'])})"
+            )
+
+
+# ---------------------------------------------------------------------------
+# NoExtractionStrategy
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_no_extraction(local_server):
+    """Crawl with NoExtractionStrategy and verify the framework skips
+    structured extraction (passthrough behavior). The crawler deliberately
+    bypasses extraction for NoExtractionStrategy, leaving extracted_content
+    as None. The actual page content is still available via markdown and html."""
+    strategy = NoExtractionStrategy()
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(url=f"{local_server}/", config=config)
+        assert result.success
+        # The framework explicitly skips extraction for NoExtractionStrategy,
+        # so extracted_content should be None (passthrough -- no processing).
+        assert result.extracted_content is None
+        # But the page content is still fully available
+        assert result.html is not None and len(result.html) > 0
+        assert result.markdown is not None and "Welcome" in result.markdown
+
+
+# ---------------------------------------------------------------------------
+# CosineStrategy (optional - requires sklearn)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.skipif(not HAS_COSINE, reason="CosineStrategy requires sklearn+torch")
+def test_cosine_basic():
+    """Test CosineStrategy extract() directly with pre-chunked text to verify clustering works."""
+    # CosineStrategy.extract() expects text with <|DEL|> or \\n\\n separators.
+    # We test the strategy directly to avoid browser overhead and isolate the logic.
+    topics = [
+        "Machine learning algorithms process large datasets to identify complex patterns "
+        "and make accurate predictions using neural networks and deep learning models.",
+        "Cloud computing provides scalable infrastructure for deploying web applications "
+        "globally across multiple regions and availability zones for high availability.",
+        "Database optimization requires careful indexing strategies and query performance "
+        "tuning to handle millions of transactions per second efficiently.",
+        "Network security involves configuring firewalls intrusion detection systems and "
+        "encrypted communications to protect against cyber threats and attacks.",
+        "Mobile development frameworks enable building cross-platform applications with "
+        "shared codebases that deploy to both iOS and Android platforms.",
+    ]
+    text = "<|DEL|>".join(topics)
+
+    strategy = CosineStrategy(
+        semantic_filter=None,
+        word_count_threshold=5,
+        max_dist=0.5,
+    )
+    result = strategy.extract(url="http://test.com", html=text)
+    assert isinstance(result, list)
+    assert len(result) > 0, "Expected clusters from CosineStrategy"
+    # Each cluster should have 'content' and 'index' keys
+    for item in result:
+        assert "content" in item
+        assert "index" in item
+
+
+# ---------------------------------------------------------------------------
+# Extraction with real URLs
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_extraction_real_quotes_css():
+    """Full pipeline: crawl quotes.toscrape.com, extract with JsonCss,
+    verify structured quote data including text and author."""
+    schema = {
+        "baseSelector": "div.quote",
+        "fields": [
+            {"name": "text", "selector": "span.text", "type": "text"},
+            {"name": "author", "selector": "small.author", "type": "text"},
+            {
+                "name": "tags",
+                "selector": "div.tags",
+                "type": "nested",
+                "fields": [
+                    {
+                        "name": "tag_list",
+                        "selector": "a.tag",
+                        "type": "text",
+                    },
+                ],
+            },
+        ],
+    }
+    strategy = JsonCssExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url="https://quotes.toscrape.com", config=config
+        )
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) >= 5, f"Expected at least 5 quotes, got {len(extracted)}"
+        for quote in extracted:
+            assert quote.get("text"), "Quote text should not be empty"
+            assert quote.get("author"), "Quote author should not be empty"
+
+
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_extraction_real_books_css():
+    """Crawl books.toscrape.com and extract book listings with titles and prices."""
+    schema = {
+        "baseSelector": "article.product_pod",
+        "fields": [
+            {"name": "title", "selector": "h3 a", "type": "attribute", "attribute": "title"},
+            {"name": "price", "selector": "p.price_color", "type": "text"},
+            {"name": "availability", "selector": "p.availability", "type": "text"},
+        ],
+    }
+    strategy = JsonCssExtractionStrategy(schema=schema)
+    config = CrawlerRunConfig(extraction_strategy=strategy)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        result = await crawler.arun(
+            url="https://books.toscrape.com", config=config
+        )
+        assert result.success
+        extracted = json.loads(result.extracted_content)
+        assert len(extracted) >= 10, f"Expected at least 10 books, got {len(extracted)}"
+        for book in extracted:
+            assert book.get("title"), "Book title should not be empty"
+            assert book.get("price"), "Book price should not be empty"
diff --git a/tests/regression/test_reg_utils.py b/tests/regression/test_reg_utils.py
new file mode 100644
index 0000000..dfc63c4
--- /dev/null
+++ b/tests/regression/test_reg_utils.py
@@ -0,0 +1,500 @@
+"""
+Regression tests for Crawl4AI utility functions.
+
+Covers extract_xml_data, URL normalization, CacheContext/CacheMode,
+sanitize_input_encode, content hashing, and image scoring.
+"""
+
+import pytest
+
+from crawl4ai.utils import (
+    extract_xml_data,
+    extract_xml_data_legacy,
+    normalize_url,
+    normalize_url_for_deep_crawl,
+    efficient_normalize_url_for_deep_crawl,
+    sanitize_input_encode,
+    generate_content_hash,
+)
+from crawl4ai.cache_context import CacheContext, CacheMode
+
+
+# ===================================================================
+# extract_xml_data
+# ===================================================================
+
+class TestExtractXmlData:
+    """Verify extract_xml_data correctly parses tag content from strings."""
+
+    def test_basic_single_tag(self):
+        """Basic extraction of a single tag should return its content."""
+        result = extract_xml_data(["blocks"], "hello")
+        assert result["blocks"] == "hello"
+
+    def test_multiple_tags(self):
+        """Extracting multiple tags should return both."""
+        result = extract_xml_data(["a", "b"], "12")
+        assert result["a"] == "1"
+        assert result["b"] == "2"
+
+    def test_longest_match(self):
+        """When multiple occurrences exist, return the longest content."""
+        text = "short some text this is the longer content here"
+        result = extract_xml_data(["blocks"], text)
+        assert result["blocks"] == "this is the longer content here"
+
+    def test_nested_mention_bug_fix_1183(self):
+        """Fix for #1183: nested mention of tag name should not confuse extraction.
+
+        When  block mentions  in prose, the extraction should
+        return the actual  content, not the prose mention.
+        """
+        text = (
+            "The user wants me to extract  data from the page."
+            "real extracted data"
+        )
+        result = extract_xml_data(["blocks"], text)
+        assert result["blocks"] == "real extracted data"
+
+    def test_missing_tag_returns_empty(self):
+        """Missing tag should return empty string."""
+        result = extract_xml_data(["missing"], "content")
+        assert result["missing"] == ""
+
+    def test_empty_content(self):
+        """Empty tag content should return empty string."""
+        result = extract_xml_data(["blocks"], "")
+        assert result["blocks"] == ""
+
+    def test_multiline_content(self):
+        """Content spanning multiple lines should be extracted."""
+        text = "\nline 1\nline 2\nline 3\n"
+        result = extract_xml_data(["blocks"], text)
+        assert "line 1" in result["blocks"]
+        assert "line 2" in result["blocks"]
+        assert "line 3" in result["blocks"]
+
+    def test_special_chars_in_content(self):
+        """JSON-like content with special characters should be preserved."""
+        text = '{"key": "value", "num": 42}'
+        result = extract_xml_data(["blocks"], text)
+        assert '"key": "value"' in result["blocks"]
+        assert '"num": 42' in result["blocks"]
+
+    def test_content_with_angle_brackets(self):
+        """Content with HTML-like angle brackets should work if not same tag."""
+        text = "some bold text"
+        result = extract_xml_data(["blocks"], text)
+        assert "bold" in result["blocks"]
+
+    def test_multiple_tags_some_missing(self):
+        """Mixed present and missing tags should return values for present, empty for missing."""
+        result = extract_xml_data(["found", "missing"], "yes")
+        assert result["found"] == "yes"
+        assert result["missing"] == ""
+
+    def test_whitespace_stripped(self):
+        """Content should be stripped of leading/trailing whitespace."""
+        result = extract_xml_data(["blocks"], "  trimmed  ")
+        assert result["blocks"] == "trimmed"
+
+
+class TestExtractXmlDataLegacy:
+    """Verify the legacy extract_xml_data function works."""
+
+    def test_basic_extraction(self):
+        """Legacy function should extract basic tag content."""
+        result = extract_xml_data_legacy(["blocks"], "hello")
+        assert result["blocks"] == "hello"
+
+    def test_missing_tag(self):
+        """Legacy function should return empty string for missing tags."""
+        result = extract_xml_data_legacy(["missing"], "no tags here")
+        assert result["missing"] == ""
+
+
+# ===================================================================
+# URL normalization
+# ===================================================================
+
+class TestNormalizeUrl:
+    """Verify normalize_url handles various URL edge cases."""
+
+    def test_trailing_slash_preserved(self):
+        """Trailing slash should be preserved (fix for #1520)."""
+        result = normalize_url("/foo/bar/", "http://x.com")
+        assert result.endswith("/foo/bar/")
+
+    def test_no_trailing_slash_not_added(self):
+        """URL without trailing slash should NOT have one added."""
+        result = normalize_url("/foo/bar", "http://x.com")
+        assert result.endswith("/foo/bar")
+        assert not result.endswith("/foo/bar/")
+
+    def test_root_path(self):
+        """Root path '/' should be preserved."""
+        result = normalize_url("/", "http://x.com")
+        assert result == "http://x.com/"
+
+    def test_query_param_case_preservation(self):
+        """Query parameter values should NOT be lowercased (fix for #1489).
+
+        cHash=AbCd must remain as-is, not become chash=abcd.
+        """
+        result = normalize_url("/page?cHash=AbCd", "http://x.com")
+        assert "cHash=AbCd" in result
+
+    def test_tracking_params_removed(self):
+        """Common tracking parameters should be removed."""
+        result = normalize_url(
+            "/page?utm_source=google&utm_medium=cpc&real_param=keep",
+            "http://x.com",
+        )
+        assert "utm_source" not in result
+        assert "utm_medium" not in result
+        assert "real_param=keep" in result
+
+    def test_fbclid_removed(self):
+        """fbclid tracking parameter should be removed."""
+        result = normalize_url("/page?fbclid=abc123&keep=yes", "http://x.com")
+        assert "fbclid" not in result
+        assert "keep=yes" in result
+
+    def test_gclid_removed(self):
+        """gclid tracking parameter should be removed."""
+        result = normalize_url("/page?gclid=xyz&keep=yes", "http://x.com")
+        assert "gclid" not in result
+        assert "keep=yes" in result
+
+    def test_tracking_removal_case_insensitive(self):
+        """Tracking parameter removal should be case-insensitive."""
+        # The normalize_url uses k.lower() for comparison
+        result = normalize_url("/page?UTM_SOURCE=test&data=1", "http://x.com")
+        # UTM_SOURCE (uppercase) should be removed since comparison is case-insensitive
+        assert "data=1" in result
+
+    def test_query_sorting(self):
+        """Query parameters should be sorted alphabetically."""
+        result = normalize_url("/page?z=1&a=2&m=3", "http://x.com")
+        # Parameters should appear in alphabetical order
+        idx_a = result.index("a=2")
+        idx_m = result.index("m=3")
+        idx_z = result.index("z=1")
+        assert idx_a < idx_m < idx_z
+
+    def test_fragment_removed_by_default(self):
+        """Fragment (#section) should be removed by default."""
+        result = normalize_url("/page#section", "http://x.com")
+        assert "#section" not in result
+
+    def test_fragment_kept_when_requested(self):
+        """Fragment should be kept when keep_fragment=True."""
+        result = normalize_url("/page#section", "http://x.com", keep_fragment=True)
+        assert "#section" in result
+
+    def test_relative_url_resolution(self):
+        """Relative URLs should be resolved against base_url."""
+        result = normalize_url("page2", "http://x.com/dir/page1")
+        assert result == "http://x.com/dir/page2"
+
+    def test_empty_href_returns_none(self):
+        """Empty href should return None."""
+        result = normalize_url("", "http://x.com")
+        assert result is None
+
+    def test_none_href_returns_none(self):
+        """None href should return None."""
+        result = normalize_url(None, "http://x.com")
+        assert result is None
+
+    def test_hostname_lowercased(self):
+        """Hostname should be lowercased for consistency."""
+        result = normalize_url("/page", "http://EXAMPLE.COM/path")
+        assert "example.com" in result
+
+    def test_no_query_params_still_works(self):
+        """URL without query params should normalize without issue."""
+        result = normalize_url("/simple/path", "http://x.com")
+        assert "http://x.com/simple/path" == result
+
+
+class TestNormalizeUrlForDeepCrawl:
+    """Verify normalize_url_for_deep_crawl handles deep crawl edge cases."""
+
+    def test_trailing_slash_preserved(self):
+        """Trailing slash should be preserved in deep crawl normalization."""
+        result = normalize_url_for_deep_crawl("/foo/bar/", "http://x.com")
+        assert result is not None
+        assert result.endswith("/foo/bar/")
+
+    def test_empty_href_returns_none(self):
+        """Empty href should return None."""
+        result = normalize_url_for_deep_crawl("", "http://x.com")
+        assert result is None
+
+    def test_none_href_returns_none(self):
+        """None href should return None."""
+        result = normalize_url_for_deep_crawl(None, "http://x.com")
+        assert result is None
+
+    def test_fragment_removed(self):
+        """Fragment should be removed in deep crawl normalization."""
+        result = normalize_url_for_deep_crawl("/page#anchor", "http://x.com")
+        assert "#anchor" not in result
+
+    def test_tracking_params_removed(self):
+        """utm_source and similar tracking params should be removed."""
+        result = normalize_url_for_deep_crawl(
+            "/page?utm_source=google&keep=yes", "http://x.com"
+        )
+        assert "utm_source" not in result
+        assert "keep=yes" in result
+
+    def test_hostname_lowercased(self):
+        """Hostname should be lowercased."""
+        result = normalize_url_for_deep_crawl("/page", "http://EXAMPLE.COM")
+        assert "example.com" in result
+
+
+class TestEfficientNormalizeUrlForDeepCrawl:
+    """Verify efficient_normalize_url_for_deep_crawl caching and correctness."""
+
+    def test_trailing_slash_preserved(self):
+        """Trailing slash should be preserved."""
+        result = efficient_normalize_url_for_deep_crawl("/foo/bar/", "http://x.com")
+        assert result is not None
+        assert result.endswith("/foo/bar/")
+
+    def test_cached_results_consistent(self):
+        """Calling twice with same args should return same result (cached)."""
+        result1 = efficient_normalize_url_for_deep_crawl("/cached", "http://x.com")
+        result2 = efficient_normalize_url_for_deep_crawl("/cached", "http://x.com")
+        assert result1 == result2
+
+    def test_empty_href_returns_none(self):
+        """Empty href should return None."""
+        result = efficient_normalize_url_for_deep_crawl("", "http://x.com")
+        assert result is None
+
+    def test_none_href_returns_none(self):
+        """None href should return None."""
+        result = efficient_normalize_url_for_deep_crawl(None, "http://x.com")
+        assert result is None
+
+    def test_fragment_removed(self):
+        """Fragment should be removed."""
+        result = efficient_normalize_url_for_deep_crawl("/page#top", "http://x.com")
+        assert "#top" not in result
+
+    def test_hostname_lowercased(self):
+        """Hostname should be lowercased."""
+        result = efficient_normalize_url_for_deep_crawl("/path", "http://UPPER.COM")
+        assert "upper.com" in result
+
+    def test_relative_url_resolution(self):
+        """Relative URLs should be resolved correctly."""
+        result = efficient_normalize_url_for_deep_crawl(
+            "child", "http://x.com/parent/"
+        )
+        assert result == "http://x.com/parent/child"
+
+
+# ===================================================================
+# CacheContext / CacheMode
+# ===================================================================
+
+class TestCacheMode:
+    """Verify CacheContext behavior for each CacheMode."""
+
+    def test_enabled_reads_and_writes(self):
+        """CacheMode.ENABLED should allow both reads and writes."""
+        ctx = CacheContext("http://example.com", CacheMode.ENABLED)
+        assert ctx.should_read() is True
+        assert ctx.should_write() is True
+
+    def test_disabled_no_reads_no_writes(self):
+        """CacheMode.DISABLED should block both reads and writes."""
+        ctx = CacheContext("http://example.com", CacheMode.DISABLED)
+        assert ctx.should_read() is False
+        assert ctx.should_write() is False
+
+    def test_bypass_no_reads_but_writes(self):
+        """CacheMode.BYPASS should skip reads but allow writes."""
+        ctx = CacheContext("http://example.com", CacheMode.BYPASS)
+        assert ctx.should_read() is False
+        assert ctx.should_write() is False
+
+    def test_read_only_reads_no_writes(self):
+        """CacheMode.READ_ONLY should allow reads, block writes."""
+        ctx = CacheContext("http://example.com", CacheMode.READ_ONLY)
+        assert ctx.should_read() is True
+        assert ctx.should_write() is False
+
+    def test_write_only_no_reads_but_writes(self):
+        """CacheMode.WRITE_ONLY should block reads, allow writes."""
+        ctx = CacheContext("http://example.com", CacheMode.WRITE_ONLY)
+        assert ctx.should_read() is False
+        assert ctx.should_write() is True
+
+    def test_raw_url_not_cacheable(self):
+        """raw:// URLs should not be cacheable regardless of mode."""
+        ctx = CacheContext("raw://test", CacheMode.ENABLED)
+        assert ctx.should_read() is False
+        assert ctx.should_write() is False
+
+    def test_raw_url_is_raw_html(self):
+        """raw:// URLs should be flagged as raw HTML."""
+        ctx = CacheContext("raw://test", CacheMode.ENABLED)
+        assert ctx.is_raw_html is True
+        assert ctx.is_web_url is False
+
+    def test_http_url_is_cacheable(self):
+        """http:// URLs should be cacheable."""
+        ctx = CacheContext("http://example.com", CacheMode.ENABLED)
+        assert ctx.is_cacheable is True
+        assert ctx.is_web_url is True
+
+    def test_https_url_is_cacheable(self):
+        """https:// URLs should be cacheable."""
+        ctx = CacheContext("https://example.com", CacheMode.ENABLED)
+        assert ctx.is_cacheable is True
+
+    def test_file_url_is_cacheable(self):
+        """file:// URLs should be cacheable."""
+        ctx = CacheContext("file:///tmp/test.html", CacheMode.ENABLED)
+        assert ctx.is_cacheable is True
+        assert ctx.is_local_file is True
+
+    def test_always_bypass_overrides_everything(self):
+        """always_bypass=True should force read=False, write=False."""
+        ctx = CacheContext("http://example.com", CacheMode.ENABLED, always_bypass=True)
+        assert ctx.should_read() is False
+        assert ctx.should_write() is False
+
+    def test_display_url_for_web(self):
+        """Display URL for web URLs should be the URL itself."""
+        ctx = CacheContext("http://example.com", CacheMode.ENABLED)
+        assert ctx.display_url == "http://example.com"
+
+    def test_display_url_for_raw(self):
+        """Display URL for raw HTML should be 'Raw HTML'."""
+        ctx = CacheContext("raw://something", CacheMode.ENABLED)
+        assert ctx.display_url == "Raw HTML"
+
+
+# ===================================================================
+# sanitize_input_encode
+# ===================================================================
+
+class TestSanitizeInputEncode:
+    """Verify sanitize_input_encode handles encoding edge cases."""
+
+    def test_normal_utf8_passthrough(self):
+        """Normal UTF-8 text should pass through unchanged."""
+        text = "Hello, world! This is normal text."
+        assert sanitize_input_encode(text) == text
+
+    def test_unicode_text_preserved(self):
+        """Unicode characters should be preserved."""
+        text = "Caf\u00e9 na\u00efve r\u00e9sum\u00e9"
+        assert sanitize_input_encode(text) == text
+
+    def test_empty_string_returns_empty(self):
+        """Empty string should return empty string."""
+        assert sanitize_input_encode("") == ""
+
+    def test_ascii_text_passthrough(self):
+        """Pure ASCII text should pass through."""
+        text = "Simple ASCII text 123"
+        assert sanitize_input_encode(text) == text
+
+    def test_cjk_characters_preserved(self):
+        """CJK characters should be preserved."""
+        text = "\u4f60\u597d\u4e16\u754c"
+        assert sanitize_input_encode(text) == text
+
+    def test_emoji_preserved(self):
+        """Emoji characters should be preserved in UTF-8."""
+        text = "Hello \U0001f600 World"
+        result = sanitize_input_encode(text)
+        assert "Hello" in result
+        assert "World" in result
+
+
+# ===================================================================
+# Content hashing
+# ===================================================================
+
+class TestGenerateContentHash:
+    """Verify generate_content_hash produces consistent results."""
+
+    def test_same_content_same_hash(self):
+        """Same content should produce same hash."""
+        hash1 = generate_content_hash("hello world")
+        hash2 = generate_content_hash("hello world")
+        assert hash1 == hash2
+
+    def test_different_content_different_hash(self):
+        """Different content should produce different hashes."""
+        hash1 = generate_content_hash("hello world")
+        hash2 = generate_content_hash("goodbye world")
+        assert hash1 != hash2
+
+    def test_empty_content_valid_hash(self):
+        """Empty content should produce a valid hash (not an error)."""
+        h = generate_content_hash("")
+        assert isinstance(h, str)
+        assert len(h) > 0
+
+    def test_hash_is_hex_string(self):
+        """Hash should be a hexadecimal string."""
+        h = generate_content_hash("test content")
+        assert all(c in "0123456789abcdef" for c in h)
+
+    def test_hash_deterministic_across_calls(self):
+        """Hash should be deterministic, not random."""
+        content = "The quick brown fox jumps over the lazy dog"
+        hashes = [generate_content_hash(content) for _ in range(10)]
+        assert len(set(hashes)) == 1
+
+    def test_whitespace_sensitive(self):
+        """Hash should be sensitive to whitespace differences."""
+        h1 = generate_content_hash("hello world")
+        h2 = generate_content_hash("hello  world")
+        assert h1 != h2
+
+    def test_case_sensitive(self):
+        """Hash should be case-sensitive."""
+        h1 = generate_content_hash("Hello")
+        h2 = generate_content_hash("hello")
+        assert h1 != h2
+
+    def test_long_content(self):
+        """Long content should hash without error."""
+        content = "x" * 1_000_000
+        h = generate_content_hash(content)
+        assert isinstance(h, str)
+        assert len(h) > 0
+
+
+# ===================================================================
+# Image scoring (import-guarded)
+# ===================================================================
+
+class TestImageScoring:
+    """Test image scoring logic if available.
+
+    score_image_for_usefulness is a nested function, so we test
+    the concept indirectly by checking that the module loads and
+    the scoring constants exist.
+    """
+
+    def test_image_score_threshold_exists(self):
+        """IMAGE_SCORE_THRESHOLD config constant should exist."""
+        from crawl4ai.config import IMAGE_SCORE_THRESHOLD
+        assert isinstance(IMAGE_SCORE_THRESHOLD, (int, float))
+
+    def test_image_description_threshold_exists(self):
+        """IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD should exist."""
+        from crawl4ai.config import IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD
+        assert isinstance(IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, (int, float))
diff --git a/tests/releases/test_release_0.6.4.py b/tests/releases/test_release_0.6.4.py
new file mode 100644
index 0000000..06bd8f9
--- /dev/null
+++ b/tests/releases/test_release_0.6.4.py
@@ -0,0 +1,151 @@
+import pytest
+import asyncio
+import time
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig, CacheMode
+
+
+@pytest.mark.asyncio
+async def test_wait_for_timeout_separate_from_page_timeout():
+    """Test that wait_for has its own timeout separate from page_timeout"""
+    browser_config = BrowserConfig(headless=True)
+    
+    # Test with short wait_for_timeout but longer page_timeout
+    config = CrawlerRunConfig(
+        wait_for="css:.nonexistent-element",
+        wait_for_timeout=2000,  # 2 seconds
+        page_timeout=10000,     # 10 seconds
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        start_time = time.time()
+        result = await crawler.arun("https://example.com", config=config)
+        elapsed = time.time() - start_time
+        
+        # Should timeout after ~2 seconds (wait_for_timeout), not 10 seconds
+        assert elapsed < 5, f"Expected timeout around 2s, but took {elapsed:.2f}s"
+        assert result.success, "Crawl should still succeed even if wait_for times out"
+
+
+@pytest.mark.asyncio
+async def test_wait_for_timeout_with_existing_element():
+    """Test that wait_for_timeout works correctly when element exists"""
+    browser_config = BrowserConfig(headless=True)
+    
+    config = CrawlerRunConfig(
+        wait_for="css:body",  # This should exist quickly
+        wait_for_timeout=5000,
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        start_time = time.time()
+        result = await crawler.arun("https://example.com", config=config)
+        elapsed = time.time() - start_time
+        
+        # Should complete quickly since body element exists
+        assert elapsed < 3, f"Expected quick completion, but took {elapsed:.2f}s"
+        assert result.success
+        assert "
+    
+    
+        Test GA Integration
+        
+        
+        
+    
+    
+        
      Test Page
      
+        
      Testing Google Analytics integration
      
+    
+    
+    """
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(f"raw://{html_content}", config=config)
+        
+        assert result.success
+        # Check that GA scripts are preserved in the HTML
+        assert "googletagmanager.com/gtag/js" in result.html
+        assert "dataLayer" in result.html
+        assert "gtag('config'" in result.html
+
+
+@pytest.mark.asyncio
+async def test_mkdocs_no_duplicate_gtag():
+    """Test that there are no duplicate gtag.js entries in documentation"""
+    browser_config = BrowserConfig(headless=True)
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    
+    # Simulate MkDocs-like HTML structure
+    html_content = """
+    
+    
+    
+        Crawl4AI Documentation
+        
+        
+    
+    
+        
      Crawl4AI Documentation
      
+        
      Welcome to the documentation
      
+    
+    
+    """
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(f"raw://{html_content}", config=config)
+        
+        assert result.success
+        # Count occurrences of gtag.js to ensure no duplicates
+        gtag_count = result.html.count("googletagmanager.com/gtag/js")
+        assert gtag_count <= 1, f"Found {gtag_count} gtag.js scripts, expected at most 1"
+        
+        # Ensure the analytics functionality is still there
+        if gtag_count == 1:
+            assert "dataLayer" in result.html
+            assert "gtag('config'" in result.html
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
\ No newline at end of file
diff --git a/tests/releases/test_release_0.7.0.py b/tests/releases/test_release_0.7.0.py
new file mode 100644
index 0000000..a0885a7
--- /dev/null
+++ b/tests/releases/test_release_0.7.0.py
@@ -0,0 +1,317 @@
+#!/usr/bin/env python3
+
+import asyncio
+import pytest
+import os
+import json
+import tempfile
+from pathlib import Path
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategy, LLMConfig
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.async_url_seeder import AsyncUrlSeeder
+from crawl4ai.utils import RobotsParser
+
+
+class TestCrawl4AIv070:
+    """Test suite for Crawl4AI v0.7.0 changes"""
+    
+    @pytest.mark.asyncio
+    async def test_raw_url_parsing(self):
+        """Test raw:// URL parsing logic fix"""
+        html_content = "
      Test Content
      This is a test paragraph.
      "
+        
+        async with AsyncWebCrawler() as crawler:
+            # Test raw:// prefix
+            result1 = await crawler.arun(f"raw://{html_content}")
+            assert result1.success
+            assert "Test Content" in result1.markdown
+            
+            # Test raw: prefix
+            result2 = await crawler.arun(f"raw:{html_content}")
+            assert result2.success
+            assert "Test Content" in result2.markdown
+    
+    @pytest.mark.asyncio
+    async def test_max_pages_limit_batch_processing(self):
+        """Test max_pages limit is respected during batch processing"""
+        urls = [
+            "https://httpbin.org/html",
+            "https://httpbin.org/json",
+            "https://httpbin.org/xml"
+        ]
+        
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            max_pages=2
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            results = await crawler.arun_many(urls, config=config)
+            # Should only process 2 pages due to max_pages limit
+            successful_results = [r for r in results if r.success]
+            assert len(successful_results) <= 2
+    
+    @pytest.mark.asyncio
+    async def test_navigation_abort_handling(self):
+        """Test handling of navigation aborts during file downloads"""
+        async with AsyncWebCrawler() as crawler:
+            # Test with a URL that might cause navigation issues
+            result = await crawler.arun(
+                "https://httpbin.org/status/404",
+                config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+            )
+            # Should not crash even with navigation issues
+            assert result is not None
+    
+    @pytest.mark.asyncio
+    async def test_screenshot_capture_fix(self):
+        """Test screenshot capture improvements"""
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            screenshot=True
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            assert result.success
+            assert result.screenshot is not None
+            assert len(result.screenshot) > 0
+    
+    @pytest.mark.asyncio
+    async def test_redirect_status_codes(self):
+        """Test that real redirect status codes are surfaced"""
+        async with AsyncWebCrawler() as crawler:
+            # Test with a redirect URL
+            result = await crawler.arun(
+                "https://httpbin.org/redirect/1",
+                config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+            )
+            assert result.success
+            # Should have redirect information
+            assert result.status_code in [200, 301, 302, 303, 307, 308]
+    
+    @pytest.mark.asyncio
+    async def test_local_file_processing(self):
+        """Test local file processing with captured_console initialization"""
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.html', delete=False) as f:
+            f.write("
      Local File Test
      ")
+            temp_file = f.name
+        
+        try:
+            async with AsyncWebCrawler() as crawler:
+                result = await crawler.arun(f"file://{temp_file}")
+                assert result.success
+                assert "Local File Test" in result.markdown
+        finally:
+            os.unlink(temp_file)
+    
+    @pytest.mark.asyncio
+    async def test_robots_txt_wildcard_support(self):
+        """Test robots.txt wildcard rules support"""
+        parser = RobotsParser()
+        
+        # Test wildcard patterns
+        robots_content = "User-agent: *\nDisallow: /admin/*\nDisallow: *.pdf"
+        
+        # This should work without throwing exceptions
+        assert parser is not None
+    
+    @pytest.mark.asyncio
+    async def test_exclude_external_images(self):
+        """Test exclude_external_images flag"""
+        html_with_images = '''
+        
+            Local
+            External
+        
+        '''
+        
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            exclude_external_images=True
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(f"raw://{html_with_images}", config=config)
+            assert result.success
+            # External images should be excluded
+            assert "external.com" not in result.cleaned_html
+    
+    @pytest.mark.asyncio
+    async def test_llm_extraction_strategy_fix(self):
+        """Test LLM extraction strategy choices error fix"""
+        if not os.getenv("OPENAI_API_KEY"):
+            pytest.skip("OpenAI API key not available")
+        
+        llm_config = LLMConfig(
+            provider="openai/gpt-4o-mini",
+            api_token=os.getenv("OPENAI_API_KEY")
+        )
+        
+        strategy = LLMExtractionStrategy(
+            llm_config=llm_config,
+            instruction="Extract the main heading",
+            extraction_type="block"
+        )
+        
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            extraction_strategy=strategy
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            assert result.success
+            # Should not throw 'str' object has no attribute 'choices' error
+            assert result.extracted_content is not None
+    
+    @pytest.mark.asyncio
+    async def test_wait_for_timeout(self):
+        """Test separate timeout for wait_for condition"""
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            wait_for="css:non-existent-element",
+            wait_for_timeout=1000  # 1 second timeout
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            # Should timeout gracefully and still return result
+            assert result is not None
+    
+    @pytest.mark.asyncio
+    async def test_bm25_content_filter_language_parameter(self):
+        """Test BM25 filter with language parameter for stemming"""
+        content_filter = BM25ContentFilter(
+            user_query="test content",
+            language="english",
+            use_stemming=True
+        )
+        
+        markdown_generator = DefaultMarkdownGenerator(
+            content_filter=content_filter
+        )
+        
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=markdown_generator
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            assert result.success
+            assert result.markdown is not None
+    
+    @pytest.mark.asyncio
+    async def test_url_normalization(self):
+        """Test URL normalization for invalid schemes and trailing slashes"""
+        async with AsyncWebCrawler() as crawler:
+            # Test with trailing slash
+            result = await crawler.arun(
+                "https://httpbin.org/html/",
+                config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+            )
+            assert result.success
+    
+    @pytest.mark.asyncio
+    async def test_max_scroll_steps(self):
+        """Test max_scroll_steps parameter for full page scanning"""
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            scan_full_page=True,
+            max_scroll_steps=3
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            assert result.success
+    
+    @pytest.mark.asyncio
+    async def test_async_url_seeder(self):
+        """Test AsyncUrlSeeder functionality"""
+        seeder = AsyncUrlSeeder(
+            base_url="https://httpbin.org",
+            max_depth=1,
+            max_urls=5
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            urls = await seeder.seed(crawler)
+            assert isinstance(urls, list)
+            assert len(urls) <= 5
+    
+    @pytest.mark.asyncio
+    async def test_pdf_processing_timeout(self):
+        """Test PDF processing with timeout"""
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            pdf=True,
+            pdf_timeout=10000  # 10 seconds
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            assert result.success
+            # PDF might be None for HTML pages, but should not hang
+            assert result.pdf is not None or result.pdf is None
+    
+    @pytest.mark.asyncio
+    async def test_browser_session_management(self):
+        """Test improved browser session management"""
+        browser_config = BrowserConfig(
+            headless=True,
+            use_persistent_context=True
+        )
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(
+                "https://httpbin.org/html",
+                config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+            )
+            assert result.success
+    
+    @pytest.mark.asyncio
+    async def test_memory_management(self):
+        """Test memory management features"""
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            memory_threshold_percent=80.0,
+            check_interval=1.0,
+            memory_wait_timeout=600  # 10 minutes default
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            assert result.success
+    
+    @pytest.mark.asyncio
+    async def test_virtual_scroll_support(self):
+        """Test virtual scroll support for modern web scraping"""
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            scan_full_page=True,
+            virtual_scroll=True
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            assert result.success
+    
+    @pytest.mark.asyncio
+    async def test_adaptive_crawling(self):
+        """Test adaptive crawling feature"""
+        config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            adaptive_crawling=True
+        )
+        
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun("https://httpbin.org/html", config=config)
+            assert result.success
+
+
+if __name__ == "__main__":
+    # Run the tests
+    pytest.main([__file__, "-v"])
diff --git a/tests/test_arun_many.py b/tests/test_arun_many.py
new file mode 100644
index 0000000..2a315a2
--- /dev/null
+++ b/tests/test_arun_many.py
@@ -0,0 +1,42 @@
+"""
+Test example for multiple crawler configs feature
+"""
+import asyncio
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+
+async def test_run_many():
+    default_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        # scraping_strategy=PDFContentScrapingStrategy()
+    )
+    
+    test_urls = [
+        # "https://blog.python.org/",  # Blog URL  
+        "https://www.python.org/",  # Generic HTTPS page
+        "https://www.kidocode.com/",  # Generic HTTPS page
+        "https://www.example.com/",  # Generic HTTPS page
+        # "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
+    ]
+    
+    async with AsyncWebCrawler() as crawler:
+        # Single config - traditional usage still works
+        print("Test 1: Single config (backwards compatible)")
+        result = await crawler.arun_many(
+            urls=test_urls[:2],
+            config=default_config
+        )
+        print(f"Crawled {len(result)} URLs with single config\n")
+        for item in result:
+            print(f"  {item.url} -> {item.status_code}")
+        
+
+if __name__ == "__main__":
+    asyncio.run(test_run_many())
diff --git a/tests/test_async_logger_stderr.py b/tests/test_async_logger_stderr.py
new file mode 100644
index 0000000..ca68796
--- /dev/null
+++ b/tests/test_async_logger_stderr.py
@@ -0,0 +1,156 @@
+"""
+Tests for issue #1968: AsyncLogger must write to stderr by default so that
+stdout-based transports (e.g. MCP stdio) are not corrupted.
+"""
+
+import importlib.util
+import io
+import sys
+from pathlib import Path
+
+import pytest
+from rich.console import Console
+
+# Load async_logger directly without triggering the full crawl4ai __init__
+# (which pulls in many optional deps like aiofiles, OpenSSL, playwright …).
+_spec = importlib.util.spec_from_file_location(
+    "crawl4ai.async_logger",
+    Path(__file__).parent.parent / "crawl4ai" / "async_logger.py",
+)
+_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_mod)  # type: ignore[union-attr]
+
+AsyncLogger = _mod.AsyncLogger
+LogLevel = _mod.LogLevel
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _capture_consoles():
+    """Return StringIO objects for stdout and stderr plus a no-color Console
+    for each, suitable for injecting into AsyncLogger in tests."""
+    out = io.StringIO()
+    err = io.StringIO()
+    stdout_console = Console(file=out, no_color=True, highlight=False)
+    stderr_console = Console(file=err, no_color=True, highlight=False)
+    return out, err, stdout_console, stderr_console
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestAsyncLoggerDefaultsToStderr:
+    """The default AsyncLogger must NOT write to stdout."""
+
+    def test_default_console_writes_to_stderr_not_stdout(self, capsys):
+        """Logging to the default AsyncLogger must not pollute stdout."""
+        logger = AsyncLogger(verbose=True)
+
+        logger.info("hello from logger", tag="TEST")
+        logger.error("error message", tag="TEST")
+        logger.warning("warning message", tag="TEST")
+
+        captured = capsys.readouterr()
+        # stdout must remain pristine (no log lines mixed in)
+        assert captured.out == "", (
+            "AsyncLogger wrote to stdout — this breaks MCP stdio transport!\n"
+            f"stdout content: {captured.out!r}"
+        )
+        # stderr should contain the output
+        assert "hello from logger" in captured.err, (
+            "Expected log output on stderr but found nothing.\n"
+            f"stderr content: {captured.err!r}"
+        )
+
+    def test_default_console_is_stderr(self):
+        """The internal console.file should be sys.stderr (or equivalent)."""
+        logger = AsyncLogger(verbose=True)
+        # Rich Console with stderr=True writes to sys.stderr
+        assert logger.console.file is sys.stderr, (
+            f"Expected logger.console.file to be sys.stderr, "
+            f"got {logger.console.file!r}"
+        )
+
+
+class TestAsyncLoggerCustomConsole:
+    """Callers can inject a custom Console (e.g. stdout for non-MCP use)."""
+
+    def test_custom_console_is_respected(self):
+        """When a custom Console is passed, it must be used verbatim."""
+        buf = io.StringIO()
+        custom = Console(file=buf, no_color=True, highlight=False)
+        logger = AsyncLogger(verbose=True, console=custom)
+
+        logger.info("custom target", tag="TEST")
+
+        output = buf.getvalue()
+        assert "custom target" in output, (
+            f"Expected log output in custom console, got: {output!r}"
+        )
+
+    def test_stdout_console_can_be_injected(self, capsys):
+        """Passing Console(file=sys.stdout) restores legacy behaviour."""
+        stdout_console = Console(file=sys.stdout, no_color=True, highlight=False)
+        logger = AsyncLogger(verbose=True, console=stdout_console)
+
+        logger.info("going to stdout", tag="TEST")
+
+        captured = capsys.readouterr()
+        assert "going to stdout" in captured.out
+
+
+class TestAsyncLoggerVerboseFalse:
+    """verbose=False must suppress console output regardless of the target stream."""
+
+    def test_no_output_when_verbose_false(self, capsys):
+        logger = AsyncLogger(verbose=False)
+        logger.info("silent message", tag="TEST")
+        logger.error("silent error", tag="TEST")
+
+        captured = capsys.readouterr()
+        assert captured.out == ""
+        assert captured.err == ""
+
+
+class TestAsyncLoggerFileLogging:
+    """File logging path should still work after the stderr change."""
+
+    def test_file_logging_still_works(self, tmp_path):
+        log_file = tmp_path / "test.log"
+        logger = AsyncLogger(log_file=str(log_file), verbose=False)
+        logger.info("file log entry", tag="FILE")
+
+        content = log_file.read_text(encoding="utf-8")
+        assert "file log entry" in content
+
+
+class TestMCPScenario:
+    """Simulates what MCP stdio transport sees on stdout."""
+
+    def test_stdout_clean_across_all_log_levels(self, capsys):
+        """Simulate MCP usage: crawl + log, stdout must only have JSON."""
+        logger = AsyncLogger(verbose=True)
+        fake_json_response = '{"jsonrpc": "2.0", "result": "ok", "id": 1}'
+
+        # Simulate interleaved logging (as would happen during a crawl)
+        logger.info("Crawling started", tag="CRAWL")
+        logger.warning("Slow response", tag="CRAWL")
+        logger.success("Done", tag="CRAWL")
+
+        # MCP server would write JSON to stdout
+        print(fake_json_response)
+
+        captured = capsys.readouterr()
+
+        # stdout must contain ONLY the JSON line
+        assert captured.out.strip() == fake_json_response, (
+            "Stdout was polluted by logger output!\n"
+            f"stdout: {captured.out!r}"
+        )
+        # All log output should be on stderr
+        assert "Crawling started" in captured.err
diff --git a/tests/test_bug_batch_1622_1786_1796.py b/tests/test_bug_batch_1622_1786_1796.py
new file mode 100644
index 0000000..1442509
--- /dev/null
+++ b/tests/test_bug_batch_1622_1786_1796.py
@@ -0,0 +1,314 @@
+"""
+Tests for bug fix batch: PR #1622, #1786, #1796
+
+- #1622: _resolve_head should verify redirect targets are alive
+- #1786: arun_many should wire mean_delay/max_range into dispatcher
+- #1796: process_iframes should use DOMParser instead of innerHTML
+"""
+import asyncio
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+import httpx
+
+
+# ── PR #1622: Redirect target verification in _resolve_head ──────────────
+
+
+@pytest.fixture
+def seeder():
+    """Create an AsyncUrlSeeder with a mocked HTTP client."""
+    from crawl4ai.async_url_seeder import AsyncUrlSeeder
+
+    s = AsyncUrlSeeder()
+    s.client = AsyncMock(spec=httpx.AsyncClient)
+    return s
+
+
+def _make_response(status_code, headers=None, url="https://example.com"):
+    """Helper to create a mock httpx Response."""
+    resp = MagicMock(spec=httpx.Response)
+    resp.status_code = status_code
+    resp.headers = headers or {}
+    resp.url = httpx.URL(url)
+    return resp
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_direct_2xx(seeder):
+    """Direct 2xx hit should return the URL."""
+    seeder.client.head = AsyncMock(
+        return_value=_make_response(200, url="https://example.com/page")
+    )
+    result = await seeder._resolve_head("https://example.com/page")
+    assert result == "https://example.com/page"
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_redirect_to_live_target(seeder):
+    """3xx redirect to a live target should return the target URL."""
+    redirect_resp = _make_response(
+        301, headers={"location": "https://example.com/new-page"}
+    )
+    target_resp = _make_response(200, url="https://example.com/new-page")
+
+    seeder.client.head = AsyncMock(side_effect=[redirect_resp, target_resp])
+    result = await seeder._resolve_head("https://example.com/old-page")
+    assert result == "https://example.com/new-page"
+    assert seeder.client.head.call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_redirect_to_dead_target(seeder):
+    """3xx redirect to a dead (non-2xx) target should return None."""
+    redirect_resp = _make_response(
+        302, headers={"location": "https://example.com/dead"}
+    )
+    target_resp = _make_response(404, url="https://example.com/dead")
+
+    seeder.client.head = AsyncMock(side_effect=[redirect_resp, target_resp])
+    result = await seeder._resolve_head("https://example.com/old")
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_redirect_target_timeout(seeder):
+    """3xx redirect where target times out should return None."""
+    redirect_resp = _make_response(
+        301, headers={"location": "https://example.com/slow"}
+    )
+
+    seeder.client.head = AsyncMock(
+        side_effect=[redirect_resp, httpx.TimeoutException("timeout")]
+    )
+    result = await seeder._resolve_head("https://example.com/old")
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_self_redirect(seeder):
+    """Self-redirect (Location == original URL) should return None."""
+    redirect_resp = _make_response(
+        301, headers={"location": "https://example.com/loop"}
+    )
+    seeder.client.head = AsyncMock(return_value=redirect_resp)
+    result = await seeder._resolve_head("https://example.com/loop")
+    assert result is None
+    # Should NOT make a second request for self-redirect
+    assert seeder.client.head.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_relative_redirect(seeder):
+    """Relative Location header should be resolved against original URL."""
+    redirect_resp = _make_response(301, headers={"location": "/new-path"})
+    target_resp = _make_response(200, url="https://example.com/new-path")
+
+    seeder.client.head = AsyncMock(side_effect=[redirect_resp, target_resp])
+    result = await seeder._resolve_head("https://example.com/old-path")
+    assert result == "https://example.com/new-path"
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_4xx_returns_none(seeder):
+    """4xx status should return None."""
+    seeder.client.head = AsyncMock(return_value=_make_response(404))
+    result = await seeder._resolve_head("https://example.com/missing")
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_network_error(seeder):
+    """Network error should return None (not raise)."""
+    seeder.client.head = AsyncMock(
+        side_effect=httpx.ConnectError("connection refused")
+    )
+    result = await seeder._resolve_head("https://example.com/down")
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_resolve_head_no_location_header(seeder):
+    """3xx without Location header should return None."""
+    seeder.client.head = AsyncMock(return_value=_make_response(301, headers={}))
+    result = await seeder._resolve_head("https://example.com/no-loc")
+    assert result is None
+
+
+# ── PR #1786: mean_delay / max_range wired into dispatcher ───────────────
+
+
+class TestDispatcherWiring:
+    """Test that arun_many wires CrawlerRunConfig delay params into the dispatcher."""
+
+    def test_default_config_values(self):
+        """CrawlerRunConfig should have mean_delay=0.1 and max_range=0.3 by default."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+
+        cfg = CrawlerRunConfig()
+        assert cfg.mean_delay == 0.1
+        assert cfg.max_range == 0.3
+
+    def test_custom_config_values(self):
+        """CrawlerRunConfig should accept custom mean_delay and max_range."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+
+        cfg = CrawlerRunConfig(mean_delay=2.0, max_range=1.0)
+        assert cfg.mean_delay == 2.0
+        assert cfg.max_range == 1.0
+
+    @pytest.mark.asyncio
+    async def test_dispatcher_uses_config_delays(self):
+        """When no dispatcher is provided, arun_many should create one using config delays."""
+        from crawl4ai.async_webcrawler import AsyncWebCrawler
+        from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig
+        from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher, RateLimiter
+
+        captured_dispatcher = {}
+
+        original_init = MemoryAdaptiveDispatcher.__init__
+
+        def patched_init(self, *args, **kwargs):
+            original_init(self, *args, **kwargs)
+            captured_dispatcher["rate_limiter"] = self.rate_limiter
+
+        with patch.object(MemoryAdaptiveDispatcher, "__init__", patched_init):
+            # We just need to trigger the dispatcher creation path
+            # We'll patch run_urls to avoid actually crawling
+            with patch.object(
+                MemoryAdaptiveDispatcher, "run_urls", new_callable=AsyncMock
+            ) as mock_run:
+                mock_run.return_value = []
+
+                crawler = AsyncWebCrawler(config=BrowserConfig())
+                crawler.ready = True  # skip browser setup
+                crawler.crawler_strategy = MagicMock()
+
+                cfg = CrawlerRunConfig(mean_delay=2.0, max_range=1.5)
+                try:
+                    await crawler.arun_many(urls=["https://example.com"], config=cfg)
+                except Exception:
+                    pass  # may fail on result processing, that's fine
+
+                rl = captured_dispatcher.get("rate_limiter")
+                assert rl is not None, "Dispatcher should have been created"
+                assert rl.base_delay == (2.0, 3.5), (
+                    f"Expected base_delay=(2.0, 3.5), got {rl.base_delay}"
+                )
+
+    @pytest.mark.asyncio
+    async def test_dispatcher_uses_first_config_from_list(self):
+        """When config is a list, should use the first config's delays."""
+        from crawl4ai.async_webcrawler import AsyncWebCrawler
+        from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig
+        from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher, RateLimiter
+
+        captured_dispatcher = {}
+
+        original_init = MemoryAdaptiveDispatcher.__init__
+
+        def patched_init(self, *args, **kwargs):
+            original_init(self, *args, **kwargs)
+            captured_dispatcher["rate_limiter"] = self.rate_limiter
+
+        with patch.object(MemoryAdaptiveDispatcher, "__init__", patched_init):
+            with patch.object(
+                MemoryAdaptiveDispatcher, "run_urls", new_callable=AsyncMock
+            ) as mock_run:
+                mock_run.return_value = []
+
+                crawler = AsyncWebCrawler(config=BrowserConfig())
+                crawler.ready = True
+                crawler.crawler_strategy = MagicMock()
+
+                cfg1 = CrawlerRunConfig(mean_delay=5.0, max_range=2.0)
+                cfg2 = CrawlerRunConfig(mean_delay=0.5, max_range=0.1)
+                try:
+                    await crawler.arun_many(
+                        urls=["https://a.com", "https://b.com"],
+                        config=[cfg1, cfg2],
+                    )
+                except Exception:
+                    pass
+
+                rl = captured_dispatcher.get("rate_limiter")
+                assert rl is not None
+                assert rl.base_delay == (5.0, 7.0), (
+                    f"Expected base_delay=(5.0, 7.0) from first config, got {rl.base_delay}"
+                )
+
+    @pytest.mark.asyncio
+    async def test_explicit_dispatcher_not_overridden(self):
+        """When user provides their own dispatcher, config delays should NOT override it."""
+        from crawl4ai.async_webcrawler import AsyncWebCrawler
+        from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig
+        from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher, RateLimiter
+
+        custom_rl = RateLimiter(base_delay=(10.0, 20.0))
+        custom_dispatcher = MemoryAdaptiveDispatcher(rate_limiter=custom_rl)
+
+        with patch.object(
+            MemoryAdaptiveDispatcher, "run_urls", new_callable=AsyncMock
+        ) as mock_run:
+            mock_run.return_value = []
+
+            crawler = AsyncWebCrawler(config=BrowserConfig())
+            crawler.ready = True
+            crawler.crawler_strategy = MagicMock()
+
+            cfg = CrawlerRunConfig(mean_delay=0.5, max_range=0.1)
+            try:
+                await crawler.arun_many(
+                    urls=["https://example.com"],
+                    config=cfg,
+                    dispatcher=custom_dispatcher,
+                )
+            except Exception:
+                pass
+
+            # Custom dispatcher's rate limiter should be untouched
+            assert custom_rl.base_delay == (10.0, 20.0)
+
+
+# ── PR #1796: DOMParser in process_iframes ───────────────────────────────
+
+
+class TestProcessIframesDOMParser:
+    """Verify that process_iframes uses DOMParser instead of innerHTML."""
+
+    def test_source_code_uses_domparser(self):
+        """The process_iframes method should use DOMParser, not innerHTML for injection."""
+        import inspect
+        from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+        source = inspect.getsource(AsyncPlaywrightCrawlerStrategy.process_iframes)
+
+        # Should contain DOMParser usage
+        assert "DOMParser" in source, "process_iframes should use DOMParser"
+        assert "parseFromString" in source, "process_iframes should call parseFromString"
+        assert "doc.body.firstChild" in source, (
+            "process_iframes should move nodes from parsed doc"
+        )
+
+        # The old innerHTML assignment pattern should NOT be present
+        # Note: document.body.innerHTML for READING iframe content is fine
+        # The dangerous pattern is div.innerHTML = `{_iframe}` for WRITING
+        lines = source.split("\n")
+        for line in lines:
+            stripped = line.strip()
+            # Only flag div.innerHTML assignment, not reading from document.body
+            if "div.innerHTML" in stripped and "=" in stripped:
+                pytest.fail(
+                    f"Found unsafe innerHTML assignment: {stripped}"
+                )
+
+    def test_js_snippet_structure(self):
+        """The JS snippet should properly create DOM nodes from parsed HTML."""
+        import inspect
+        from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+        source = inspect.getsource(AsyncPlaywrightCrawlerStrategy.process_iframes)
+
+        # Verify the correct pattern: parse then move child nodes
+        assert "new DOMParser()" in source
+        assert "'text/html'" in source
+        assert "appendChild" in source
diff --git a/tests/test_cdp_changes.py b/tests/test_cdp_changes.py
new file mode 100644
index 0000000..a9b8233
--- /dev/null
+++ b/tests/test_cdp_changes.py
@@ -0,0 +1,506 @@
+"""
+Tests for CDP connection caching and configurable close delay.
+
+Two test suites:
+  1. Regression — default behavior unchanged, basic CDP still works
+  2. Stress    — race conditions, parallel crawlers, locking, cache correctness
+
+All tests are real (no mocks). Requires a running Chrome on port 9222:
+    chrome --headless=new --no-sandbox --remote-debugging-port=9222
+"""
+
+import asyncio
+import time
+import pytest
+
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.browser_manager import _CDPConnectionCache, BrowserManager
+
+CDP_URL = "http://localhost:9222"
+TEST_URL = "https://example.com"
+
+
+# ---------------------------------------------------------------------------
+#  Helpers
+# ---------------------------------------------------------------------------
+
+async def _quick_crawl(browser_cfg: BrowserConfig, url: str = TEST_URL) -> bool:
+    """Run a single crawl, return True if the page loaded successfully."""
+    run_cfg = CrawlerRunConfig(
+        wait_until="domcontentloaded",
+        page_timeout=15000,
+        verbose=False,
+    )
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        result = await crawler.arun(url=url, config=run_cfg)
+    return result.success and result.status_code == 200
+
+
+# ===========================================================================
+#  SUITE 1 — REGRESSION (default behaviour, no new flags)
+# ===========================================================================
+
+class TestRegression:
+    """Verify nothing is broken when the new parameters keep their defaults."""
+
+    @pytest.mark.asyncio
+    async def test_default_cdp_crawl(self):
+        """Basic CDP crawl with default settings still works."""
+        cfg = BrowserConfig(cdp_url=CDP_URL, headless=True)
+        assert await _quick_crawl(cfg)
+
+    @pytest.mark.asyncio
+    async def test_default_params_values(self):
+        """BrowserConfig defaults are backward-compatible."""
+        cfg = BrowserConfig()
+        assert cfg.cdp_close_delay == 1.0
+        assert cfg.cache_cdp_connection is False
+
+    @pytest.mark.asyncio
+    async def test_cdp_cleanup_on_close_still_works(self):
+        """cdp_cleanup_on_close=True path (existing feature) still works."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cdp_cleanup_on_close=True,
+        )
+        assert await _quick_crawl(cfg)
+
+    @pytest.mark.asyncio
+    async def test_sequential_crawls_default(self):
+        """Two sequential crawls with default settings both succeed."""
+        cfg = BrowserConfig(cdp_url=CDP_URL, headless=True)
+        assert await _quick_crawl(cfg)
+        assert await _quick_crawl(cfg)
+
+    @pytest.mark.asyncio
+    async def test_cdp_close_delay_default_timing(self):
+        """Default close delay is ~1s (not 0, not much more)."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cdp_cleanup_on_close=True,
+        )
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=15000,
+            verbose=False,
+        )
+        t0 = time.monotonic()
+        async with AsyncWebCrawler(config=cfg) as crawler:
+            await crawler.arun(url=TEST_URL, config=run_cfg)
+        elapsed = time.monotonic() - t0
+        # The 1s sleep must be present (at least ~0.9s of close overhead)
+        assert elapsed >= 0.9, f"Close was too fast ({elapsed:.2f}s), sleep may be missing"
+
+
+# ===========================================================================
+#  SUITE 2 — STRESS (cache, parallelism, race conditions, locking)
+# ===========================================================================
+
+class TestStress:
+    """Hammer the CDP cache and configurable delay under pressure."""
+
+    # -- cdp_close_delay -------------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_close_delay_zero_skips_sleep(self):
+        """cdp_close_delay=0 should make close noticeably faster."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cdp_cleanup_on_close=True,
+            cdp_close_delay=0,
+        )
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=15000,
+            verbose=False,
+        )
+        t0 = time.monotonic()
+        async with AsyncWebCrawler(config=cfg) as crawler:
+            await crawler.arun(url=TEST_URL, config=run_cfg)
+        elapsed = time.monotonic() - t0
+        # Without the 1s sleep the whole thing should be well under 1s of close overhead
+        assert elapsed < 5.0, f"Close too slow with delay=0 ({elapsed:.2f}s)"
+
+    @pytest.mark.asyncio
+    async def test_close_delay_custom_value(self):
+        """A custom delay value is respected."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cdp_cleanup_on_close=True,
+            cdp_close_delay=0.2,
+        )
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=15000,
+            verbose=False,
+        )
+        t0 = time.monotonic()
+        async with AsyncWebCrawler(config=cfg) as crawler:
+            await crawler.arun(url=TEST_URL, config=run_cfg)
+        elapsed = time.monotonic() - t0
+        # Should include at least the 0.2s delay
+        assert elapsed >= 0.2
+
+    # -- cache_cdp_connection: basic -----------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_cache_basic_crawl(self):
+        """A single crawl with caching enabled works."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+        assert await _quick_crawl(cfg)
+        # Clean up cache after test
+        await _CDPConnectionCache.close_all()
+
+    @pytest.mark.asyncio
+    async def test_cache_sequential_reuse(self):
+        """Two sequential crawlers reuse the same cached connection."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+
+        # First crawl — creates the cache entry
+        async with AsyncWebCrawler(config=cfg) as crawler:
+            r1 = await crawler.arun(
+                url=TEST_URL,
+                config=CrawlerRunConfig(wait_until="domcontentloaded", page_timeout=15000, verbose=False),
+            )
+        assert r1.success
+
+        # Cache should still have an entry (ref went 1 -> 0, so it closed).
+        # But if we acquire again immediately it should create a fresh one.
+        # The key thing: this must not crash or hang.
+
+        async with AsyncWebCrawler(config=cfg) as crawler:
+            r2 = await crawler.arun(
+                url=TEST_URL,
+                config=CrawlerRunConfig(wait_until="domcontentloaded", page_timeout=15000, verbose=False),
+            )
+        assert r2.success
+
+        await _CDPConnectionCache.close_all()
+
+    @pytest.mark.asyncio
+    async def test_cache_overlapping_instances(self):
+        """Two crawlers alive at the same time share the cache (ref_count=2)."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=15000,
+            verbose=False,
+        )
+
+        crawler1 = AsyncWebCrawler(config=cfg)
+        await crawler1.start()
+
+        crawler2 = AsyncWebCrawler(config=cfg)
+        await crawler2.start()
+
+        # Both should work concurrently
+        r1, r2 = await asyncio.gather(
+            crawler1.arun(url=TEST_URL, config=run_cfg),
+            crawler2.arun(url=TEST_URL, config=run_cfg),
+        )
+        assert r1.success
+        assert r2.success
+
+        # Close one — connection must survive for the other
+        await crawler1.close()
+
+        # Second crawler should still work after first closed
+        r3 = await crawler2.arun(url=TEST_URL, config=run_cfg)
+        assert r3.success
+
+        await crawler2.close()
+        await _CDPConnectionCache.close_all()
+
+    # -- cache: ref counting -------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_cache_ref_count_lifecycle(self):
+        """Verify ref count goes up and comes back down correctly."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+
+        c1 = AsyncWebCrawler(config=cfg)
+        await c1.start()
+        # Cache should have ref_count=1
+        assert CDP_URL in _CDPConnectionCache._cache
+        _, _, count = _CDPConnectionCache._cache[CDP_URL]
+        assert count == 1
+
+        c2 = AsyncWebCrawler(config=cfg)
+        await c2.start()
+        _, _, count = _CDPConnectionCache._cache[CDP_URL]
+        assert count == 2
+
+        await c1.close()
+        _, _, count = _CDPConnectionCache._cache[CDP_URL]
+        assert count == 1
+
+        await c2.close()
+        # Last reference released — entry should be gone
+        assert CDP_URL not in _CDPConnectionCache._cache
+
+    # -- cache: speed benefit ------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_cache_faster_than_uncached(self):
+        """Cached sequential crawls should be faster than uncached."""
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=15000,
+            verbose=False,
+        )
+
+        # Uncached: two sequential crawls (each does full playwright start/stop)
+        cfg_no_cache = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cdp_cleanup_on_close=True,
+            cdp_close_delay=0.5,
+        )
+        t0 = time.monotonic()
+        for _ in range(2):
+            async with AsyncWebCrawler(config=cfg_no_cache) as crawler:
+                await crawler.arun(url=TEST_URL, config=run_cfg)
+        uncached_time = time.monotonic() - t0
+
+        # Cached: two sequential crawls (share playwright/CDP)
+        cfg_cache = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+        t0 = time.monotonic()
+        for _ in range(2):
+            async with AsyncWebCrawler(config=cfg_cache) as crawler:
+                await crawler.arun(url=TEST_URL, config=run_cfg)
+        cached_time = time.monotonic() - t0
+
+        await _CDPConnectionCache.close_all()
+
+        # Cached should be faster (the uncached has 2x 0.5s delay alone)
+        assert cached_time < uncached_time, (
+            f"Cached ({cached_time:.2f}s) was not faster than uncached ({uncached_time:.2f}s)"
+        )
+
+    # -- parallel stress -----------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_parallel_cached_crawlers(self):
+        """Launch 5 crawlers in parallel, all sharing the cache."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+            create_isolated_context=True,
+        )
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=20000,
+            verbose=False,
+        )
+
+        async def crawl_one(idx: int):
+            async with AsyncWebCrawler(config=cfg) as crawler:
+                result = await crawler.arun(url=TEST_URL, config=run_cfg)
+            return idx, result.success
+
+        results = await asyncio.gather(*[crawl_one(i) for i in range(5)])
+        for idx, success in results:
+            assert success, f"Crawler {idx} failed"
+
+        await _CDPConnectionCache.close_all()
+
+    @pytest.mark.asyncio
+    async def test_parallel_mixed_cached_uncached(self):
+        """Mix of cached and uncached crawlers running in parallel.
+
+        Uses create_isolated_context=True because parallel crawlers sharing
+        a single default context will cause navigation conflicts — this is
+        the expected pattern for concurrent CDP access.
+        """
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=20000,
+            verbose=False,
+        )
+
+        async def crawl_cached():
+            cfg = BrowserConfig(
+                cdp_url=CDP_URL, headless=True,
+                cache_cdp_connection=True, create_isolated_context=True,
+            )
+            async with AsyncWebCrawler(config=cfg) as crawler:
+                r = await crawler.arun(url=TEST_URL, config=run_cfg)
+            return "cached", r.success
+
+        async def crawl_uncached():
+            cfg = BrowserConfig(
+                cdp_url=CDP_URL, headless=True,
+                create_isolated_context=True,
+            )
+            async with AsyncWebCrawler(config=cfg) as crawler:
+                r = await crawler.arun(url=TEST_URL, config=run_cfg)
+            return "uncached", r.success
+
+        tasks = [crawl_cached(), crawl_uncached(), crawl_cached(), crawl_uncached()]
+        results = await asyncio.gather(*tasks)
+        for label, success in results:
+            assert success, f"{label} crawler failed"
+
+        await _CDPConnectionCache.close_all()
+
+    @pytest.mark.asyncio
+    async def test_rapid_open_close_cache(self):
+        """Rapidly open and close 10 crawlers sequentially — no leaks/hangs."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=15000,
+            verbose=False,
+        )
+
+        for i in range(10):
+            async with AsyncWebCrawler(config=cfg) as crawler:
+                result = await crawler.arun(url=TEST_URL, config=run_cfg)
+                assert result.success, f"Iteration {i} failed"
+
+        await _CDPConnectionCache.close_all()
+
+    @pytest.mark.asyncio
+    async def test_cache_close_all_idempotent(self):
+        """Calling close_all() multiple times doesn't crash."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+        async with AsyncWebCrawler(config=cfg) as crawler:
+            r = await crawler.arun(
+                url=TEST_URL,
+                config=CrawlerRunConfig(wait_until="domcontentloaded", page_timeout=15000, verbose=False),
+            )
+            assert r.success
+
+        await _CDPConnectionCache.close_all()
+        await _CDPConnectionCache.close_all()  # second call must not raise
+        await _CDPConnectionCache.close_all()  # third call must not raise
+
+    @pytest.mark.asyncio
+    async def test_stale_connection_recovery(self):
+        """If the cached browser disconnects, next acquire recovers."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+        run_cfg = CrawlerRunConfig(
+            wait_until="domcontentloaded",
+            page_timeout=15000,
+            verbose=False,
+        )
+
+        # Build up a cache entry
+        c1 = AsyncWebCrawler(config=cfg)
+        await c1.start()
+
+        # Forcibly disconnect the cached browser to simulate staleness
+        if CDP_URL in _CDPConnectionCache._cache:
+            _, browser, _ = _CDPConnectionCache._cache[CDP_URL]
+            try:
+                await browser.close()
+            except Exception:
+                pass
+
+        await c1.close()
+
+        # Next crawler should detect stale and create a fresh connection
+        async with AsyncWebCrawler(config=cfg) as crawler:
+            result = await crawler.arun(url=TEST_URL, config=run_cfg)
+            assert result.success
+
+        await _CDPConnectionCache.close_all()
+
+    @pytest.mark.asyncio
+    async def test_parallel_start_race(self):
+        """Multiple crawlers calling start() simultaneously — lock prevents races."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+
+        crawlers = [AsyncWebCrawler(config=cfg) for _ in range(5)]
+
+        # Start all at once — this hammers _CDPConnectionCache.acquire() concurrently
+        await asyncio.gather(*[c.start() for c in crawlers])
+
+        # All should have the same browser reference
+        browsers = set()
+        for c in crawlers:
+            bm = c.crawler_strategy.browser_manager
+            browsers.add(id(bm.browser))
+
+        # With caching, they should all share the same browser object
+        assert len(browsers) == 1, f"Expected 1 shared browser, got {len(browsers)}"
+
+        # Ref count should be 5
+        _, _, count = _CDPConnectionCache._cache[CDP_URL]
+        assert count == 5
+
+        # Close all
+        await asyncio.gather(*[c.close() for c in crawlers])
+
+        # Cache should be empty
+        assert CDP_URL not in _CDPConnectionCache._cache
+
+    @pytest.mark.asyncio
+    async def test_parallel_close_race(self):
+        """Multiple crawlers closing simultaneously — no double-free."""
+        cfg = BrowserConfig(
+            cdp_url=CDP_URL,
+            headless=True,
+            cache_cdp_connection=True,
+        )
+
+        crawlers = [AsyncWebCrawler(config=cfg) for _ in range(5)]
+        await asyncio.gather(*[c.start() for c in crawlers])
+
+        # Close all at once — hammers _CDPConnectionCache.release() concurrently
+        await asyncio.gather(*[c.close() for c in crawlers])
+
+        # Cache must be clean
+        assert CDP_URL not in _CDPConnectionCache._cache
+
+        # Must still work after everything is closed
+        async with AsyncWebCrawler(config=cfg) as crawler:
+            r = await crawler.arun(
+                url=TEST_URL,
+                config=CrawlerRunConfig(wait_until="domcontentloaded", page_timeout=15000, verbose=False),
+            )
+            assert r.success
+
+        await _CDPConnectionCache.close_all()
diff --git a/tests/test_cli_docs.py b/tests/test_cli_docs.py
new file mode 100644
index 0000000..6941f20
--- /dev/null
+++ b/tests/test_cli_docs.py
@@ -0,0 +1,44 @@
+import asyncio
+from crawl4ai.docs_manager import DocsManager
+from click.testing import CliRunner
+from crawl4ai.cli import cli
+
+
+def test_cli():
+    """Test all CLI commands"""
+    runner = CliRunner()
+
+    print("\n1. Testing docs update...")
+    # Use sync version for testing
+    docs_manager = DocsManager()
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(docs_manager.fetch_docs())
+
+    # print("\n2. Testing listing...")
+    # result = runner.invoke(cli, ['docs', 'list'])
+    # print(f"Status: {'✅' if result.exit_code == 0 else '❌'}")
+    # print(result.output)
+
+    # print("\n2. Testing index building...")
+    # result = runner.invoke(cli, ['docs', 'index'])
+    # print(f"Status: {'✅' if result.exit_code == 0 else '❌'}")
+    # print(f"Output: {result.output}")
+
+    # print("\n3. Testing search...")
+    # result = runner.invoke(cli, ['docs', 'search', 'how to use crawler', '--build-index'])
+    # print(f"Status: {'✅' if result.exit_code == 0 else '❌'}")
+    # print(f"First 200 chars: {result.output[:200]}...")
+
+    # print("\n4. Testing combine with sections...")
+    # result = runner.invoke(cli, ['docs', 'combine', 'chunking_strategies', 'extraction_strategies', '--mode', 'extended'])
+    # print(f"Status: {'✅' if result.exit_code == 0 else '❌'}")
+    # print(f"First 200 chars: {result.output[:200]}...")
+
+    print("\n5. Testing combine all sections...")
+    result = runner.invoke(cli, ["docs", "combine", "--mode", "condensed"])
+    print(f"Status: {'✅' if result.exit_code == 0 else '❌'}")
+    print(f"First 200 chars: {result.output[:200]}...")
+
+
+if __name__ == "__main__":
+    test_cli()
diff --git a/tests/test_cloud_bugs_batch.py b/tests/test_cloud_bugs_batch.py
new file mode 100644
index 0000000..de61609
--- /dev/null
+++ b/tests/test_cloud_bugs_batch.py
@@ -0,0 +1,479 @@
+"""
+Comprehensive test suite for cloud-reported bug fixes:
+  - Bug 1: Anti-bot false positives on browser-rendered JSON
+  - Bug 2: URLPatternFilter PREFIX match using full URL instead of path
+  - Bug 3: PDFContentScrapingStrategy not in ALLOWED_DESERIALIZE_TYPES
+
+Tests include: unit, edge case, adversarial, and regression checks.
+"""
+
+import sys
+import os
+import re
+import json
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from crawl4ai.antibot_detector import is_blocked, _looks_like_data, _structural_integrity_check
+from crawl4ai.deep_crawling.filters import URLPatternFilter
+from crawl4ai.async_configs import ALLOWED_DESERIALIZE_TYPES, to_serializable_dict, from_serializable_dict
+
+PASS = 0
+FAIL = 0
+
+def check(name, actual, expected, detail=""):
+    global PASS, FAIL
+    ok = actual == expected
+    if ok:
+        PASS += 1
+    else:
+        FAIL += 1
+        print(f"  FAIL: {name}")
+        print(f"         got: {actual!r}")
+        print(f"         expected: {expected!r}")
+        if detail:
+            print(f"         detail: {detail}")
+
+
+# =====================================================================
+# BUG 1: Anti-bot false positives on browser-rendered JSON
+# =====================================================================
+print("\n" + "=" * 70)
+print("BUG 1: Anti-bot false positives on browser-rendered JSON")
+print("=" * 70)
+
+# --- 1A: _looks_like_data() unit tests ---
+print("\n--- _looks_like_data() ---")
+
+check("raw JSON object", _looks_like_data('{"origin": "1.2.3.4"}'), True)
+check("raw JSON array", _looks_like_data('[1, 2, 3]'), True)
+check("raw XML", _looks_like_data(''), True)
+check("empty string", _looks_like_data(''), False)
+check("whitespace only", _looks_like_data('   \n  '), False)
+check("normal HTML page", _looks_like_data('
      Hello
      '), False)
+check("", _looks_like_data(''), False)
+check("", _looks_like_data(''), False)
+
+# Browser-wrapped JSON (the core bug)
+check("browser-wrapped JSON object",
+    _looks_like_data('
      {"origin": "1.2.3.4"}
      '),
+    True)
+
+check("browser-wrapped JSON array",
+    _looks_like_data('
      [{"id": 1}, {"id": 2}]
      '),
+    True)
+
+check("browser-wrapped JSON (uppercase HTML)",
+    _looks_like_data('
      {"key": "val"}
      '),
+    True)
+
+check("browser-wrapped JSON (DOCTYPE)",
+    _looks_like_data('
      {"x": 1}
      '),
+    True)
+
+check("browser-wrapped JSON with whitespace before pre",
+    _looks_like_data('  \n  
      {"x": 1}
      '),
+    True)
+
+# Should NOT detect as data — normal HTML with 
      +check("HTML page with code block (not JSON in pre)",
+    _looks_like_data('
      Tutorial
      def hello():\n    print("hi")
      '),
+    False)
+
+check("HTML with 
       but text content, not JSON",
+    _looks_like_data('
      This is just preformatted text, not JSON.
      '),
+    False)
+
+# --- 1B: is_blocked() integration tests for browser-wrapped JSON ---
+print("\n--- is_blocked() with browser-rendered JSON ---")
+
+# httpbin.org /ip response (tiny — ~90 bytes HTML)
+httpbin_ip = '
      {"origin": "203.0.113.42"}
      '
+blocked, reason = is_blocked(200, httpbin_ip)
+check("httpbin /ip (200, small browser-wrapped JSON)", blocked, False, reason)
+
+# httpbin.org /anything response (medium)
+httpbin_anything = '
      {"args": {}, "data": "", "files": {}, "form": {}, "headers": {"Accept": "*/*", "Host": "httpbin.org", "User-Agent": "Mozilla/5.0"}, "json": null, "method": "GET", "origin": "203.0.113.42", "url": "https://httpbin.org/anything"}
      '
+blocked, reason = is_blocked(200, httpbin_anything)
+check("httpbin /anything (200, medium browser-wrapped JSON)", blocked, False, reason)
+
+# httpbin.org /delay/N response
+httpbin_delay = '
      {"args": {}, "data": "", "headers": {"Host": "httpbin.org"}, "origin": "1.2.3.4", "url": "https://httpbin.org/delay/2"}
      '
+blocked, reason = is_blocked(200, httpbin_delay)
+check("httpbin /delay/2 (200, browser-wrapped JSON)", blocked, False, reason)
+
+# httpbin.org /headers response
+httpbin_headers = '
      {"headers": {"Accept": "text/html", "Accept-Encoding": "gzip", "Host": "httpbin.org", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)", "X-Forwarded-For": "1.2.3.4", "X-Forwarded-Proto": "https"}}
      '
+blocked, reason = is_blocked(200, httpbin_headers)
+check("httpbin /headers (200, browser-wrapped JSON)", blocked, False, reason)
+
+# Browser-wrapped JSON array
+json_array_page = '
      [{"id": 1, "name": "foo"}, {"id": 2, "name": "bar"}]
      '
+blocked, reason = is_blocked(200, json_array_page)
+check("browser-wrapped JSON array (200)", blocked, False, reason)
+
+# --- 1C: Ensure real block pages still detected ---
+print("\n--- Regression: real block pages still detected ---")
+
+# Empty shell (should still be blocked)
+blocked, reason = is_blocked(200, '')
+check("empty shell (200, no content)", blocked, True, reason)
+
+# Anti-bot redirect page
+blocked, reason = is_blocked(200, '')
+check("script-only redirect (200)", blocked, True, reason)
+
+# Small page with no content elements (not JSON)
+blocked, reason = is_blocked(200, '
      x
      ')
+check("tiny div-only page (200)", blocked, True, reason)
+
+# 403 with browser-wrapped JSON should NOT be blocked (it's data)
+blocked, reason = is_blocked(403, '{"error": "forbidden", "code": 403}')
+check("403 raw JSON (data response)", blocked, False, reason)
+
+# 403 with HTML should still be blocked
+blocked, reason = is_blocked(403, '
      Forbidden
      ')
+check("403 HTML page (blocked)", blocked, True, reason)
+
+# --- 1D: 
       now counts as content element ---
+print("\n--- 
       as content element ---")
+
+# Page with only 
       (code block) should not be flagged as "no content elements"
+html_with_pre = '
      function hello() {\n  console.log("world");\n}\n\nThis is a code example that demonstrates JavaScript functions. It shows how to define and use basic functions with console output for debugging purposes.
      '
+blocked, reason = is_blocked(200, html_with_pre)
+check("page with 
       code block (200)", blocked, False, reason)
+
+# Page with 
       containing log output
+html_pre_logs = '
      2024-01-15 10:30:45 INFO  Starting server on port 8080\n2024-01-15 10:30:46 INFO  Database connected successfully\n2024-01-15 10:30:47 INFO  Application ready to accept connections on all interfaces
      '
+blocked, reason = is_blocked(200, html_pre_logs)
+check("page with 
       log output (200)", blocked, False, reason)
+
+# --- 1E: Adversarial: attacker tries to bypass detection using 
       ---
+print("\n--- Adversarial: 
       shouldn't defeat real block detection ---")
+
+# Tier 1 pattern in page with 
       should still be detected
+blocked, reason = is_blocked(403, '
      Reference #18.2d351ab8.1557333295.a4e16ab
      ')
+check("Akamai ref in 
       (403)", blocked, True, reason)
+
+blocked, reason = is_blocked(200, '
      window._pxAppId = "PX123";
      ')
+check("PerimeterX in 
       (200)", blocked, True, reason)
+
+# Empty 
       should still trigger
+blocked, reason = is_blocked(200, '
      ')
+check("empty 
       (200, minimal text)", blocked, True, reason)
+
+# 
       with whitespace only
+blocked, reason = is_blocked(200, '
         
      ')
+check("
       with only whitespace (200)", blocked, True, reason)
+
+
+# =====================================================================
+# BUG 2: URLPatternFilter PREFIX match
+# =====================================================================
+print("\n" + "=" * 70)
+print("BUG 2: URLPatternFilter PREFIX match")
+print("=" * 70)
+
+# --- 2A: Path-only prefix patterns (the original bug) ---
+print("\n--- Path-only prefix patterns ---")
+
+f = URLPatternFilter(patterns=["/docs/*"])
+check("/docs/* matches /docs/page1", f.apply("https://example.com/docs/page1"), True)
+check("/docs/* matches /docs/", f.apply("https://example.com/docs/"), True)
+check("/docs/* matches /docs/sub/page", f.apply("https://example.com/docs/sub/page"), True)
+f2 = URLPatternFilter(patterns=["/docs/*"])  # fresh filter (lru_cache)
+check("/docs/* no match /api/docs", f2.apply("https://example.com/api/docs"), False)
+f3 = URLPatternFilter(patterns=["/docs/*"])
+check("/docs/* no match /other", f3.apply("https://example.com/other"), False)
+
+# --- 2B: Full-URL prefix patterns (must still work) ---
+print("\n--- Full-URL prefix patterns ---")
+
+f4 = URLPatternFilter(patterns=["https://example.com/blog/*"])
+check("full URL prefix matches", f4.apply("https://example.com/blog/post-1"), True)
+check("full URL prefix matches subpath", f4.apply("https://example.com/blog/2024/post-1"), True)
+f5 = URLPatternFilter(patterns=["https://example.com/blog/*"])
+check("full URL prefix no match different domain", f5.apply("https://other.com/blog/post-1"), False)
+f6 = URLPatternFilter(patterns=["https://example.com/blog/*"])
+check("full URL prefix no match blogxx", f6.apply("https://example.com/blogxx/post-1"), False)
+
+# --- 2C: Path prefix with query strings ---
+print("\n--- Prefix with query strings ---")
+
+f7 = URLPatternFilter(patterns=["/api/*"])
+check("/api/* matches /api/v1", f7.apply("https://example.com/api/v1"), True)
+check("/api/* matches /api/v1?key=123", f7.apply("https://example.com/api/v1?key=123"), True)
+f8 = URLPatternFilter(patterns=["/api/*"])
+check("/api/* no match /apiv2/", f8.apply("https://example.com/apiv2/"), False)
+
+# --- 2D: Suffix patterns still work ---
+print("\n--- Suffix patterns ---")
+
+f9 = URLPatternFilter(patterns=["*.pdf"])
+check("*.pdf matches report.pdf", f9.apply("https://example.com/report.pdf"), True)
+check("*.pdf matches nested pdf", f9.apply("https://example.com/docs/report.pdf"), True)
+f10 = URLPatternFilter(patterns=["*.pdf"])
+check("*.pdf no match .html", f10.apply("https://example.com/page.html"), False)
+
+# --- 2E: Reverse mode ---
+print("\n--- Reverse mode ---")
+
+f11 = URLPatternFilter(patterns=["/private/*"], reverse=True)
+check("reverse: /private/* excludes /private/page", f11.apply("https://example.com/private/page"), False)
+f12 = URLPatternFilter(patterns=["/private/*"], reverse=True)
+check("reverse: /private/* allows /public/page", f12.apply("https://example.com/public/page"), True)
+
+# --- 2F: Adversarial URL patterns ---
+print("\n--- Adversarial URL edge cases ---")
+
+f13 = URLPatternFilter(patterns=["/docs/*"])
+check("prefix with port in URL", f13.apply("https://example.com:8080/docs/page"), True)
+f14 = URLPatternFilter(patterns=["/docs/*"])
+check("prefix with auth in URL", f14.apply("https://user:pass@example.com/docs/page"), True)
+f15 = URLPatternFilter(patterns=["/docs/*"])
+check("prefix with fragment", f15.apply("https://example.com/docs#section"), True)
+
+# URL-encoded path
+f16 = URLPatternFilter(patterns=["/docs/*"])
+check("prefix with encoded path", f16.apply("https://example.com/docs/my%20page"), True)
+
+# Multiple prefix patterns
+f17 = URLPatternFilter(patterns=["/docs/*", "/api/*"])
+check("multi-prefix: /docs/ matches", f17.apply("https://example.com/docs/page"), True)
+check("multi-prefix: /api/ matches", f17.apply("https://example.com/api/v1"), True)
+f18 = URLPatternFilter(patterns=["/docs/*", "/api/*"])
+check("multi-prefix: /other/ no match", f18.apply("https://example.com/other/page"), False)
+
+# --- 2G: Complex (PATH) patterns still work ---
+print("\n--- Complex glob patterns ---")
+
+f19 = URLPatternFilter(patterns=["*/docs/*/guide"])
+check("glob */docs/*/guide matches", f19.apply("https://example.com/docs/v2/guide"), True)
+f20 = URLPatternFilter(patterns=["*/docs/*/guide"])
+check("glob */docs/*/guide no match", f20.apply("https://example.com/docs/v2/tutorial"), False)
+
+# --- 2H: Domain patterns still work ---
+print("\n--- Domain patterns ---")
+
+# Note: *.example.com (without ://) is classified as SUFFIX, not DOMAIN.
+# Use http://*.example.com for domain matching.
+f21 = URLPatternFilter(patterns=["http://*.example.com/*"])
+check("domain http://*.example.com/* matches sub.example.com", f21.apply("http://sub.example.com/page"), True)
+f22 = URLPatternFilter(patterns=["http://*.example.com/*"])
+check("domain http://*.example.com/* no match other.com", f22.apply("http://other.com/page"), False)
+
+# --- 2I: Regex patterns still work ---
+print("\n--- Regex patterns ---")
+
+f23 = URLPatternFilter(patterns=[r"^https://example\.com/v\d+/"])
+check("regex matches /v1/", f23.apply("https://example.com/v1/page"), True)
+f24 = URLPatternFilter(patterns=[r"^https://example\.com/v\d+/"])
+check("regex no match /vx/", f24.apply("https://example.com/vx/page"), False)
+
+
+# =====================================================================
+# BUG 3: PDFContentScrapingStrategy deserialization
+# =====================================================================
+print("\n" + "=" * 70)
+print("BUG 3: PDFContentScrapingStrategy deserialization")
+print("=" * 70)
+
+# --- 3A: Type is in allowlist ---
+print("\n--- Allowlist check ---")
+
+check("PDFContentScrapingStrategy in ALLOWED_DESERIALIZE_TYPES",
+    "PDFContentScrapingStrategy" in ALLOWED_DESERIALIZE_TYPES, True)
+
+# --- 3B: Roundtrip serialization ---
+print("\n--- Serialization roundtrip ---")
+
+try:
+    from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+    strategy = PDFContentScrapingStrategy(extract_images=False, batch_size=8)
+    serialized = to_serializable_dict(strategy)
+    check("serialization type field", serialized.get("type"), "PDFContentScrapingStrategy")
+    check("serialization has params", "params" in serialized, True)
+
+    # Deserialize back — verifies it resolves to the correct class (the original bug)
+    deserialized = from_serializable_dict(serialized)
+    check("deserialization type", type(deserialized).__name__, "PDFContentScrapingStrategy")
+    # The strategy passes params to NaivePDFProcessorStrategy internally,
+    # so verify via the inner processor
+    check("deserialization creates valid processor",
+        hasattr(deserialized, 'pdf_processor'), True)
+
+    print("  (roundtrip OK)")
+except ImportError as e:
+    print(f"  SKIP: PDFContentScrapingStrategy import failed: {e}")
+except Exception as e:
+    FAIL += 1
+    print(f"  FAIL: Serialization roundtrip failed: {e}")
+
+# --- 3C: CrawlerRunConfig with PDFContentScrapingStrategy ---
+print("\n--- CrawlerRunConfig with PDFContentScrapingStrategy ---")
+
+try:
+    from crawl4ai import CrawlerRunConfig
+    from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+    config = CrawlerRunConfig(
+        scraping_strategy=PDFContentScrapingStrategy(extract_images=False, batch_size=4)
+    )
+    serialized = to_serializable_dict(config)
+    deserialized = from_serializable_dict(serialized)
+    check("CrawlerRunConfig roundtrip with PDF strategy",
+        type(deserialized.scraping_strategy).__name__, "PDFContentScrapingStrategy")
+    check("PDF strategy has processor after roundtrip",
+        hasattr(deserialized.scraping_strategy, 'pdf_processor'), True)
+
+    print("  (config roundtrip OK)")
+except ImportError as e:
+    print(f"  SKIP: Import failed: {e}")
+except Exception as e:
+    FAIL += 1
+    print(f"  FAIL: Config roundtrip failed: {e}")
+
+# --- 3D: Other types still deserialize correctly (regression) ---
+print("\n--- Regression: other types still work ---")
+
+try:
+    from crawl4ai import CrawlerRunConfig, CacheMode, DefaultMarkdownGenerator
+    from crawl4ai import LXMLWebScrapingStrategy, RegexChunking
+
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(),
+        scraping_strategy=LXMLWebScrapingStrategy(),
+        chunking_strategy=RegexChunking(),
+    )
+    serialized = to_serializable_dict(config)
+    deserialized = from_serializable_dict(serialized)
+    check("CrawlerRunConfig basic roundtrip type", type(deserialized).__name__, "CrawlerRunConfig")
+    check("CrawlerRunConfig cache_mode preserved", deserialized.cache_mode, CacheMode.BYPASS)
+
+    print("  (regression OK)")
+except Exception as e:
+    FAIL += 1
+    print(f"  FAIL: Regression roundtrip failed: {e}")
+
+
+# =====================================================================
+# ADVERSARIAL / EDGE CASES — Cross-cutting
+# =====================================================================
+print("\n" + "=" * 70)
+print("ADVERSARIAL / EDGE CASES")
+print("=" * 70)
+
+# --- Antibot: various browser JSON wrapping styles ---
+print("\n--- Browser JSON wrapping variants ---")
+
+# Chrome style
+chrome_json = '
      {"origin": "1.2.3.4"}
      '
+blocked, reason = is_blocked(200, chrome_json)
+check("Chrome-style JSON wrap", blocked, False, reason)
+
+# Firefox style (no inline style on pre)
+firefox_json = '
      {"origin": "1.2.3.4"}
      '
+blocked, reason = is_blocked(200, firefox_json)
+check("Firefox-style JSON wrap", blocked, False, reason)
+
+# With extra whitespace/newlines
+whitespace_json = '\n\n\n\n  
      \n{"origin": "1.2.3.4"}
      \n\n'
+blocked, reason = is_blocked(200, whitespace_json)
+check("JSON wrap with newlines", blocked, False, reason)
+
+# Deeply nested JSON
+big_json = '
      ' + json.dumps({"data": [{"id": i, "name": f"item_{i}", "values": list(range(10))} for i in range(100)]}) + '
      '
+blocked, reason = is_blocked(200, big_json)
+check("large nested JSON in browser wrap", blocked, False, reason)
+
+# JSON with special chars
+special_json = '
      {"html": "
      hello
      ", "url": "https://example.com?a=1&b=2"}
      '
+blocked, reason = is_blocked(200, special_json)
+check("JSON with embedded HTML/URL", blocked, False, reason)
+
+# --- Antibot: responses that look similar but should still be blocked ---
+print("\n--- Similar-looking pages that SHOULD be blocked ---")
+
+# HTML page that happens to have 
       but isn't JSON
+blocked, reason = is_blocked(200, '
      Access Denied
      ')
+check("
      Access Denied
       (200, small)", blocked, True, reason)
+
+# Empty body with 
       but no JSON
+blocked, reason = is_blocked(200, '
         
      ')
+check("
       with whitespace (200)", blocked, True, reason)
+
+# 
       with non-JSON that starts with { but invalid
+blocked, reason = is_blocked(200, '
      {not valid json at all, this is just text
      ')
+# This is ambiguous — looks like it could be data. Our check just looks at { or [ prefix.
+# It will be detected as data and NOT blocked, which is the safer choice.
+check("
      {non-json text} treated as data (200)", blocked, False, reason)
+
+# --- URLPatternFilter: empty and edge-case inputs ---
+print("\n--- URLPatternFilter edge cases ---")
+
+# Empty URL
+f_edge = URLPatternFilter(patterns=["/docs/*"])
+check("empty URL no match", f_edge.apply(""), False)
+
+# URL with no path
+f_edge2 = URLPatternFilter(patterns=["/docs/*"])
+check("domain-only URL no match", f_edge2.apply("https://example.com"), False)
+
+# Root path
+f_edge3 = URLPatternFilter(patterns=["/*"])
+check("/* matches any path", f_edge3.apply("https://example.com/anything"), True)
+
+# Exact prefix match (path equals prefix exactly)
+f_edge4 = URLPatternFilter(patterns=["/docs/*"])
+check("/docs/* matches /docs exactly (prefix == path)", f_edge4.apply("https://example.com/docs"), True)
+# /docs without trailing / matches because len(url_path) == len(prefix) is the exact-match case
+
+# Very long URL
+long_path = "/docs/" + "a" * 2000
+f_edge5 = URLPatternFilter(patterns=["/docs/*"])
+check("very long path matches", f_edge5.apply(f"https://example.com{long_path}"), True)
+
+# Unicode in path
+f_edge6 = URLPatternFilter(patterns=["/docs/*"])
+check("unicode path matches", f_edge6.apply("https://example.com/docs/页面"), True)
+
+# --- Deserialization: security edge cases ---
+print("\n--- Deserialization security ---")
+
+# Ensure disallowed types still raise
+try:
+    from_serializable_dict({"type": "os.system", "params": {"command": "whoami"}})
+    FAIL += 1
+    print("  FAIL: should have raised ValueError for disallowed type")
+except (ValueError, AttributeError):
+    PASS += 1
+    print("  PASS: disallowed type 'os.system' correctly rejected")
+except Exception as e:
+    PASS += 1  # Any error is fine, as long as it doesn't execute
+    print(f"  PASS: disallowed type rejected with {type(e).__name__}")
+
+try:
+    from_serializable_dict({"type": "__import__", "params": {"name": "os"}})
+    FAIL += 1
+    print("  FAIL: should have raised for __import__")
+except (ValueError, AttributeError):
+    PASS += 1
+    print("  PASS: disallowed type '__import__' correctly rejected")
+except Exception as e:
+    PASS += 1
+    print(f"  PASS: disallowed type rejected with {type(e).__name__}")
+
+
+# =====================================================================
+# SUMMARY
+# =====================================================================
+print(f"\n{'=' * 70}")
+print(f"RESULTS: {PASS} passed, {FAIL} failed out of {PASS + FAIL} tests")
+print(f"{'=' * 70}")
+if FAIL > 0:
+    print("SOME TESTS FAILED!")
+    sys.exit(1)
+else:
+    print("ALL TESTS PASSED!")
diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py
new file mode 100644
index 0000000..700886a
--- /dev/null
+++ b/tests/test_config_defaults.py
@@ -0,0 +1,263 @@
+"""Tests for BrowserConfig.set_defaults / CrawlerRunConfig.set_defaults."""
+
+import pytest
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+
+@pytest.fixture(autouse=True)
+def _reset_defaults():
+    """Ensure every test starts and ends with a clean slate."""
+    BrowserConfig.reset_defaults()
+    CrawlerRunConfig.reset_defaults()
+    yield
+    BrowserConfig.reset_defaults()
+    CrawlerRunConfig.reset_defaults()
+
+
+# ── Basic API ──────────────────────────────────────────────────────────
+
+
+class TestBasicAPI:
+    def test_set_and_get_defaults(self):
+        BrowserConfig.set_defaults(headless=False, viewport_width=1920)
+        d = BrowserConfig.get_defaults()
+        assert d == {"headless": False, "viewport_width": 1920}
+
+    def test_get_defaults_returns_copy(self):
+        BrowserConfig.set_defaults(headers={"X-Foo": "bar"})
+        d = BrowserConfig.get_defaults()
+        d["headers"]["X-Foo"] = "changed"
+        assert BrowserConfig.get_defaults()["headers"]["X-Foo"] == "bar"
+
+    def test_reset_all(self):
+        BrowserConfig.set_defaults(headless=False)
+        BrowserConfig.reset_defaults()
+        assert BrowserConfig.get_defaults() == {}
+
+    def test_reset_selective(self):
+        BrowserConfig.set_defaults(headless=False, viewport_width=1920)
+        BrowserConfig.reset_defaults("headless")
+        assert BrowserConfig.get_defaults() == {"viewport_width": 1920}
+
+    def test_invalid_param_raises(self):
+        with pytest.raises(ValueError, match="Invalid parameter"):
+            BrowserConfig.set_defaults(not_a_real_param=42)
+
+    def test_invalid_param_among_valid(self):
+        with pytest.raises(ValueError):
+            BrowserConfig.set_defaults(headless=False, bogus=True)
+        # Nothing should have been stored
+        assert BrowserConfig.get_defaults() == {}
+
+    def test_set_defaults_overwrites(self):
+        BrowserConfig.set_defaults(headless=False)
+        BrowserConfig.set_defaults(headless=True)
+        assert BrowserConfig.get_defaults()["headless"] is True
+
+    def test_crawler_run_config_basic(self):
+        CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True)
+        d = CrawlerRunConfig.get_defaults()
+        assert d == {"verbose": False, "scan_full_page": True}
+
+
+# ── Default injection ──────────────────────────────────────────────────
+
+
+class TestDefaultInjection:
+    def test_browser_config_defaults_applied(self):
+        BrowserConfig.set_defaults(
+            headless=False,
+            cache_cdp_connection=True,
+            cdp_close_delay=0,
+        )
+        cfg = BrowserConfig()
+        assert cfg.headless is False
+        assert cfg.cache_cdp_connection is True
+        assert cfg.cdp_close_delay == 0
+
+    def test_crawler_run_config_defaults_applied(self):
+        CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True)
+        cfg = CrawlerRunConfig()
+        assert cfg.verbose is False
+        assert cfg.scan_full_page is True
+
+    def test_partial_defaults(self):
+        BrowserConfig.set_defaults(headless=False)
+        cfg = BrowserConfig()
+        assert cfg.headless is False
+        # Other params keep their hardcoded defaults
+        assert cfg.browser_type == "chromium"
+        assert cfg.viewport_width == 1080
+
+    def test_multiple_instances_get_defaults(self):
+        BrowserConfig.set_defaults(headless=False)
+        c1 = BrowserConfig()
+        c2 = BrowserConfig()
+        assert c1.headless is False
+        assert c2.headless is False
+
+
+# ── Explicit override ──────────────────────────────────────────────────
+
+
+class TestExplicitOverride:
+    def test_explicit_kwarg_wins(self):
+        BrowserConfig.set_defaults(headless=False)
+        cfg = BrowserConfig(headless=True)
+        assert cfg.headless is True
+
+    def test_explicit_same_as_default_still_wins(self):
+        """Even if user passes the same value as user-default, it should be treated as explicit."""
+        BrowserConfig.set_defaults(headless=False)
+        cfg = BrowserConfig(headless=False)
+        assert cfg.headless is False
+
+    def test_explicit_none_wins(self):
+        BrowserConfig.set_defaults(cdp_url="ws://localhost:9222")
+        cfg = BrowserConfig(cdp_url=None)
+        assert cfg.cdp_url is None
+
+    def test_mixed_explicit_and_default(self):
+        BrowserConfig.set_defaults(headless=False, viewport_width=1920)
+        cfg = BrowserConfig(headless=True)
+        assert cfg.headless is True  # explicit
+        assert cfg.viewport_width == 1920  # from user default
+
+
+# ── Mutable isolation ──────────────────────────────────────────────────
+
+
+class TestMutableIsolation:
+    def test_list_default_not_shared(self):
+        BrowserConfig.set_defaults(cookies=[{"name": "a", "value": "1"}])
+        c1 = BrowserConfig()
+        c2 = BrowserConfig()
+        c1.cookies.append({"name": "b", "value": "2"})
+        assert len(c2.cookies) == 1  # c2 should be unaffected
+
+    def test_dict_default_not_shared(self):
+        BrowserConfig.set_defaults(headers={"X-Foo": "bar"})
+        c1 = BrowserConfig()
+        c2 = BrowserConfig()
+        c1.headers["X-New"] = "val"
+        assert "X-New" not in c2.headers
+
+    def test_set_defaults_input_not_mutated(self):
+        original = {"X-Foo": "bar"}
+        BrowserConfig.set_defaults(headers=original)
+        cfg = BrowserConfig()
+        cfg.headers["X-Added"] = "val"
+        assert "X-Added" not in original
+        assert "X-Added" not in BrowserConfig.get_defaults()["headers"]
+
+
+# ── Special processing ─────────────────────────────────────────────────
+
+
+class TestSpecialProcessing:
+    def test_browser_mode_builtin_sets_managed(self):
+        BrowserConfig.set_defaults(browser_mode="builtin")
+        cfg = BrowserConfig()
+        assert cfg.use_managed_browser is True
+
+    def test_viewport_dict_overrides_dimensions(self):
+        BrowserConfig.set_defaults(viewport={"width": 1920, "height": 1080})
+        cfg = BrowserConfig()
+        assert cfg.viewport_width == 1920
+        assert cfg.viewport_height == 1080
+
+    def test_proxy_string_converted_to_proxy_config(self):
+        BrowserConfig.set_defaults(proxy="http://user:pass@proxy:8080")
+        cfg = BrowserConfig()
+        assert cfg.proxy_config is not None
+        assert cfg.proxy_config.server == "http://proxy:8080"
+
+    def test_crawler_run_config_proxy_dict_converted(self):
+        CrawlerRunConfig.set_defaults(
+            proxy_config={"server": "http://proxy:8080"}
+        )
+        cfg = CrawlerRunConfig()
+        from crawl4ai.async_configs import ProxyConfig
+        assert isinstance(cfg.proxy_config, ProxyConfig)
+
+
+# ── Clone / from_kwargs ────────────────────────────────────────────────
+
+
+class TestCloneAndFromKwargs:
+    def test_clone_preserves_user_default_values(self):
+        BrowserConfig.set_defaults(headless=False, viewport_width=1920)
+        cfg = BrowserConfig()
+        cloned = cfg.clone()
+        assert cloned.headless is False
+        assert cloned.viewport_width == 1920
+
+    def test_clone_with_override(self):
+        BrowserConfig.set_defaults(headless=False)
+        cfg = BrowserConfig()
+        cloned = cfg.clone(headless=True)
+        assert cloned.headless is True
+
+    def test_from_kwargs_explicit_values(self):
+        BrowserConfig.set_defaults(headless=False)
+        cfg = BrowserConfig.from_kwargs({"headless": True})
+        assert cfg.headless is True
+
+
+# ── Dump / Load round-trip ─────────────────────────────────────────────
+
+
+class TestDumpLoad:
+    def test_dump_load_preserves_user_defaults(self):
+        BrowserConfig.set_defaults(headless=False, viewport_width=1920)
+        cfg = BrowserConfig()
+        data = cfg.dump()
+        loaded = BrowserConfig.load(data)
+        assert loaded.headless is False
+        assert loaded.viewport_width == 1920
+
+    def test_dump_load_survives_reset(self):
+        """Values should be baked into serialized data, independent of class defaults."""
+        BrowserConfig.set_defaults(headless=False)
+        cfg = BrowserConfig()
+        data = cfg.dump()
+        BrowserConfig.reset_defaults()
+        loaded = BrowserConfig.load(data)
+        assert loaded.headless is False
+
+    def test_crawler_run_config_dump_load(self):
+        CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True)
+        cfg = CrawlerRunConfig()
+        data = cfg.dump()
+        CrawlerRunConfig.reset_defaults()
+        loaded = CrawlerRunConfig.load(data)
+        assert loaded.verbose is False
+        assert loaded.scan_full_page is True
+
+    def test_to_dict_includes_user_default_values(self):
+        BrowserConfig.set_defaults(headless=False)
+        cfg = BrowserConfig()
+        d = cfg.to_dict()
+        assert d["headless"] is False
+
+
+# ── Class isolation ────────────────────────────────────────────────────
+
+
+class TestClassIsolation:
+    def test_browser_defaults_dont_leak_to_crawler(self):
+        BrowserConfig.set_defaults(verbose=False)
+        cfg = CrawlerRunConfig()
+        assert cfg.verbose is True  # CrawlerRunConfig hardcoded default
+
+    def test_crawler_defaults_dont_leak_to_browser(self):
+        CrawlerRunConfig.set_defaults(verbose=False)
+        cfg = BrowserConfig()
+        assert cfg.verbose is True  # BrowserConfig hardcoded default
+
+    def test_independent_reset(self):
+        BrowserConfig.set_defaults(headless=False)
+        CrawlerRunConfig.set_defaults(verbose=False)
+        BrowserConfig.reset_defaults()
+        assert BrowserConfig.get_defaults() == {}
+        assert CrawlerRunConfig.get_defaults() == {"verbose": False}
diff --git a/tests/test_config_matching_only.py b/tests/test_config_matching_only.py
new file mode 100644
index 0000000..0f21666
--- /dev/null
+++ b/tests/test_config_matching_only.py
@@ -0,0 +1,131 @@
+"""
+Test only the config matching logic without running crawler
+"""
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from crawl4ai.async_configs import CrawlerRunConfig, MatchMode
+
+def test_all_matching_scenarios():
+    print("Testing CrawlerRunConfig.is_match() method")
+    print("=" * 50)
+    
+    # Test 1: Single string pattern
+    print("\n1. Single string pattern (glob style)")
+    config = CrawlerRunConfig(
+        url_matcher="*.pdf",
+        # For example we can set this => scraping_strategy=PDFContentScrapingStrategy()
+    )
+    test_urls = [
+        ("https://example.com/file.pdf", True),
+        ("https://example.com/doc.PDF", False),  # Case sensitive
+        ("https://example.com/file.txt", False),
+        ("file.pdf", True),
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 2: List of patterns with OR
+    print("\n2. List of patterns with OR (default)")
+    config = CrawlerRunConfig(
+        url_matcher=["*/article/*", "*/blog/*", "*.html"],
+        match_mode=MatchMode.OR
+    )
+    test_urls = [
+        ("https://example.com/article/news", True),
+        ("https://example.com/blog/post", True),
+        ("https://example.com/page.html", True),
+        ("https://example.com/page.php", False),
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 3: Custom function
+    print("\n3. Custom function matcher")
+    config = CrawlerRunConfig(
+        url_matcher=lambda url: 'api' in url and (url.endswith('.json') or url.endswith('.xml'))
+    )
+    test_urls = [
+        ("https://api.example.com/data.json", True),
+        ("https://api.example.com/data.xml", True),
+        ("https://api.example.com/data.html", False),
+        ("https://example.com/data.json", False),  # No 'api'
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 4: Mixed list with AND
+    print("\n4. Mixed patterns and functions with AND")
+    config = CrawlerRunConfig(
+        url_matcher=[
+            "https://*",  # Must be HTTPS
+            lambda url: '.com' in url,  # Must have .com
+            lambda url: len(url) < 50  # Must be short
+        ],
+        match_mode=MatchMode.AND
+    )
+    test_urls = [
+        ("https://example.com/page", True),
+        ("http://example.com/page", False),  # Not HTTPS
+        ("https://example.org/page", False),  # No .com
+        ("https://example.com/" + "x" * 50, False),  # Too long
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 5: Complex real-world scenario
+    print("\n5. Complex pattern combinations")
+    config = CrawlerRunConfig(
+        url_matcher=[
+            "*/api/v[0-9]/*",  # API versioned endpoints
+            lambda url: 'graphql' in url,  # GraphQL endpoints
+            "*.json"  # JSON files
+        ],
+        match_mode=MatchMode.OR
+    )
+    test_urls = [
+        ("https://example.com/api/v1/users", True),
+        ("https://example.com/api/v2/posts", True),
+        ("https://example.com/graphql", True),
+        ("https://example.com/data.json", True),
+        ("https://example.com/api/users", False),  # No version
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 6: Edge cases
+    print("\n6. Edge cases")
+    
+    # No matcher
+    config = CrawlerRunConfig()
+    result = config.is_match("https://example.com")
+    print(f"  {'✓' if not result else '✗'} No matcher -> {result}")
+    
+    # Empty list
+    config = CrawlerRunConfig(url_matcher=[])
+    result = config.is_match("https://example.com")
+    print(f"  {'✓' if not result else '✗'} Empty list -> {result}")
+    
+    # None in list (should be skipped)
+    config = CrawlerRunConfig(url_matcher=["*.pdf", None, "*.doc"])
+    result = config.is_match("test.pdf")
+    print(f"  {'✓' if result else '✗'} List with None -> {result}")
+    
+    print("\n" + "=" * 50)
+    print("All matching tests completed!")
+
+if __name__ == "__main__":
+    test_all_matching_scenarios()
\ No newline at end of file
diff --git a/tests/test_config_selection.py b/tests/test_config_selection.py
new file mode 100644
index 0000000..97245f9
--- /dev/null
+++ b/tests/test_config_selection.py
@@ -0,0 +1,87 @@
+"""
+Test config selection logic in dispatchers
+"""
+import asyncio
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from crawl4ai.async_configs import CrawlerRunConfig, MatchMode
+from crawl4ai.async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher
+
+class TestDispatcher(BaseDispatcher):
+    """Simple test dispatcher to verify config selection"""
+    
+    async def crawl_url(self, url, config, task_id, **kwargs):
+        # Just return which config was selected
+        selected = self.select_config(url, config)
+        return {"url": url, "config_id": id(selected)}
+    
+    async def run_urls(self, urls, crawler, config):
+        results = []
+        for url in urls:
+            result = await self.crawl_url(url, config, "test")
+            results.append(result)
+        return results
+
+async def test_dispatcher_config_selection():
+    print("Testing dispatcher config selection")
+    print("=" * 50)
+    
+    # Create test configs with different matchers
+    pdf_config = CrawlerRunConfig(url_matcher="*.pdf")
+    api_config = CrawlerRunConfig(url_matcher=lambda url: 'api' in url)
+    default_config = CrawlerRunConfig()  # No matcher
+    
+    configs = [pdf_config, api_config, default_config]
+    
+    # Create test dispatcher
+    dispatcher = TestDispatcher()
+    
+    # Test single config
+    print("\nTest 1: Single config")
+    result = await dispatcher.crawl_url("https://example.com/file.pdf", pdf_config, "test1")
+    assert result["config_id"] == id(pdf_config)
+    print("✓ Single config works")
+    
+    # Test config list selection
+    print("\nTest 2: Config list selection")
+    test_cases = [
+        ("https://example.com/file.pdf", id(pdf_config)),
+        ("https://api.example.com/data", id(api_config)),
+        ("https://example.com/page", id(configs[0])),  # No match, uses first
+    ]
+    
+    for url, expected_id in test_cases:
+        result = await dispatcher.crawl_url(url, configs, "test")
+        assert result["config_id"] == expected_id, f"URL {url} got wrong config"
+        print(f"✓ {url} -> correct config selected")
+    
+    # Test with MemoryAdaptiveDispatcher
+    print("\nTest 3: MemoryAdaptiveDispatcher config selection")
+    mem_dispatcher = MemoryAdaptiveDispatcher()
+    
+    # Test select_config method directly
+    selected = mem_dispatcher.select_config("https://example.com/doc.pdf", configs)
+    assert selected == pdf_config
+    print("✓ MemoryAdaptiveDispatcher.select_config works")
+    
+    # Test empty config list
+    print("\nTest 4: Edge cases")
+    selected = mem_dispatcher.select_config("https://example.com", [])
+    assert isinstance(selected, CrawlerRunConfig)  # Should return default
+    print("✓ Empty config list returns default config")
+    
+    # Test None config
+    selected = mem_dispatcher.select_config("https://example.com", None)
+    assert isinstance(selected, CrawlerRunConfig)  # Should return default
+    print("✓ None config returns default config")
+    
+    print("\n" + "=" * 50)
+    print("All dispatcher tests passed! ✓")
+
+if __name__ == "__main__":
+    asyncio.run(test_dispatcher_config_selection())
\ No newline at end of file
diff --git a/tests/test_docker.py b/tests/test_docker.py
new file mode 100644
index 0000000..c507ae5
--- /dev/null
+++ b/tests/test_docker.py
@@ -0,0 +1,299 @@
+import requests
+import json
+import time
+import sys
+import base64
+import os
+from typing import Dict, Any
+
+
+class Crawl4AiTester:
+    def __init__(self, base_url: str = "http://localhost:11235"):
+        self.base_url = base_url
+
+    def submit_and_wait(
+        self, request_data: Dict[str, Any], timeout: int = 300
+    ) -> Dict[str, Any]:
+        # Submit crawl job
+        response = requests.post(f"{self.base_url}/crawl", json=request_data)
+        task_id = response.json()["task_id"]
+        print(f"Task ID: {task_id}")
+
+        # Poll for result
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Task {task_id} did not complete within {timeout} seconds"
+                )
+
+            result = requests.get(f"{self.base_url}/task/{task_id}")
+            status = result.json()
+
+            if status["status"] == "failed":
+                print("Task failed:", status.get("error"))
+                raise Exception(f"Task failed: {status.get('error')}")
+
+            if status["status"] == "completed":
+                return status
+
+            time.sleep(2)
+
+
+def test_docker_deployment(version="basic"):
+    tester = Crawl4AiTester()
+    print(f"Testing Crawl4AI Docker {version} version")
+
+    # Health check with timeout and retry
+    max_retries = 5
+    for i in range(max_retries):
+        try:
+            health = requests.get(f"{tester.base_url}/health", timeout=10)
+            print("Health check:", health.json())
+            break
+        except requests.exceptions.RequestException:
+            if i == max_retries - 1:
+                print(f"Failed to connect after {max_retries} attempts")
+                sys.exit(1)
+            print(f"Waiting for service to start (attempt {i+1}/{max_retries})...")
+            time.sleep(5)
+
+    # Test cases based on version
+    test_basic_crawl(tester)
+
+    # if version in ["full", "transformer"]:
+    #     test_cosine_extraction(tester)
+
+    # test_js_execution(tester)
+    # test_css_selector(tester)
+    # test_structured_extraction(tester)
+    # test_llm_extraction(tester)
+    # test_llm_with_ollama(tester)
+    # test_screenshot(tester)
+
+
+def test_basic_crawl(tester: Crawl4AiTester):
+    print("\n=== Testing Basic Crawl ===")
+    request = {"urls": ["https://www.nbcnews.com/business"], "priority": 10}
+
+    result = tester.submit_and_wait(request)
+    print(f"Basic crawl result length: {len(result['result']['markdown'])}")
+    assert result["result"]["success"]
+    assert len(result["result"]["markdown"]) > 0
+
+
+def test_js_execution(tester: Crawl4AiTester):
+    print("\n=== Testing JS Execution ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "priority": 8,
+        "js_code": [
+            "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+        ],
+        "wait_for": "article.tease-card:nth-child(10)",
+        "crawler_params": {"headless": True},
+    }
+
+    result = tester.submit_and_wait(request)
+    print(f"JS execution result length: {len(result['result']['markdown'])}")
+    assert result["result"]["success"]
+
+
+def test_css_selector(tester: Crawl4AiTester):
+    print("\n=== Testing CSS Selector ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "priority": 7,
+        "css_selector": ".wide-tease-item__description",
+        "crawler_params": {"headless": True},
+        "extra": {"word_count_threshold": 10},
+    }
+
+    result = tester.submit_and_wait(request)
+    print(f"CSS selector result length: {len(result['result']['markdown'])}")
+    assert result["result"]["success"]
+
+
+def test_structured_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing Structured Extraction ===")
+    schema = {
+        "name": "Coinbase Crypto Prices",
+        "baseSelector": ".cds-tableRow-t45thuk",
+        "fields": [
+            {
+                "name": "crypto",
+                "selector": "td:nth-child(1) h2",
+                "type": "text",
+            },
+            {
+                "name": "symbol",
+                "selector": "td:nth-child(1) p",
+                "type": "text",
+            },
+            {
+                "name": "price",
+                "selector": "td:nth-child(2)",
+                "type": "text",
+            },
+        ],
+    }
+
+    request = {
+        "urls": ["https://www.coinbase.com/explore"],
+        "priority": 9,
+        "extraction_config": {"type": "json_css", "params": {"schema": schema}},
+    }
+
+    result = tester.submit_and_wait(request)
+    extracted = json.loads(result["result"]["extracted_content"])
+    print(f"Extracted {len(extracted)} items")
+    print("Sample item:", json.dumps(extracted[0], indent=2))
+    assert result["result"]["success"]
+    assert len(extracted) > 0
+
+
+def test_llm_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing LLM Extraction ===")
+    schema = {
+        "type": "object",
+        "properties": {
+            "model_name": {
+                "type": "string",
+                "description": "Name of the OpenAI model.",
+            },
+            "input_fee": {
+                "type": "string",
+                "description": "Fee for input token for the OpenAI model.",
+            },
+            "output_fee": {
+                "type": "string",
+                "description": "Fee for output token for the OpenAI model.",
+            },
+        },
+        "required": ["model_name", "input_fee", "output_fee"],
+    }
+
+    request = {
+        "urls": ["https://openai.com/api/pricing"],
+        "priority": 8,
+        "extraction_config": {
+            "type": "llm",
+            "params": {
+                "provider": "openai/gpt-4o-mini",
+                "api_token": os.getenv("OPENAI_API_KEY"),
+                "schema": schema,
+                "extraction_type": "schema",
+                "instruction": """From the crawled content, extract all mentioned model names along with their fees for input and output tokens.""",
+            },
+        },
+        "crawler_params": {"word_count_threshold": 1},
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["extracted_content"])
+        print(f"Extracted {len(extracted)} model pricing entries")
+        print("Sample entry:", json.dumps(extracted[0], indent=2))
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"LLM extraction test failed (might be due to missing API key): {str(e)}")
+
+
+def test_llm_with_ollama(tester: Crawl4AiTester):
+    print("\n=== Testing LLM with Ollama ===")
+    schema = {
+        "type": "object",
+        "properties": {
+            "article_title": {
+                "type": "string",
+                "description": "The main title of the news article",
+            },
+            "summary": {
+                "type": "string",
+                "description": "A brief summary of the article content",
+            },
+            "main_topics": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Main topics or themes discussed in the article",
+            },
+        },
+    }
+
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "priority": 8,
+        "extraction_config": {
+            "type": "llm",
+            "params": {
+                "provider": "ollama/llama2",
+                "schema": schema,
+                "extraction_type": "schema",
+                "instruction": "Extract the main article information including title, summary, and main topics.",
+            },
+        },
+        "extra": {"word_count_threshold": 1},
+        "crawler_params": {"verbose": True},
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["extracted_content"])
+        print("Extracted content:", json.dumps(extracted, indent=2))
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"Ollama extraction test failed: {str(e)}")
+
+
+def test_cosine_extraction(tester: Crawl4AiTester):
+    print("\n=== Testing Cosine Extraction ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "priority": 8,
+        "extraction_config": {
+            "type": "cosine",
+            "params": {
+                "semantic_filter": "business finance economy",
+                "word_count_threshold": 10,
+                "max_dist": 0.2,
+                "top_k": 3,
+            },
+        },
+    }
+
+    try:
+        result = tester.submit_and_wait(request)
+        extracted = json.loads(result["result"]["extracted_content"])
+        print(f"Extracted {len(extracted)} text clusters")
+        print("First cluster tags:", extracted[0]["tags"])
+        assert result["result"]["success"]
+    except Exception as e:
+        print(f"Cosine extraction test failed: {str(e)}")
+
+
+def test_screenshot(tester: Crawl4AiTester):
+    print("\n=== Testing Screenshot ===")
+    request = {
+        "urls": ["https://www.nbcnews.com/business"],
+        "priority": 5,
+        "screenshot": True,
+        "crawler_params": {"headless": True},
+    }
+
+    result = tester.submit_and_wait(request)
+    print("Screenshot captured:", bool(result["result"]["screenshot"]))
+
+    if result["result"]["screenshot"]:
+        # Save screenshot
+        screenshot_data = base64.b64decode(result["result"]["screenshot"])
+        with open("test_screenshot.jpg", "wb") as f:
+            f.write(screenshot_data)
+        print("Screenshot saved as test_screenshot.jpg")
+
+    assert result["result"]["success"]
+
+
+if __name__ == "__main__":
+    version = sys.argv[1] if len(sys.argv) > 1 else "basic"
+    # version = "full"
+    test_docker_deployment(version)
diff --git a/tests/test_docker_api_with_llm_provider.py b/tests/test_docker_api_with_llm_provider.py
new file mode 100644
index 0000000..f17368a
--- /dev/null
+++ b/tests/test_docker_api_with_llm_provider.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Test script to verify Docker API with LLM provider configuration."""
+
+import requests
+import json
+import time
+
+BASE_URL = "http://localhost:11235"
+
+def test_health():
+    """Test health endpoint."""
+    print("1. Testing health endpoint...")
+    response = requests.get(f"{BASE_URL}/health")
+    print(f"   Status: {response.status_code}")
+    print(f"   Response: {response.json()}")
+    print()
+
+def test_schema():
+    """Test schema endpoint to see configuration."""
+    print("2. Testing schema endpoint...")
+    response = requests.get(f"{BASE_URL}/schema")
+    print(f"   Status: {response.status_code}")
+    # Print only browser config to keep output concise
+    print(f"   Browser config keys: {list(response.json().get('browser', {}).keys())[:5]}...")
+    print()
+
+def test_markdown_with_llm_filter():
+    """Test markdown endpoint with LLM filter (should use configured provider)."""
+    print("3. Testing markdown endpoint with LLM filter...")
+    print("   This should use the Groq provider from LLM_PROVIDER env var")
+    
+    # Note: This will fail with dummy API keys, but we can see if it tries to use Groq
+    payload = {
+        "url": "https://httpbin.org/html",
+        "f": "llm",
+        "q": "Extract the main content"
+    }
+    
+    response = requests.post(f"{BASE_URL}/md", json=payload)
+    print(f"   Status: {response.status_code}")
+    
+    if response.status_code != 200:
+        print(f"   Error: {response.text[:200]}...")
+    else:
+        print(f"   Success! Markdown length: {len(response.json().get('markdown', ''))} chars")
+    print()
+
+def test_markdown_with_provider_override():
+    """Test markdown endpoint with provider override in request."""
+    print("4. Testing markdown endpoint with provider override...")
+    print("   This should use OpenAI provider from request parameter")
+    
+    payload = {
+        "url": "https://httpbin.org/html",
+        "f": "llm",
+        "q": "Extract the main content",
+        "provider": "openai/gpt-4"  # Override to use OpenAI
+    }
+    
+    response = requests.post(f"{BASE_URL}/md", json=payload)
+    print(f"   Status: {response.status_code}")
+    
+    if response.status_code != 200:
+        print(f"   Error: {response.text[:200]}...")
+    else:
+        print(f"   Success! Markdown length: {len(response.json().get('markdown', ''))} chars")
+    print()
+
+def test_simple_crawl():
+    """Test simple crawl without LLM."""
+    print("5. Testing simple crawl (no LLM required)...")
+    
+    payload = {
+        "urls": ["https://httpbin.org/html"],
+        "browser_config": {
+            "type": "BrowserConfig",
+            "params": {"headless": True}
+        },
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {"cache_mode": "bypass"}
+        }
+    }
+    
+    response = requests.post(f"{BASE_URL}/crawl", json=payload)
+    print(f"   Status: {response.status_code}")
+    
+    if response.status_code == 200:
+        result = response.json()
+        print(f"   Success: {result.get('success')}")
+        print(f"   Results count: {len(result.get('results', []))}")
+        if result.get('results'):
+            print(f"   First result success: {result['results'][0].get('success')}")
+    else:
+        print(f"   Error: {response.text[:200]}...")
+    print()
+
+def test_playground():
+    """Test if playground is accessible."""
+    print("6. Testing playground interface...")
+    response = requests.get(f"{BASE_URL}/playground")
+    print(f"   Status: {response.status_code}")
+    print(f"   Content-Type: {response.headers.get('content-type')}")
+    print()
+
+if __name__ == "__main__":
+    print("=== Crawl4AI Docker API Tests ===\n")
+    print(f"Testing API at {BASE_URL}\n")
+    
+    # Wait a bit for server to be fully ready
+    time.sleep(2)
+    
+    test_health()
+    test_schema()
+    test_simple_crawl()
+    test_playground()
+    
+    print("\nTesting LLM functionality (these may fail with dummy API keys):\n")
+    test_markdown_with_llm_filter()
+    test_markdown_with_provider_override()
+    
+    print("\nTests completed!")
\ No newline at end of file
diff --git a/tests/test_eval_security_adversarial.py b/tests/test_eval_security_adversarial.py
new file mode 100644
index 0000000..7ec5f99
--- /dev/null
+++ b/tests/test_eval_security_adversarial.py
@@ -0,0 +1,838 @@
+#!/usr/bin/env python3
+"""
+Adversarial security tests for all eval/exec paths in crawl4ai.
+
+Tests three attack surfaces:
+1. _compute_field expression path (extraction_strategy.py) - MUST be fully disabled
+2. _safe_eval_config (deploy/docker/server.py) - MUST block all escapes
+3. hook_manager exec (deploy/docker/hook_manager.py) - MUST restrict builtins
+
+Each section tries progressively creative bypass techniques.
+"""
+
+import ast
+import sys
+import os
+import unittest
+import logging
+
+# Ensure crawl4ai is importable
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "deploy", "docker"))
+
+
+# ============================================================================
+# PART 1: _compute_field expression path - MUST BE COMPLETELY DEAD
+# ============================================================================
+
+class TestComputeFieldExpressionKilled(unittest.TestCase):
+    """The expression key in computed fields must NEVER evaluate anything.
+    It should log a warning and return default. Period."""
+
+    @classmethod
+    def setUpClass(cls):
+        from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+        schema = {"baseSelector": "div", "fields": [
+            {"name": "x", "selector": "span", "type": "text"}
+        ]}
+        cls.strategy = JsonCssExtractionStrategy(schema)
+
+    def _try_expression(self, expr, item=None, default="BLOCKED"):
+        """Helper: run expression through _compute_field, expect default back."""
+        field = {"name": "test", "type": "computed", "expression": expr, "default": default}
+        return self.strategy._compute_field(item or {}, field)
+
+    # -- Basic RCE attempts --
+
+    def test_import_os_system(self):
+        self.assertEqual(self._try_expression("__import__('os').system('id')"), "BLOCKED")
+
+    def test_import_subprocess(self):
+        self.assertEqual(self._try_expression("__import__('subprocess').check_output('id', shell=True)"), "BLOCKED")
+
+    def test_open_etc_passwd(self):
+        self.assertEqual(self._try_expression("open('/etc/passwd').read()"), "BLOCKED")
+
+    def test_eval_inside_eval(self):
+        self.assertEqual(self._try_expression("eval('__import__(\"os\").system(\"id\")')"), "BLOCKED")
+
+    def test_exec_code(self):
+        self.assertEqual(self._try_expression("exec('import os; os.system(\"id\")')"), "BLOCKED")
+
+    # -- The original vuln report exploit --
+
+    def test_original_exploit_payload(self):
+        """Exact payload from the vulnerability report."""
+        payload = (
+            "(lambda: (g := (f := type(type).mro).__func__.__globals__), "
+            "g['__builtins__']['__import__']('os').popen('id').read()))()"
+        )
+        self.assertEqual(self._try_expression(payload), "BLOCKED")
+
+    # -- Frame/generator traversal --
+
+    def test_gi_frame(self):
+        self.assertEqual(self._try_expression("(x for x in [1]).gi_frame.f_builtins['__import__']('os')"), "BLOCKED")
+
+    def test_f_back(self):
+        self.assertEqual(self._try_expression("(x for x in [1]).gi_frame.f_back.f_builtins"), "BLOCKED")
+
+    def test_cr_frame(self):
+        self.assertEqual(self._try_expression("x.cr_frame.f_globals"), "BLOCKED")
+
+    # -- Dunder traversal --
+
+    def test_class_bases_subclasses(self):
+        self.assertEqual(self._try_expression("().__class__.__bases__[0].__subclasses__()"), "BLOCKED")
+
+    def test_class_mro(self):
+        self.assertEqual(self._try_expression("''.__class__.__mro__[1].__subclasses__()"), "BLOCKED")
+
+    def test_globals_access(self):
+        self.assertEqual(self._try_expression("(lambda: 0).__globals__"), "BLOCKED")
+
+    def test_init_globals(self):
+        self.assertEqual(self._try_expression("''.__class__.__init__.__globals__"), "BLOCKED")
+
+    # -- Format string bypass (the one I flagged) --
+
+    def test_format_string_dunder_access(self):
+        """Format strings bypass AST attribute checks - dunder access happens at runtime."""
+        self.assertEqual(
+            self._try_expression("'{0.__class__.__init__.__globals__}'.format('')"),
+            "BLOCKED"
+        )
+
+    def test_fstring_dunder_access(self):
+        self.assertEqual(
+            self._try_expression("f'{\"\".__class__.__init__.__globals__}'"),
+            "BLOCKED"
+        )
+
+    # -- Lambda/generator tricks --
+
+    def test_lambda_exec(self):
+        self.assertEqual(self._try_expression("(lambda: exec('import os'))()"), "BLOCKED")
+
+    def test_generator_with_side_effects(self):
+        self.assertEqual(self._try_expression("list(x for x in __import__('os').listdir('/'))"), "BLOCKED")
+
+    def test_nested_lambda(self):
+        self.assertEqual(self._try_expression("(lambda f: f(f))(lambda f: 'pwned')"), "BLOCKED")
+
+    # -- Comprehension tricks --
+
+    def test_listcomp_with_import(self):
+        self.assertEqual(self._try_expression("[__import__('os') for _ in [1]]"), "BLOCKED")
+
+    def test_dictcomp_with_import(self):
+        self.assertEqual(self._try_expression("{k: __import__('os') for k in [1]}"), "BLOCKED")
+
+    def test_setcomp_with_import(self):
+        self.assertEqual(self._try_expression("{__import__('os') for _ in [1]}"), "BLOCKED")
+
+    # -- Indirect access --
+
+    def test_getattr_bypass(self):
+        self.assertEqual(self._try_expression("getattr(getattr('', '__class__'), '__bases__')"), "BLOCKED")
+
+    def test_vars_bypass(self):
+        self.assertEqual(self._try_expression("vars()"), "BLOCKED")
+
+    def test_dir_probe(self):
+        self.assertEqual(self._try_expression("dir(__builtins__)"), "BLOCKED")
+
+    def test_type_call(self):
+        self.assertEqual(self._try_expression("type.__bases__[0].__subclasses__()"), "BLOCKED")
+
+    # -- Benign expressions also return default (expression is fully disabled) --
+
+    def test_simple_math_also_disabled(self):
+        """Even harmless math must return default - no eval at all."""
+        self.assertEqual(self._try_expression("price * 2", {"price": 100}), "BLOCKED")
+
+    def test_string_method_also_disabled(self):
+        self.assertEqual(self._try_expression("name.upper()", {"name": "test"}), "BLOCKED")
+
+    def test_string_concat_also_disabled(self):
+        self.assertEqual(self._try_expression("a + b", {"a": "hello", "b": "world"}), "BLOCKED")
+
+    # -- Verify function key still works --
+
+    def test_function_key_works(self):
+        field = {"name": "test", "type": "computed", "function": lambda item: item["x"] * 3}
+        result = self.strategy._compute_field({"x": 10}, field)
+        self.assertEqual(result, 30)
+
+    def test_function_key_with_complex_logic(self):
+        def compute(item):
+            return f"{item['first']} {item['last']}".upper()
+        field = {"name": "test", "type": "computed", "function": compute}
+        result = self.strategy._compute_field({"first": "John", "last": "Doe"}, field)
+        self.assertEqual(result, "JOHN DOE")
+
+
+# ============================================================================
+# PART 2: _safe_eval_config - server.py config deserializer
+# ============================================================================
+
+class TestSafeEvalConfigAdversarial(unittest.TestCase):
+    """Attack the server.py _safe_eval_config AST validation logic.
+    Self-contained: copies the validation logic to avoid needing FastAPI/Redis.
+    Must allow CrawlerRunConfig(...) / BrowserConfig(...) but block everything else."""
+
+    @classmethod
+    def setUpClass(cls):
+        import crawl4ai as _c4
+        from crawl4ai import CrawlerRunConfig, BrowserConfig
+
+        _SAFE_CONFIG_ALLOWED_NAMES = {
+            "CrawlerRunConfig", "BrowserConfig", "HTTPCrawlerConfig",
+            "LLMConfig", "ProxyConfig", "GeolocationConfig",
+            "SeedingConfig", "VirtualScrollConfig", "LinkPreviewConfig",
+            "JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
+            "JsonLxmlExtractionStrategy", "LLMExtractionStrategy",
+            "CosineStrategy", "RegexExtractionStrategy",
+            "DefaultMarkdownGenerator",
+            "PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
+            "LXMLWebScrapingStrategy",
+            "RegexChunking",
+            "BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy",
+            "FilterChain", "URLPatternFilter", "DomainFilter",
+            "ContentTypeFilter", "URLFilter", "SEOFilter", "ContentRelevanceFilter",
+            "KeywordRelevanceScorer", "URLScorer", "CompositeScorer",
+            "DomainAuthorityScorer", "FreshnessScorer", "PathDepthScorer",
+            "CacheMode", "MatchMode", "DisplayMode",
+            "MemoryAdaptiveDispatcher", "SemaphoreDispatcher",
+            "DefaultTableExtraction", "NoTableExtraction",
+            "RoundRobinProxyStrategy",
+        }
+
+        _SAFE_CONFIG_ALLOWED_ATTRS = frozenset({
+            "BYPASS", "READ_ONLY", "WRITE_ONLY", "ENABLED", "DISABLED",
+            "READ_WRITE", "BYPASS_CACHE", "STANDARD", "COMPACT", "DETAILED",
+            "value", "name",
+        })
+
+        def safe_eval_config(expr):
+            tree = ast.parse(expr, mode="eval")
+            if not isinstance(tree.body, ast.Call):
+                raise ValueError("Expression must be a single constructor call")
+            call = tree.body
+            if not (isinstance(call.func, ast.Name) and call.func.id in {"CrawlerRunConfig", "BrowserConfig"}):
+                raise ValueError("Only CrawlerRunConfig(...) or BrowserConfig(...) are allowed")
+            for node in ast.walk(call):
+                if isinstance(node, ast.Call) and node is not call:
+                    raise ValueError("Nested function calls are not permitted")
+                if isinstance(node, ast.Lambda):
+                    raise ValueError("Lambda expressions are not permitted")
+                if isinstance(node, (ast.GeneratorExp, ast.ListComp, ast.SetComp, ast.DictComp)):
+                    raise ValueError("Comprehensions and generators are not permitted")
+                if isinstance(node, ast.Attribute):
+                    if node.attr not in _SAFE_CONFIG_ALLOWED_ATTRS:
+                        raise ValueError(f"Attribute access '{node.attr}' is not permitted")
+                if isinstance(node, ast.Name) and node.id not in _SAFE_CONFIG_ALLOWED_NAMES:
+                    if node.id not in {"True", "False", "None"}:
+                        raise ValueError(f"Name '{node.id}' is not permitted")
+            safe_env = {}
+            for name in _SAFE_CONFIG_ALLOWED_NAMES:
+                obj = getattr(_c4, name, None)
+                if obj is not None:
+                    safe_env[name] = obj
+            safe_env.update({"True": True, "False": False, "None": None})
+            obj = eval(compile(tree, "", "eval"), {"__builtins__": {}}, safe_env)
+            return obj.dump()
+
+        cls.safe_eval_config = staticmethod(safe_eval_config)
+
+    # -- Must work: legitimate config --
+
+    def test_basic_crawler_run_config(self):
+        result = self.safe_eval_config("CrawlerRunConfig()")
+        self.assertIsInstance(result, dict)
+
+    def test_basic_browser_config(self):
+        result = self.safe_eval_config("BrowserConfig()")
+        self.assertIsInstance(result, dict)
+
+    def test_config_with_simple_args(self):
+        result = self.safe_eval_config("BrowserConfig(headless=True)")
+        self.assertIsInstance(result, dict)
+
+    def test_config_with_string_args(self):
+        result = self.safe_eval_config('CrawlerRunConfig(wait_until="load")')
+        self.assertIsInstance(result, dict)
+
+    # -- Must block: not a config constructor --
+
+    def test_arbitrary_function_call(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("print('hello')")
+
+    def test_import_call(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("__import__('os')")
+
+    def test_bare_expression(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("1 + 1")
+
+    def test_eval_call(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("eval('1+1')")
+
+    # -- Must block: nested function calls --
+
+    def test_nested_import_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=__import__('os').popen('id').read())")
+
+    def test_nested_eval_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=eval('bad'))")
+
+    def test_nested_open_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=open('/etc/passwd').read())")
+
+    # -- Must block: lambda/generator in args --
+
+    def test_lambda_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=lambda: __import__('os'))")
+
+    def test_generator_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=(x for x in [1]))")
+
+    def test_listcomp_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=[x for x in [1]])")
+
+    def test_dictcomp_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code={x: 1 for x in [1]})")
+
+    def test_setcomp_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code={x for x in [1]})")
+
+    # -- Must block: attribute traversal attacks --
+
+    def test_dunder_class_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=''.__class__)")
+
+    def test_dunder_globals_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=''.__class__.__init__.__globals__)")
+
+    def test_dunder_bases_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=().__class__.__bases__)")
+
+    def test_gi_frame_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=x.gi_frame)")
+
+    def test_f_builtins_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=x.f_builtins)")
+
+    def test_f_back_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=x.f_back)")
+
+    def test_f_globals_in_args(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=x.f_globals)")
+
+    # -- Must block: name references to non-allowlisted objects --
+
+    def test_os_name_ref(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=os)")
+
+    def test_sys_name_ref(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=sys)")
+
+    def test_builtins_name_ref(self):
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code=__builtins__)")
+
+    # -- Must block: string-based escapes --
+
+    def test_format_string_dunder(self):
+        """Format strings evaluated at runtime - blocked because format() is a nested call."""
+        with self.assertRaises(ValueError):
+            self.safe_eval_config("CrawlerRunConfig(js_code='{0.__class__}'.format(''))")
+
+    # -- Must block: walrus operator / assignment --
+
+    def test_walrus_operator(self):
+        with self.assertRaises((ValueError, SyntaxError)):
+            self.safe_eval_config("CrawlerRunConfig(js_code=(x := __import__('os')))")
+
+
+# ============================================================================
+# PART 3: _safe_eval_expression DELETED
+# The function and _SAFE_EVAL_BUILTINS were removed from extraction_strategy.py.
+# Dead security-sensitive code is a liability.
+# ============================================================================
+
+class TestSafeEvalExpressionDeleted(unittest.TestCase):
+    """Verify _safe_eval_expression is gone from the codebase."""
+
+    def test_function_not_importable(self):
+        """_safe_eval_expression must not exist in extraction_strategy."""
+        from crawl4ai import extraction_strategy
+        self.assertFalse(
+            hasattr(extraction_strategy, '_safe_eval_expression'),
+            "_safe_eval_expression should be deleted - dead security code is a liability"
+        )
+
+    def test_safe_eval_builtins_not_importable(self):
+        """_SAFE_EVAL_BUILTINS must not exist in extraction_strategy."""
+        from crawl4ai import extraction_strategy
+        self.assertFalse(
+            hasattr(extraction_strategy, '_SAFE_EVAL_BUILTINS'),
+            "_SAFE_EVAL_BUILTINS should be deleted along with _safe_eval_expression"
+        )
+
+
+# ============================================================================
+# PART 4: hook_manager builtins - verify getattr/setattr are gone
+# ============================================================================
+
+class TestHookManagerBuiltins(unittest.TestCase):
+    """Verify hook_manager no longer provides getattr/setattr."""
+
+    def test_getattr_removed_from_source(self):
+        """Read hook_manager.py and verify getattr not in allowed_builtins."""
+        hook_path = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), "..",
+            "deploy", "docker", "hook_manager.py"
+        )
+        with open(hook_path, "r") as f:
+            source = f.read()
+
+        # Parse the source and find the allowed_builtins list
+        tree = ast.parse(source)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Assign):
+                for target in node.targets:
+                    if isinstance(target, ast.Name) and target.id == "allowed_builtins":
+                        if isinstance(node.value, ast.List):
+                            values = [
+                                elt.value for elt in node.value.elts
+                                if isinstance(elt, ast.Constant)
+                            ]
+                            # Batch 2 hardening: hasattr, type, __build_class__ also removed
+                            self.assertNotIn("getattr", values,
+                                "getattr must not be in hook allowed_builtins (sandbox escape)")
+                            self.assertNotIn("setattr", values,
+                                "setattr must not be in hook allowed_builtins (sandbox escape)")
+                            self.assertNotIn("hasattr", values,
+                                "hasattr removed in batch 2 (info disclosure via probing)")
+                            self.assertNotIn("type", values,
+                                "type removed in batch 2 (__subclasses__ MRO chain escape)")
+                            self.assertNotIn("__build_class__", values,
+                                "__build_class__ removed in batch 2 (__init_subclass__ abuse)")
+                            return
+
+        self.fail("Could not find allowed_builtins in hook_manager.py")
+
+
+# ============================================================================
+# PART 5: Meta-checks - verify no unprotected eval/exec paths exist
+# ============================================================================
+
+class TestNoUnprotectedEval(unittest.TestCase):
+    """Scan the codebase for eval/exec calls to catch regressions."""
+
+    def _scan_python_files(self, directory, exclude_dirs=None):
+        """Find all eval()/exec() calls in Python files."""
+        exclude_dirs = exclude_dirs or {"__pycache__", ".git", "node_modules", "venv", ".venv", "build", "dist", ".eggs"}
+        hits = []
+        for root, dirs, files in os.walk(directory):
+            dirs[:] = [d for d in dirs if d not in exclude_dirs]
+            for fname in files:
+                if not fname.endswith(".py"):
+                    continue
+                fpath = os.path.join(root, fname)
+                try:
+                    with open(fpath) as f:
+                        source = f.read()
+                    tree = ast.parse(source, filename=fpath)
+                    for node in ast.walk(tree):
+                        if isinstance(node, ast.Call):
+                            func = node.func
+                            if isinstance(func, ast.Name) and func.id in ("eval", "exec"):
+                                hits.append((fpath, node.lineno, func.id))
+                except (SyntaxError, UnicodeDecodeError):
+                    continue
+        return hits
+
+    def test_all_eval_exec_are_known(self):
+        """Every eval/exec in the repo must be in the known-safe list."""
+        repo_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
+
+        # Known, audited locations (file suffix, call type)
+        known_safe = {
+            # server.py _safe_eval_config - hardened with allowlist
+            ("deploy/docker/server.py", "eval"),
+            # hook_manager.py - restricted namespace, hooks gated behind env var
+            ("deploy/docker/hook_manager.py", "exec"),
+            # NOTE: extraction_strategy.py eval was DELETED, not just disabled
+        }
+
+        hits = self._scan_python_files(repo_root)
+        unknown = []
+        for fpath, lineno, call_type in hits:
+            rel = os.path.relpath(fpath, repo_root)
+            # Skip test files
+            if "test" in rel.lower():
+                continue
+            # Check if known
+            is_known = any(
+                rel.replace("\\", "/").endswith(known_file) and call_type == known_call
+                for known_file, known_call in known_safe
+            )
+            if not is_known:
+                unknown.append(f"  {rel}:{lineno} - {call_type}()")
+
+        if unknown:
+            self.fail(
+                f"Found {len(unknown)} unknown eval/exec call(s):\n"
+                + "\n".join(unknown)
+                + "\n\nAudit these and add to known_safe if they are properly protected."
+            )
+
+
+# ============================================================================
+# PART 6: Hook manager sandbox escape tests
+# ============================================================================
+
+class TestHookManagerSandboxEscapes(unittest.TestCase):
+    """Try every trick to escape the hook_manager exec() sandbox.
+    Hooks are the most dangerous surface: exec() on user-supplied code."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Build the hook sandbox exactly as hook_manager.py does."""
+        import builtins
+        import types
+
+        safe_builtins = {}
+        allowed_builtins = [
+            'print', 'len', 'str', 'int', 'float', 'bool',
+            'list', 'dict', 'set', 'tuple', 'range', 'enumerate',
+            'zip', 'map', 'filter', 'any', 'all', 'sum', 'min', 'max',
+            'sorted', 'reversed', 'abs', 'round', 'isinstance', 'type',
+            'hasattr', 'callable', 'iter', 'next',
+            '__build_class__'
+        ]
+        for name in allowed_builtins:
+            if hasattr(builtins, name):
+                safe_builtins[name] = getattr(builtins, name)
+
+        cls.safe_builtins = safe_builtins
+
+    def _make_namespace(self):
+        """Create a fresh hook namespace with sanitized imports (as hook_manager does).
+        Mirrors the actual hook_manager.py injection approach: import in our scope,
+        sanitize, then inject into namespace. exec("import X", ns) doesn't work
+        because ns lacks __import__."""
+        import asyncio as _asyncio_mod
+        import json as _json_mod
+        import re as _re_mod
+        import types
+        from typing import Dict, List, Optional
+
+        namespace = {
+            '__name__': 'test_hook',
+            '__builtins__': dict(self.safe_builtins),
+        }
+
+        # Sanitize asyncio: strip subprocess access
+        safe_asyncio = types.ModuleType("asyncio")
+        for attr in dir(_asyncio_mod):
+            if attr not in ("subprocess", "create_subprocess_exec",
+                            "create_subprocess_shell"):
+                try:
+                    setattr(safe_asyncio, attr, getattr(_asyncio_mod, attr))
+                except (AttributeError, TypeError):
+                    pass
+
+        namespace["asyncio"] = safe_asyncio
+        namespace["json"] = _json_mod
+        namespace["re"] = _re_mod
+        namespace["Dict"] = Dict
+        namespace["List"] = List
+        namespace["Optional"] = Optional
+
+        return namespace
+
+    def _exec_hook(self, code):
+        """Execute hook code in sandbox, return namespace."""
+        ns = self._make_namespace()
+        exec(code, ns)
+        return ns
+
+    # -- The original RCE that was proven exploitable --
+
+    def test_asyncio_subprocess_blocked(self):
+        """asyncio.subprocess must not be accessible (was RCE vector)."""
+        ns = self._make_namespace()
+        self.assertFalse(
+            hasattr(ns["asyncio"], "subprocess"),
+            "asyncio.subprocess must be stripped from hook namespace"
+        )
+
+    def test_asyncio_create_subprocess_shell_blocked(self):
+        """asyncio.create_subprocess_shell must not be accessible."""
+        ns = self._make_namespace()
+        self.assertFalse(
+            hasattr(ns["asyncio"], "create_subprocess_shell"),
+            "asyncio.create_subprocess_shell must be stripped"
+        )
+
+    def test_asyncio_create_subprocess_exec_blocked(self):
+        """asyncio.create_subprocess_exec must not be accessible."""
+        ns = self._make_namespace()
+        self.assertFalse(
+            hasattr(ns["asyncio"], "create_subprocess_exec"),
+            "asyncio.create_subprocess_exec must be stripped"
+        )
+
+    def test_asyncio_subprocess_rce_attempt(self):
+        """Actually try the RCE via asyncio.subprocess -- must fail."""
+        code = '''
+async def evil(page, ctx):
+    sp = asyncio.subprocess
+    proc = await sp.create_subprocess_shell('id', stdout=sp.PIPE)
+    out, _ = await proc.communicate()
+    return out.decode()
+'''
+        with self.assertRaises(AttributeError):
+            ns = self._exec_hook(code)
+            import asyncio
+            asyncio.get_event_loop().run_until_complete(ns['evil'](None, None))
+
+    # -- asyncio useful functions still work --
+
+    def test_asyncio_sleep_still_works(self):
+        """asyncio.sleep must still be available for hooks."""
+        ns = self._make_namespace()
+        self.assertTrue(hasattr(ns["asyncio"], "sleep"))
+
+    def test_asyncio_gather_still_works(self):
+        """asyncio.gather must still be available."""
+        ns = self._make_namespace()
+        self.assertTrue(hasattr(ns["asyncio"], "gather"))
+
+    def test_asyncio_event_still_works(self):
+        """asyncio.Event must still be available."""
+        ns = self._make_namespace()
+        self.assertTrue(hasattr(ns["asyncio"], "Event"))
+
+    # -- Try importing os/subprocess directly --
+
+    def test_import_os_blocked(self):
+        """Direct 'import os' must fail (no __import__)."""
+        with self.assertRaises(ImportError):
+            self._exec_hook("import os")
+
+    def test_import_subprocess_blocked(self):
+        with self.assertRaises(ImportError):
+            self._exec_hook("import subprocess")
+
+    def test_import_sys_blocked(self):
+        with self.assertRaises(ImportError):
+            self._exec_hook("import sys")
+
+    # -- Try __import__ smuggling --
+
+    def test_dunder_import_not_available(self):
+        """__import__ must not be in builtins."""
+        ns = self._make_namespace()
+        self.assertNotIn('__import__', ns['__builtins__'])
+
+    def test_builtins_import_via_type(self):
+        """type().__bases__ subclass scanning can list classes but can't get __import__."""
+        ns = self._exec_hook("""
+result = [c.__name__ for c in type.__bases__[0].__subclasses__()[:5]]
+""")
+        # The subclass list is accessible, but without __import__ in builtins
+        # there's no path to import os/subprocess for RCE
+        self.assertNotIn('__import__', ns['__builtins__'])
+
+    # -- Try reaching os via module attributes --
+
+    def test_json_os_not_reachable(self):
+        """json module should not expose os."""
+        ns = self._make_namespace()
+        self.assertFalse(hasattr(ns.get("json"), "os"))
+
+    def test_re_os_not_reachable(self):
+        """re module should not expose os."""
+        ns = self._make_namespace()
+        self.assertFalse(hasattr(ns.get("re"), "os"))
+
+    def test_asyncio_os_not_reachable(self):
+        """asyncio should not expose os."""
+        ns = self._make_namespace()
+        self.assertFalse(hasattr(ns.get("asyncio"), "os"))
+
+    # -- Try module __loader__ / __spec__ traversal --
+
+    def test_module_loader_traversal(self):
+        """Try to reach importlib via asyncio.__loader__ -- should not give RCE."""
+        ns = self._make_namespace()
+        # Even if __loader__ exists, it shouldn't provide __import__
+        asyncio_mod = ns.get("asyncio")
+        if hasattr(asyncio_mod, "__loader__"):
+            loader = asyncio_mod.__loader__
+            # loader.load_module is deprecated but check it doesn't exist
+            # The key is: without __import__ in builtins, the hook code
+            # can't call loader methods that would import modules
+            self.assertNotIn('__import__', ns['__builtins__'])
+
+    # -- Try getattr/setattr (should be removed) --
+
+    def test_getattr_not_available(self):
+        """getattr must not be in builtins (sandbox escape vector)."""
+        ns = self._make_namespace()
+        self.assertNotIn('getattr', ns['__builtins__'])
+
+    def test_setattr_not_available(self):
+        """setattr must not be in builtins."""
+        ns = self._make_namespace()
+        self.assertNotIn('setattr', ns['__builtins__'])
+
+    # -- Try frame walking from within hook --
+
+    def test_frame_walk_from_hook(self):
+        """Frame walking inside exec'd code to escape sandbox."""
+        code = '''
+import sys
+'''
+        with self.assertRaises(ImportError):
+            self._exec_hook(code)
+
+    # -- Try generator gi_frame trick (the original vuln) from hook --
+
+    def test_gi_frame_from_hook(self):
+        """The original gi_frame.f_back exploit should not give __import__."""
+        # Even if frame walking works, builtins in this frame should not have __import__
+        ns = self._make_namespace()
+        self.assertNotIn('__import__', ns['__builtins__'])
+
+
+# ============================================================================
+# PART 7: End-to-end exploit payload test
+# ============================================================================
+
+class TestEndToEndExploit(unittest.TestCase):
+    """Test the EXACT exploit from the vulnerability report against _compute_field.
+    This is the ultimate test: the actual payload that achieved RCE."""
+
+    @classmethod
+    def setUpClass(cls):
+        from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+        schema = {"baseSelector": "div", "fields": [
+            {"name": "x", "selector": "span", "type": "text"}
+        ]}
+        cls.strategy = JsonCssExtractionStrategy(schema)
+
+    def test_exact_exploit_from_report(self):
+        """The exact payload from Song Binglin's vulnerability report."""
+        EXPLOIT = (
+            "(lambda r: ["
+            "dict.update(r, {"
+            "'g': ("
+            "dict.update(r, {'f': r['g'].gi_frame.f_back}) or 0 "
+            "for x in range(1)"
+            ")"
+            "}),"
+            "list(r['g']),"
+            "(lambda w, f: w(w, f))("
+            "lambda s, f: "
+            "f.f_builtins['__import__']('os').popen(r['cmd']).read() "
+            "if f and '__import__' in f.f_builtins "
+            "else s(s, f.f_back) if f else 'frame_walk_failed',"
+            "r['f']"
+            ")"
+            "][2])({'cmd': 'id'})"
+        )
+        field = {
+            "name": "rce_output",
+            "type": "computed",
+            "expression": EXPLOIT,
+            "default": "BLOCKED",
+        }
+        result = self.strategy._compute_field({}, field)
+        self.assertEqual(result, "BLOCKED",
+            "The exact exploit payload from the vuln report must return default, never execute")
+
+    def test_simplified_gi_frame_exploit(self):
+        """Simplified version targeting gi_frame directly."""
+        field = {
+            "name": "test",
+            "type": "computed",
+            "expression": "(x for x in [1]).gi_frame.f_builtins['__import__']('os').popen('id').read()",
+            "default": "BLOCKED",
+        }
+        result = self.strategy._compute_field({}, field)
+        self.assertEqual(result, "BLOCKED")
+
+    def test_lambda_globals_exploit(self):
+        """Lambda-based __globals__ access."""
+        field = {
+            "name": "test",
+            "type": "computed",
+            "expression": "(lambda: 0).__code__.co_consts",
+            "default": "BLOCKED",
+        }
+        result = self.strategy._compute_field({}, field)
+        self.assertEqual(result, "BLOCKED")
+
+    def test_type_mro_exploit(self):
+        """type().mro() based subclass scanning."""
+        field = {
+            "name": "test",
+            "type": "computed",
+            "expression": "[c for c in ().__class__.__bases__[0].__subclasses__() if 'warning' in c.__name__][0]()._module.__builtins__['__import__']('os').popen('id').read()",
+            "default": "BLOCKED",
+        }
+        result = self.strategy._compute_field({}, field)
+        self.assertEqual(result, "BLOCKED")
+
+    def test_exploit_via_json_schema(self):
+        """Simulate how the exploit arrives: embedded in extraction schema."""
+        from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+        malicious_schema = {
+            "name": "pwned",
+            "baseSelector": "body",
+            "fields": [
+                {
+                    "name": "rce_output",
+                    "type": "computed",
+                    "expression": "__import__('os').popen('id').read()",
+                    "default": None,
+                }
+            ]
+        }
+        strategy = JsonCssExtractionStrategy(malicious_schema)
+        result = strategy._compute_field({}, malicious_schema["fields"][0])
+        self.assertIsNone(result, "Malicious schema expression must return default (None)")
+
+
+if __name__ == "__main__":
+    print("=" * 70)
+    print("Crawl4AI Adversarial Security Tests")
+    print("=" * 70)
+    print()
+    unittest.main(verbosity=2)
diff --git a/tests/test_http_timeout_unit_1894.py b/tests/test_http_timeout_unit_1894.py
new file mode 100644
index 0000000..ba73738
--- /dev/null
+++ b/tests/test_http_timeout_unit_1894.py
@@ -0,0 +1,78 @@
+"""
+Tests for #1894: AsyncHTTPCrawlerStrategy passes page_timeout (ms) directly
+to aiohttp ClientTimeout (seconds) without converting.
+
+Verifies that page_timeout (milliseconds) is correctly converted to seconds
+before being passed to aiohttp.ClientTimeout.
+"""
+import pytest
+from unittest.mock import patch, AsyncMock, MagicMock
+import aiohttp
+
+
+class TestHttpTimeoutConversion:
+    """Verify page_timeout ms → seconds conversion for aiohttp."""
+
+    def test_default_page_timeout_is_milliseconds(self):
+        """page_timeout default (60000) is in milliseconds."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        config = CrawlerRunConfig()
+        assert config.page_timeout == 60000, "Default page_timeout should be 60000ms"
+
+    def test_aiohttp_client_timeout_expects_seconds(self):
+        """Sanity check: aiohttp.ClientTimeout(total=60) means 60 seconds."""
+        timeout = aiohttp.ClientTimeout(total=60)
+        assert timeout.total == 60
+
+    def test_conversion_default_timeout(self):
+        """Default page_timeout=60000ms should convert to 60s for aiohttp."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        config = CrawlerRunConfig()
+
+        # Simulate the conversion logic from _handle_http
+        timeout_sec = (config.page_timeout / 1000) if config.page_timeout else 30
+        assert timeout_sec == 60.0
+
+    def test_conversion_custom_timeout(self):
+        """page_timeout=5000ms should convert to 5s for aiohttp."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        config = CrawlerRunConfig(page_timeout=5000)
+
+        timeout_sec = (config.page_timeout / 1000) if config.page_timeout else 30
+        assert timeout_sec == 5.0
+
+    def test_conversion_small_timeout(self):
+        """page_timeout=500ms should convert to 0.5s for aiohttp."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        config = CrawlerRunConfig(page_timeout=500)
+
+        timeout_sec = (config.page_timeout / 1000) if config.page_timeout else 30
+        assert timeout_sec == 0.5
+
+    def test_timeout_not_absurdly_large(self):
+        """Converted timeout should never be in the thousands of seconds."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        config = CrawlerRunConfig()  # default 60000ms
+
+        timeout_sec = (config.page_timeout / 1000) if config.page_timeout else 30
+        timeout = aiohttp.ClientTimeout(total=timeout_sec)
+
+        assert timeout.total <= 300, (
+            f"Timeout is {timeout.total}s ({timeout.total/3600:.1f}h) — "
+            "likely a ms/s unit mismatch"
+        )
+
+    def test_strategy_has_correct_default_timeout_in_seconds(self):
+        """AsyncHTTPCrawlerStrategy.DEFAULT_TIMEOUT should be in seconds."""
+        from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
+        assert AsyncHTTPCrawlerStrategy.DEFAULT_TIMEOUT == 30, (
+            "DEFAULT_TIMEOUT should be 30 seconds"
+        )
+
+    def test_zero_page_timeout_uses_default(self):
+        """page_timeout=0 should fall back to DEFAULT_TIMEOUT."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        config = CrawlerRunConfig(page_timeout=0)
+
+        timeout_sec = (config.page_timeout / 1000) if config.page_timeout else 30
+        assert timeout_sec == 30
diff --git a/tests/test_issue_1043_mermaid_svg.py b/tests/test_issue_1043_mermaid_svg.py
new file mode 100644
index 0000000..31f1e35
--- /dev/null
+++ b/tests/test_issue_1043_mermaid_svg.py
@@ -0,0 +1,229 @@
+"""
+Tests for issue #1043: Missing Mermaid Flowcharts
+
+Verifies that mermaid SVG diagrams are preserved as text content
+during HTML scraping, rather than being stripped entirely.
+"""
+
+import pytest
+from lxml import html as lhtml
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+
+
+@pytest.fixture
+def strategy():
+    return LXMLWebScrapingStrategy()
+
+
+def _make_html(body_content: str) -> str:
+    return f"{body_content}"
+
+
+# -- Mermaid SVG detection and replacement --
+
+FLOWCHART_SVG = """
+
      
+    
      Before diagram
      
+    
+        
      Start
      
+        
      Process Data
      
+        
      End
      
+        
      yes
      
+          
+    
      After diagram
      
+
      
+"""
+
+CLASS_DIAGRAM_SVG = """
+
      
+    
+        
      MyClass
      
+        
      +method() : void
      
+        
      -field : int
      
+          
+
      
+"""
+
+SEQUENCE_SVG = """
+
      
+    
+        
      Alice
      
+        
      Bob
      
+        
      Hello
      
+          
+
      
+"""
+
+
+class TestMermaidSVGDetection:
+    """Test that mermaid SVGs are detected by their id prefix."""
+
+    def test_flowchart_svg_detected(self, strategy):
+        html = _make_html(FLOWCHART_SVG)
+        result = strategy._scrap("http://test.com", html)
+        assert result is not None
+        cleaned = result.get("cleaned_html", "")
+        assert "Start" in cleaned
+        assert "Process Data" in cleaned
+
+    def test_non_mermaid_svg_not_affected(self, strategy):
+        """Regular SVGs without mermaid id should be unaffected."""
+        html = _make_html("""
+            
      
+                
+                
      Content here
      
+            
      
+        """)
+        result = strategy._scrap("http://test.com", html)
+        assert result is not None
+
+    def test_mermaid_svg_replaced_with_pre_code(self, strategy):
+        """Mermaid SVG should be replaced with pre/code block."""
+        html = _make_html(FLOWCHART_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "language-mermaid" in cleaned or "mermaid" in cleaned.lower()
+
+
+class TestMermaidTextExtraction:
+    """Test that text content is correctly extracted from mermaid SVGs."""
+
+    def test_node_labels_extracted(self, strategy):
+        html = _make_html(FLOWCHART_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "Start" in cleaned
+        assert "Process Data" in cleaned
+        assert "End" in cleaned
+
+    def test_edge_labels_extracted(self, strategy):
+        html = _make_html(FLOWCHART_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "yes" in cleaned
+
+    def test_class_diagram_labels_extracted(self, strategy):
+        html = _make_html(CLASS_DIAGRAM_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "MyClass" in cleaned
+        assert "+method() : void" in cleaned
+
+    def test_sequence_diagram_labels_extracted(self, strategy):
+        html = _make_html(SEQUENCE_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "Alice" in cleaned
+        assert "Bob" in cleaned
+
+    def test_duplicate_labels_deduplicated(self, strategy):
+        """Same label appearing multiple times should only appear once."""
+        html = _make_html("""
+            
      
+                
+                    
      Repeated
      
+                    
      Repeated
      
+                    
      Unique
      
+                      
+            
      
+        """)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        # Should have Repeated once, not twice
+        assert cleaned.count("Repeated") == 1
+        assert "Unique" in cleaned
+
+
+class TestMermaidDiagramType:
+    """Test that diagram type is preserved."""
+
+    def test_flowchart_type_preserved(self, strategy):
+        html = _make_html(FLOWCHART_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "flowchart" in cleaned.lower()
+
+    def test_class_type_preserved(self, strategy):
+        html = _make_html(CLASS_DIAGRAM_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "class" in cleaned.lower()
+
+    def test_sequence_type_preserved(self, strategy):
+        html = _make_html(SEQUENCE_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "sequence" in cleaned.lower()
+
+
+class TestMermaidSurroundingContent:
+    """Test that surrounding content is preserved."""
+
+    def test_text_before_diagram_preserved(self, strategy):
+        html = _make_html(FLOWCHART_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "Before diagram" in cleaned
+
+    def test_text_after_diagram_preserved(self, strategy):
+        html = _make_html(FLOWCHART_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "After diagram" in cleaned
+
+
+class TestMermaidEdgeCases:
+    """Test edge cases for mermaid SVG handling."""
+
+    def test_empty_mermaid_svg(self, strategy):
+        """SVG with no text content should be handled gracefully."""
+        html = _make_html("""
+            
      
+                
+                    
+                
+                
      Content
      
+            
      
+        """)
+        result = strategy._scrap("http://test.com", html)
+        assert result is not None
+        cleaned = result.get("cleaned_html", "")
+        assert "Content" in cleaned
+
+    def test_multiple_mermaid_svgs(self, strategy):
+        """Multiple mermaid diagrams on one page."""
+        html = _make_html(FLOWCHART_SVG + CLASS_DIAGRAM_SVG)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "Start" in cleaned
+        assert "MyClass" in cleaned
+
+    def test_mermaid_svg_no_aria(self, strategy):
+        """Mermaid SVG without aria-roledescription should use 'diagram' fallback."""
+        html = _make_html("""
+            
      
+                
+                    
      Node A
      
+                      
+            
      
+        """)
+        result = strategy._scrap("http://test.com", html)
+        cleaned = result.get("cleaned_html", "")
+        assert "Node A" in cleaned
+        assert "diagram" in cleaned.lower()
+
+    def test_mermaid_svg_malformed_no_crash(self, strategy):
+        """Malformed SVG should not crash the scraper."""
+        html = _make_html("""
+            
      
+                
+                
+                
      Still works
      
+            
      
+        """)
+        result = strategy._scrap("http://test.com", html)
+        assert result is not None
+        cleaned = result.get("cleaned_html", "")
+        assert "Still works" in cleaned
diff --git a/tests/test_issue_1213_bm25_dedup.py b/tests/test_issue_1213_bm25_dedup.py
new file mode 100644
index 0000000..03be67b
--- /dev/null
+++ b/tests/test_issue_1213_bm25_dedup.py
@@ -0,0 +1,178 @@
+"""
+Tests for GitHub issue #1213: BM25ContentFilter returns multiple copies of
+the same text when the DOM contains repeated content blocks.
+
+Fix: added deduplication by chunk text in filter_content(), keeping the
+first occurrence in document order.
+"""
+
+import pytest
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _wrap_html(body_inner: str) -> str:
+    """Wrap body HTML with boilerplate including a title for query extraction."""
+    return (
+        "Python Programming Guide"
+        ''
+        f"{body_inner}"
+    )
+
+
+# Provide enough varied paragraphs so BM25 IDF doesn't go negative.
+# The "target" paragraph is the one we'll duplicate; the others provide corpus variety.
+TARGET_TEXT = (
+    "Python is a versatile programming language widely used for web development, "
+    "data science, machine learning, and automation tasks across the industry."
+)
+
+FILLER_PARAGRAPHS = [
+    "
      JavaScript powers interactive websites and runs in every modern browser environment.
      ",
+    "
      Rust provides memory safety without garbage collection through its ownership model.
      ",
+    "
      Go was designed at Google for building scalable network services and cloud infrastructure.
      ",
+    "
      Ruby on Rails popularized convention over configuration in web application frameworks.
      ",
+    "
      TypeScript adds static type checking to JavaScript for large-scale application development.
      ",
+    "
      Java remains dominant in enterprise software and Android mobile application development.
      ",
+    "
      C++ is essential for game engines, operating systems, and high-performance computing.
      ",
+]
+
+FILLER_BLOCK = "\n".join(FILLER_PARAGRAPHS)
+
+
+def _make_test_html(target_copies: int = 1, extra_body: str = "") -> str:
+    """Build HTML with `target_copies` of TARGET_TEXT + filler for BM25 variety."""
+    target_block = f"
      {TARGET_TEXT}
      \n" * target_copies
+    return _wrap_html(target_block + FILLER_BLOCK + extra_body)
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+class TestBM25Deduplication:
+    """Core deduplication tests for issue #1213."""
+
+    def test_exact_duplicates_collapsed(self):
+        """Identical paragraphs should appear only once in output."""
+        html = _make_test_html(target_copies=5)
+        filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        matching = [r for r in results if "versatile" in r]
+        assert len(matching) == 1
+
+    def test_no_duplicates_unchanged(self):
+        """Unique paragraphs should all be preserved (none removed by dedup)."""
+        html = _make_test_html(target_copies=1)
+        filt = BM25ContentFilter(user_query="programming languages", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        # At least some pass threshold; key point is no false dedup
+        assert len(results) >= 1
+        # All results should be unique strings
+        assert len(results) == len(set(results))
+
+    def test_mixed_unique_and_duplicate(self):
+        """Duplicates removed, uniques kept."""
+        extra = f"
      {TARGET_TEXT}
      {TARGET_TEXT}
      "
+        html = _make_test_html(target_copies=1, extra_body=extra)
+        filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        matching = [r for r in results if "versatile" in r]
+        assert len(matching) == 1
+
+    def test_document_order_preserved(self):
+        """First occurrence should be kept, not the last."""
+        first_text = "Python was created by Guido van Rossum in the early nineties as a successor to the ABC language."
+        second_text = "Python supports multiple programming paradigms including procedural and functional styles."
+        body = (
+            f"
      {first_text}
      "
+            f"
      {second_text}
      "
+            f"
      {first_text}
      "  # duplicate
+            + FILLER_BLOCK
+        )
+        html = _wrap_html(body)
+        filt = BM25ContentFilter(user_query="Python history", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        guido_results = [r for r in results if "Guido" in r]
+        assert len(guido_results) == 1
+
+    def test_empty_html(self):
+        """Empty input should return empty list."""
+        filt = BM25ContentFilter()
+        assert filt.filter_content("") == []
+
+    def test_none_html(self):
+        """None input should return empty list."""
+        filt = BM25ContentFilter()
+        assert filt.filter_content(None) == []
+
+    def test_no_body(self):
+        """HTML without explicit body should still work."""
+        body = f"
      {TARGET_TEXT}
      " * 3 + FILLER_BLOCK
+        filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01)
+        results = filt.filter_content(body)
+        matching = [r for r in results if "versatile" in r]
+        assert len(matching) == 1
+
+    def test_single_paragraph_with_filler(self):
+        """Single target paragraph — nothing to deduplicate."""
+        html = _make_test_html(target_copies=1)
+        filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        matching = [r for r in results if "versatile" in r]
+        assert len(matching) == 1
+
+    def test_high_threshold_filters_all(self):
+        """Very high threshold should return nothing."""
+        html = _make_test_html(target_copies=3)
+        filt = BM25ContentFilter(bm25_threshold=9999.0)
+        results = filt.filter_content(html)
+        assert results == []
+
+    def test_duplicate_with_different_tags(self):
+        """Same text in different tag types should still be deduplicated."""
+        text = "Python enables rapid prototyping and development of complex software systems."
+        extra = f"
      {text}
      {text}
      "
+        html = _make_test_html(target_copies=0, extra_body=extra)
+        filt = BM25ContentFilter(user_query="Python development", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        matching = [r for r in results if "rapid prototyping" in r]
+        assert len(matching) == 1
+
+    def test_many_duplicates(self):
+        """30 identical paragraphs should collapse to 1."""
+        html = _make_test_html(target_copies=30)
+        filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        matching = [r for r in results if "versatile" in r]
+        assert len(matching) == 1
+
+    def test_user_query_with_duplicates(self):
+        """Dedup works correctly when a user_query is provided."""
+        html = _make_test_html(target_copies=4)
+        filt = BM25ContentFilter(user_query="Python web development", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        matching = [r for r in results if "versatile" in r]
+        assert len(matching) == 1
+
+    def test_stemming_disabled_with_duplicates(self):
+        """Dedup works with use_stemming=False."""
+        html = _make_test_html(target_copies=4)
+        filt = BM25ContentFilter(
+            user_query="Python programming", bm25_threshold=0.01, use_stemming=False
+        )
+        results = filt.filter_content(html)
+        matching = [r for r in results if "versatile" in r]
+        assert len(matching) == 1
+
+    def test_returns_list_type(self):
+        """Output should always be a list of strings."""
+        html = _make_test_html(target_copies=2)
+        filt = BM25ContentFilter(user_query="Python", bm25_threshold=0.01)
+        results = filt.filter_content(html)
+        assert isinstance(results, list)
+        for r in results:
+            assert isinstance(r, str)
diff --git a/tests/test_issue_1370_1818_1762_1509.py b/tests/test_issue_1370_1818_1762_1509.py
new file mode 100644
index 0000000..732502d
--- /dev/null
+++ b/tests/test_issue_1370_1818_1762_1509.py
@@ -0,0 +1,512 @@
+"""
+Regression tests for crawl4ai issue fixes:
+  #1762 — CLI encoding (utf-8 on all file writes)
+  #1370 — Screenshot distortion on Elementor sites (dimension freezing)
+  #1818 — Deep crawl timeout due to page reuse (window.stop + listener cleanup)
+  #1509 — deep_crawl_strategy in arun_many (bypass dispatcher)
+"""
+
+import asyncio
+import base64
+import inspect
+import re
+import textwrap
+from io import BytesIO
+from unittest.mock import AsyncMock, MagicMock, patch, mock_open, call
+
+import pytest
+from PIL import Image
+
+
+# ---------------------------------------------------------------------------
+# Issue #1762 — CLI encoding
+# ---------------------------------------------------------------------------
+
+class TestIssue1762_CLIEncoding:
+    """Verify that all file writes in cli.py use encoding='utf-8'."""
+
+    def test_save_global_config_writes_utf8(self):
+        """save_global_config must open the config file with encoding='utf-8'."""
+        from crawl4ai.cli import save_global_config
+
+        m = mock_open()
+        with patch("builtins.open", m), \
+             patch("yaml.dump") as mock_dump:
+            save_global_config({"key": "value"})
+
+        # open() should have been called with encoding="utf-8"
+        m.assert_called_once()
+        _, kwargs = m.call_args
+        assert kwargs.get("encoding") == "utf-8", (
+            "save_global_config must write with encoding='utf-8'"
+        )
+
+    def test_all_open_writes_in_cli_use_utf8(self):
+        """Every open(..., 'w' ...) call in cli.py should include encoding='utf-8'."""
+        import crawl4ai.cli as cli_module
+        source = inspect.getsource(cli_module)
+
+        # Find all open(..., "w" ...) patterns in source
+        # Match open(, "w" or 'w', ...) calls
+        write_opens = re.findall(
+            r'open\([^)]*["\']w["\'][^)]*\)', source
+        )
+        assert len(write_opens) > 0, "Should find at least one open(..., 'w') in cli.py"
+
+        for match in write_opens:
+            assert 'encoding="utf-8"' in match or "encoding='utf-8'" in match, (
+                f"Missing encoding='utf-8' in: {match}"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Issue #1370 — Screenshot distortion on Elementor sites
+# ---------------------------------------------------------------------------
+
+def _make_mock_page(page_width=1280, page_height=3000, viewport_height=900):
+    """Create a mock Playwright Page for screenshot tests."""
+    page = AsyncMock()
+    page.viewport_size = {"width": page_width, "height": viewport_height}
+
+    # get_page_dimensions returns page dimensions
+    # (accessed via self.get_page_dimensions)
+
+    # page.screenshot returns a small valid JPEG image
+    img = Image.new("RGB", (page_width, viewport_height), color="blue")
+    buf = BytesIO()
+    img.save(buf, format="JPEG")
+    page.screenshot = AsyncMock(return_value=buf.getvalue())
+
+    # page.evaluate is an AsyncMock — we inspect calls to it
+    page.evaluate = AsyncMock(return_value=None)
+
+    # page.set_viewport_size is an AsyncMock
+    page.set_viewport_size = AsyncMock()
+
+    return page
+
+
+def _make_strategy():
+    """Create a minimal AsyncPlaywrightCrawlerStrategy mock for screenshot tests."""
+    strategy = MagicMock()
+    strategy.logger = MagicMock()
+
+    # Import the real method and bind it
+    from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+    strategy.take_screenshot_scroller = (
+        AsyncPlaywrightCrawlerStrategy.take_screenshot_scroller.__get__(
+            strategy, type(strategy)
+        )
+    )
+    strategy.get_page_dimensions = AsyncMock()
+    return strategy
+
+
+@pytest.mark.asyncio
+class TestIssue1370_ScreenshotDistortion:
+    """Verify take_screenshot_scroller freezes/unfreezes dimensions and
+    saves/restores the original viewport."""
+
+    async def test_original_viewport_saved_and_restored(self):
+        page = _make_mock_page(page_width=1280, page_height=1800, viewport_height=900)
+        strategy = _make_strategy()
+        strategy.get_page_dimensions.return_value = {"width": 1280, "height": 1800}
+
+        original_viewport = page.viewport_size.copy()
+
+        await strategy.take_screenshot_scroller(page)
+
+        # The very last set_viewport_size call must restore the original viewport
+        last_viewport_call = page.set_viewport_size.call_args_list[-1]
+        assert last_viewport_call == call(original_viewport), (
+            "Original viewport should be restored after screenshot capture"
+        )
+
+    async def test_css_freeze_called_before_viewport_change(self):
+        page = _make_mock_page(page_width=1280, page_height=1800, viewport_height=900)
+        strategy = _make_strategy()
+        strategy.get_page_dimensions.return_value = {"width": 1280, "height": 1800}
+
+        await strategy.take_screenshot_scroller(page)
+
+        # The first evaluate call should be the freeze JS
+        first_eval = page.evaluate.call_args_list[0]
+        js_code = first_eval[0][0]
+        assert "crawl4aiFrozen" in js_code, (
+            "First JS evaluate should freeze element dimensions"
+        )
+        assert "important" in js_code, (
+            "Frozen dimensions should use !important"
+        )
+
+    async def test_css_unfreeze_called_after_capture(self):
+        page = _make_mock_page(page_width=1280, page_height=1800, viewport_height=900)
+        strategy = _make_strategy()
+        strategy.get_page_dimensions.return_value = {"width": 1280, "height": 1800}
+
+        await strategy.take_screenshot_scroller(page)
+
+        # Find the unfreeze call — should contain removeProperty
+        eval_calls = [c[0][0] for c in page.evaluate.call_args_list]
+        unfreeze_calls = [c for c in eval_calls if "removeProperty" in c]
+        assert len(unfreeze_calls) >= 1, (
+            "Should call JS to unfreeze (removeProperty) element dimensions"
+        )
+
+    async def test_segments_captured_and_stitched(self):
+        page_height = 2000
+        viewport_height = 900
+        page = _make_mock_page(page_width=800, page_height=page_height, viewport_height=viewport_height)
+        strategy = _make_strategy()
+        strategy.get_page_dimensions.return_value = {"width": 800, "height": page_height}
+
+        # Dynamic screenshot: last segment is the remainder
+        call_count = [0]
+        last_part = page_height % viewport_height  # 200
+        num_segments = (page_height // viewport_height) + 1  # 3
+
+        async def dynamic_screenshot(**kwargs):
+            call_count[0] += 1
+            if call_count[0] == num_segments:
+                h = last_part
+            else:
+                h = viewport_height
+            img = Image.new("RGB", (800, h), color="blue")
+            buf = BytesIO()
+            img.save(buf, format="JPEG")
+            return buf.getvalue()
+
+        page.screenshot = dynamic_screenshot
+
+        result = await strategy.take_screenshot_scroller(page)
+
+        # Result should be a valid base64 string
+        assert isinstance(result, str)
+        decoded = base64.b64decode(result)
+        img = Image.open(BytesIO(decoded))
+        # Total stitched height should equal page height
+        assert img.height == page_height, (
+            f"Stitched image height {img.height} should equal page height {page_height}"
+        )
+
+    async def test_page_shorter_than_viewport(self):
+        """When page height < viewport, we still get a valid screenshot."""
+        page = _make_mock_page(page_width=800, page_height=400, viewport_height=900)
+        strategy = _make_strategy()
+        strategy.get_page_dimensions.return_value = {"width": 800, "height": 400}
+
+        # The viewport will be set to min(400, threshold)=400, and
+        # page_height % viewport_height == 0, so the last segment is skipped
+        # but there should be at least one segment from i=0 (since 400 // 400 + 1 = 2,
+        # first segment captured, second skipped because remainder is 0).
+        # We need the screenshot mock to return an image of the right viewport size
+        img = Image.new("RGB", (800, 400), color="red")
+        buf = BytesIO()
+        img.save(buf, format="JPEG")
+        page.screenshot = AsyncMock(return_value=buf.getvalue())
+
+        result = await strategy.take_screenshot_scroller(page)
+        assert isinstance(result, str)
+        decoded = base64.b64decode(result)
+        stitched = Image.open(BytesIO(decoded))
+        assert stitched.height == 400
+
+    async def test_page_exact_multiple_of_viewport(self):
+        """When page height is exact multiple of viewport, no extra segment."""
+        page = _make_mock_page(page_width=800, page_height=1800, viewport_height=900)
+        strategy = _make_strategy()
+        strategy.get_page_dimensions.return_value = {"width": 800, "height": 1800}
+
+        result = await strategy.take_screenshot_scroller(page)
+        assert isinstance(result, str)
+        decoded = base64.b64decode(result)
+        stitched = Image.open(BytesIO(decoded))
+        # 1800 / 900 = 2 segments, each 900px → total 1800
+        assert stitched.height == 1800
+
+    async def test_page_not_multiple_of_viewport_last_segment(self):
+        """When page height is not a multiple, last segment is smaller."""
+        page_height = 2100
+        viewport_height = 900
+        page = _make_mock_page(page_width=800, page_height=page_height, viewport_height=viewport_height)
+        strategy = _make_strategy()
+        strategy.get_page_dimensions.return_value = {"width": 800, "height": page_height}
+
+        # For the last segment, viewport will be resized to 2100 % 900 = 300
+        # We need screenshot to return images of the right size based on call order
+        call_count = [0]
+        last_part = page_height % viewport_height  # 300
+
+        async def dynamic_screenshot(**kwargs):
+            call_count[0] += 1
+            # Segments: 0 (900), 1 (900), 2 (300 — last)
+            num_segments = (page_height // viewport_height) + 1  # 3
+            if call_count[0] == num_segments:
+                h = last_part
+            else:
+                h = viewport_height
+            img = Image.new("RGB", (800, h), color="green")
+            buf = BytesIO()
+            img.save(buf, format="JPEG")
+            return buf.getvalue()
+
+        page.screenshot = dynamic_screenshot
+
+        result = await strategy.take_screenshot_scroller(page)
+        decoded = base64.b64decode(result)
+        stitched = Image.open(BytesIO(decoded))
+        assert stitched.height == page_height, (
+            f"Stitched height {stitched.height} should be {page_height}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Issue #1818 — Deep crawl timeout due to page reuse
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+class TestIssue1818_DeepCrawlTimeout:
+    """Verify window.stop() and event listener cleanup behaviour."""
+
+    def _make_crawl_mocks(self, session_id=None, capture_network=True):
+        """Build strategy + config + page mocks for crawl tests."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+
+        config = MagicMock(spec=CrawlerRunConfig)
+        config.session_id = session_id
+        config.capture_network_requests = capture_network
+        config.capture_console_messages = False
+        return config
+
+    async def test_window_stop_called_when_session_id_set(self):
+        """When session_id is set, window.stop() should be called."""
+        page = AsyncMock()
+        page.evaluate = AsyncMock(return_value=None)
+
+        config = self._make_crawl_mocks(session_id="my-session")
+
+        # Simulate the logic from _crawl lines 571-575
+        if config.session_id:
+            try:
+                await page.evaluate("window.stop()")
+            except Exception:
+                pass
+
+        page.evaluate.assert_called_once_with("window.stop()")
+
+    async def test_window_stop_not_called_when_no_session_id(self):
+        """When session_id is not set, window.stop() should NOT be called."""
+        page = AsyncMock()
+        page.evaluate = AsyncMock(return_value=None)
+
+        config = self._make_crawl_mocks(session_id=None)
+
+        # Same logic — should NOT call evaluate
+        if config.session_id:
+            try:
+                await page.evaluate("window.stop()")
+            except Exception:
+                pass
+
+        page.evaluate.assert_not_called()
+
+    async def test_event_listeners_removed_with_session_id(self):
+        """Event listeners should be cleaned up even when session_id is set."""
+        page = AsyncMock()
+        page.remove_listener = MagicMock()
+
+        config = self._make_crawl_mocks(session_id="my-session", capture_network=True)
+
+        handle_request = MagicMock()
+        handle_response = MagicMock()
+        handle_failed = MagicMock()
+
+        # Simulate finally block logic from lines 1161-1174
+        # Listener cleanup is OUTSIDE the `if not config.session_id` block
+        try:
+            if config.capture_network_requests:
+                page.remove_listener("request", handle_request)
+                page.remove_listener("response", handle_response)
+                page.remove_listener("requestfailed", handle_failed)
+        except Exception:
+            pass
+
+        assert page.remove_listener.call_count == 3, (
+            "All three event listeners should be removed even with session_id"
+        )
+
+    async def test_page_not_closed_when_session_id_set(self):
+        """When session_id is set, the page should NOT be closed."""
+        page = AsyncMock()
+        config = self._make_crawl_mocks(session_id="my-session")
+
+        # Simulate the finally block logic from lines 1176-1191
+        closed = False
+        if not config.session_id:
+            closed = True
+
+        assert not closed, "Page should NOT be closed when session_id is set"
+
+    async def test_page_closed_when_no_session_id(self):
+        """When session_id is not set, the page IS closed."""
+        page = AsyncMock()
+        config = self._make_crawl_mocks(session_id=None)
+
+        closed = False
+        if not config.session_id:
+            closed = True
+
+        assert closed, "Page should be closed when session_id is not set"
+
+    async def test_source_code_listener_cleanup_outside_session_block(self):
+        """Verify in source that listener removal is in the finally block,
+        outside the 'if not config.session_id' guard."""
+        import crawl4ai.async_crawler_strategy as mod
+        source = inspect.getsource(mod)
+
+        # Find the finally block containing remove_listener and check that
+        # remove_listener comes before the "if not config.session_id" block
+        finally_pos = source.find("finally:")
+        assert finally_pos != -1, "Should have a finally block"
+
+        remove_listener_pos = source.find("remove_listener", finally_pos)
+        assert remove_listener_pos != -1, "Should have remove_listener after finally"
+
+        session_check_pos = source.find("if not config.session_id", finally_pos)
+        assert session_check_pos != -1, "Should have session_id check after finally"
+
+        assert remove_listener_pos < session_check_pos, (
+            "remove_listener should appear BEFORE the 'if not config.session_id' "
+            "block so listeners are always cleaned up"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Issue #1509 — deep_crawl_strategy in arun_many
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+class TestIssue1509_DeepCrawlArunMany:
+    """Verify arun_many bypasses dispatcher when deep_crawl_strategy is set."""
+
+    async def test_arun_many_with_deep_crawl_calls_arun_per_url(self):
+        """With deep_crawl_strategy, arun_many should call arun() for each URL."""
+        from crawl4ai.async_webcrawler import AsyncWebCrawler
+        from crawl4ai.async_configs import BrowserConfig
+
+        crawler = AsyncWebCrawler(config=BrowserConfig())
+        # Avoid real initialization
+        crawler._setup_done = True
+
+        config = MagicMock()
+        config.deep_crawl_strategy = MagicMock()  # truthy
+        config.stream = False
+
+        urls = ["https://a.com", "https://b.com", "https://c.com"]
+
+        result_a = [MagicMock(url="https://a.com/1"), MagicMock(url="https://a.com/2")]
+        result_b = [MagicMock(url="https://b.com/1")]
+        result_c = [MagicMock(url="https://c.com/1"), MagicMock(url="https://c.com/2")]
+
+        crawler.arun = AsyncMock(side_effect=[result_a, result_b, result_c])
+
+        results = await crawler.arun_many(urls=urls, config=config)
+
+        assert crawler.arun.call_count == 3
+        # All results should be flattened
+        assert len(results) == 5
+
+    async def test_arun_many_deep_crawl_results_flattened(self):
+        """Results from multiple deep-crawl URLs should be flattened into one list."""
+        from crawl4ai.async_webcrawler import AsyncWebCrawler
+        from crawl4ai.async_configs import BrowserConfig
+
+        crawler = AsyncWebCrawler(config=BrowserConfig())
+        crawler._setup_done = True
+
+        config = MagicMock()
+        config.deep_crawl_strategy = MagicMock()
+        config.stream = False
+
+        urls = ["https://x.com", "https://y.com"]
+        r1 = MagicMock(url="https://x.com/page1")
+        r2 = MagicMock(url="https://x.com/page2")
+        r3 = MagicMock(url="https://y.com/page1")
+
+        crawler.arun = AsyncMock(side_effect=[[r1, r2], [r3]])
+
+        results = await crawler.arun_many(urls=urls, config=config)
+        assert results == [r1, r2, r3]
+
+    async def test_arun_many_deep_crawl_streaming(self):
+        """In streaming mode with deep_crawl_strategy, results are yielded."""
+        from crawl4ai.async_webcrawler import AsyncWebCrawler
+        from crawl4ai.async_configs import BrowserConfig
+
+        crawler = AsyncWebCrawler(config=BrowserConfig())
+        crawler._setup_done = True
+
+        config = MagicMock()
+        config.deep_crawl_strategy = MagicMock()
+        config.stream = True
+
+        urls = ["https://a.com", "https://b.com"]
+        r1 = MagicMock(url="a1")
+        r2 = MagicMock(url="a2")
+        r3 = MagicMock(url="b1")
+
+        crawler.arun = AsyncMock(side_effect=[[r1, r2], [r3]])
+
+        gen = await crawler.arun_many(urls=urls, config=config)
+
+        collected = []
+        async for item in gen:
+            collected.append(item)
+
+        assert len(collected) == 3
+        assert collected == [r1, r2, r3]
+
+    async def test_arun_many_without_deep_crawl_uses_dispatcher(self):
+        """Without deep_crawl_strategy, arun_many should use the dispatcher."""
+        from crawl4ai.async_webcrawler import AsyncWebCrawler
+        from crawl4ai.async_configs import BrowserConfig
+
+        crawler = AsyncWebCrawler(config=BrowserConfig())
+        crawler._setup_done = True
+
+        config = MagicMock()
+        config.deep_crawl_strategy = None
+        config.stream = False
+        config.mean_delay = 0.1
+        config.max_range = 0.3
+        config.proxy_session_id = None
+
+        urls = ["https://a.com", "https://b.com"]
+
+        dispatcher = MagicMock()
+        task_result_a = MagicMock()
+        task_result_a.result = MagicMock(url="https://a.com")
+        task_result_a.result.dispatch_result = None
+        task_result_a.task_id = "task-1"
+        task_result_a.memory_usage = 100.0
+        task_result_a.peak_memory = 200.0
+        task_result_a.start_time = 0.0
+        task_result_a.end_time = 1.0
+        task_result_a.error_message = ""
+
+        task_result_b = MagicMock()
+        task_result_b.result = MagicMock(url="https://b.com")
+        task_result_b.result.dispatch_result = None
+        task_result_b.task_id = "task-2"
+        task_result_b.memory_usage = 100.0
+        task_result_b.peak_memory = 200.0
+        task_result_b.start_time = 0.0
+        task_result_b.end_time = 1.0
+        task_result_b.error_message = ""
+
+        dispatcher.run_urls = AsyncMock(return_value=[task_result_a, task_result_b])
+
+        results = await crawler.arun_many(urls=urls, config=config, dispatcher=dispatcher)
+
+        dispatcher.run_urls.assert_called_once()
+        assert len(results) == 2
diff --git a/tests/test_issue_1484_css_selector.py b/tests/test_issue_1484_css_selector.py
new file mode 100644
index 0000000..e083514
--- /dev/null
+++ b/tests/test_issue_1484_css_selector.py
@@ -0,0 +1,167 @@
+"""Tests for issue #1484: css_selector doesn't work but target_elements does.
+
+The bug: When using raw:// URLs, css_selector was accepted by _scrap() but
+never applied. Only target_elements worked.
+"""
+
+import pytest
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+
+
+SAMPLE_HTML = """
+
+
+    
      Header content
      
+    
      
+        
      Main paragraph one with enough words to pass threshold easily here.
      
+        
      Main paragraph two with enough words to pass threshold easily here.
      
+    
      
+    
      
+        
      Sidebar content with enough words to pass the word count threshold easily.
      
+    
      
+    
      Footer content here
      
+
+
+"""
+
+NESTED_HTML = """
+
+
+    
      
+        
      
+            
      
+                
      Drawer content with enough words to pass threshold filter check.
      
+            
      
+        
      
+        
      
+            
      Other content with enough words to pass threshold filter easily here.
      
+        
      
+    
      
+
+
+"""
+
+
+@pytest.fixture
+def scraper():
+    return LXMLWebScrapingStrategy()
+
+
+class TestCssSelectorBasic:
+    def test_css_selector_filters_content(self, scraper):
+        """css_selector should restrict content to matching elements."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=SAMPLE_HTML,
+            css_selector=".main-content",
+        )
+        assert result is not None
+        text = result.get("markdown", "") or result.get("cleaned_html", "")
+        assert "Main paragraph" in text
+        assert "Sidebar content" not in text
+
+    def test_css_selector_none_returns_full_content(self, scraper):
+        """No css_selector should return all content."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=SAMPLE_HTML,
+            css_selector=None,
+        )
+        assert result is not None
+        html = result.get("cleaned_html", "")
+        assert "Main paragraph" in html
+        assert "Sidebar content" in html
+
+    def test_css_selector_no_match_returns_full_content(self, scraper):
+        """Non-matching css_selector should fall back to full content."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=SAMPLE_HTML,
+            css_selector=".nonexistent-class",
+        )
+        assert result is not None
+        html = result.get("cleaned_html", "")
+        assert "Main paragraph" in html
+
+    def test_css_selector_invalid_falls_back(self, scraper):
+        """Invalid css_selector should fall back gracefully."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=SAMPLE_HTML,
+            css_selector="[[[invalid",
+        )
+        assert result is not None
+
+    def test_css_selector_nested_selector(self, scraper):
+        """Nested CSS selectors like '.el-drawer .drawer-body' should work."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=NESTED_HTML,
+            css_selector=".el-drawer .drawer-body",
+        )
+        assert result is not None
+        html = result.get("cleaned_html", "")
+        assert "Drawer content" in html
+        assert "Other content" not in html
+
+
+class TestCssSelectorWithTargetElements:
+    def test_css_selector_combined_with_target_elements(self, scraper):
+        """css_selector and target_elements together should narrow content."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=SAMPLE_HTML,
+            css_selector=".main-content",
+            target_elements=["p"],
+        )
+        assert result is not None
+        html = result.get("cleaned_html", "")
+        assert "Main paragraph" in html
+        assert "Sidebar content" not in html
+
+    def test_target_elements_alone_still_works(self, scraper):
+        """target_elements without css_selector should still work."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=SAMPLE_HTML,
+            target_elements=[".main-content"],
+        )
+        assert result is not None
+        html = result.get("cleaned_html", "")
+        assert "Main paragraph" in html
+
+    def test_css_selector_multiple_matches(self, scraper):
+        """css_selector matching multiple elements should include all."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=SAMPLE_HTML,
+            css_selector="div p",
+        )
+        assert result is not None
+        html = result.get("cleaned_html", "")
+        assert "Main paragraph one" in html
+        assert "Main paragraph two" in html
+
+    def test_css_selector_id_selector(self, scraper):
+        """ID-based css_selector should work."""
+        html = '
      Target content here with enough words.
      Other stuff
      '
+        result = scraper._scrap(
+            url="raw://test",
+            html=html,
+            css_selector="#target",
+        )
+        assert result is not None
+        cleaned = result.get("cleaned_html", "")
+        assert "Target content" in cleaned
+
+    def test_css_selector_excludes_non_matching(self, scraper):
+        """Content outside css_selector match should not appear."""
+        result = scraper._scrap(
+            url="raw://test",
+            html=SAMPLE_HTML,
+            css_selector=".sidebar",
+        )
+        assert result is not None
+        html = result.get("cleaned_html", "")
+        assert "Sidebar content" in html
+        assert "Main paragraph" not in html
diff --git a/tests/test_issue_1594_mcp_sse.py b/tests/test_issue_1594_mcp_sse.py
new file mode 100644
index 0000000..0b53f78
--- /dev/null
+++ b/tests/test_issue_1594_mcp_sse.py
@@ -0,0 +1,78 @@
+"""
+Tests for issue #1594 — MCP SSE endpoint crash.
+
+The old code used @app.get() to mount the SSE handler, which wraps it in
+Starlette middleware.  The MCP SDK's SseServerTransport calls raw ASGI
+(scope, receive, send) internally, causing a middleware lifecycle conflict
+(AssertionError).
+
+Fix: mount via starlette.routing.Route (raw ASGI, no middleware wrapping).
+"""
+
+import inspect
+import re
+
+import pytest
+
+
+class TestMCPSSERouting:
+    """Verify mcp_bridge.py uses raw ASGI routes for SSE, not @app.get()."""
+
+    def _get_source(self):
+        import importlib.util
+        spec = importlib.util.spec_from_file_location(
+            "mcp_bridge", "deploy/docker/mcp_bridge.py"
+        )
+        # We just need the source, not to execute it
+        with open("deploy/docker/mcp_bridge.py", "r") as f:
+            return f.read()
+
+    def test_no_app_get_for_sse_endpoint(self):
+        """SSE endpoint must NOT use @app.get() — that causes the crash."""
+        source = self._get_source()
+        # Should not have @app.get(...sse...) pattern
+        assert not re.search(
+            r'@app\.get\([^)]*sse[^)]*\)', source
+        ), "SSE endpoint must not use @app.get() — causes AssertionError (#1594)"
+
+    def test_uses_starlette_route_for_sse(self):
+        """SSE endpoint should use starlette.routing.Route (raw ASGI)."""
+        source = self._get_source()
+        assert "from starlette.routing import Route" in source or \
+               "from starlette.routing import Route, Mount" in source, \
+            "Should import Route from starlette.routing"
+        assert re.search(r'Route\([^)]*sse[^)]*\)', source), \
+            "SSE endpoint should be mounted via Route()"
+
+    def test_uses_mount_for_messages(self):
+        """SSE messages endpoint should use Mount(), not app.mount()."""
+        source = self._get_source()
+        assert re.search(r'Mount\([^)]*messages[^)]*\)', source), \
+            "Messages endpoint should use Mount()"
+
+    def test_sse_handler_is_raw_asgi(self):
+        """SSE handler should accept (scope, receive, send) — raw ASGI signature."""
+        source = self._get_source()
+        # The handler function should have raw ASGI params
+        assert re.search(
+            r'async def _mcp_sse_handler\(scope,\s*receive,\s*send\)', source
+        ), "SSE handler must be raw ASGI: (scope, receive, send)"
+
+    def test_connect_sse_uses_raw_params(self):
+        """connect_sse should be called with scope, receive, send directly."""
+        source = self._get_source()
+        assert re.search(
+            r'sse\.connect_sse\(scope,\s*receive,\s*send\)', source
+        ), "connect_sse must use (scope, receive, send), not request._send"
+
+    def test_no_request_send_usage(self):
+        """Must not use request._send — that's a private Starlette attribute."""
+        source = self._get_source()
+        assert "request._send" not in source, \
+            "Must not use request._send (private attribute, fragile)"
+
+    def test_routes_appended_not_decorated(self):
+        """Routes should be appended to app.routes, not decorated."""
+        source = self._get_source()
+        assert "app.routes.append" in source, \
+            "SSE routes should be appended via app.routes.append()"
diff --git a/tests/test_issue_1611_llm_provider.py b/tests/test_issue_1611_llm_provider.py
new file mode 100644
index 0000000..157d83b
--- /dev/null
+++ b/tests/test_issue_1611_llm_provider.py
@@ -0,0 +1,114 @@
+"""Tests for issue #1611: Docker API /llm endpoint ignores per-request provider.
+
+The bug: /llm endpoint hardcoded config["llm"]["provider"] without accepting
+per-request overrides. Fixed by adding provider/temperature/base_url query params.
+"""
+
+import pytest
+import sys
+import os
+import inspect
+
+# Add deploy/docker to path so we can import api.py
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'deploy', 'docker'))
+
+
+class TestHandleLlmQaSignature:
+    """Verify handle_llm_qa accepts per-request override parameters."""
+
+    def test_handle_llm_qa_accepts_provider(self):
+        from api import handle_llm_qa
+        sig = inspect.signature(handle_llm_qa)
+        assert "provider" in sig.parameters
+        assert sig.parameters["provider"].default is None
+
+    def test_handle_llm_qa_accepts_temperature(self):
+        from api import handle_llm_qa
+        sig = inspect.signature(handle_llm_qa)
+        assert "temperature" in sig.parameters
+        assert sig.parameters["temperature"].default is None
+
+    def test_handle_llm_qa_accepts_base_url(self):
+        from api import handle_llm_qa
+        sig = inspect.signature(handle_llm_qa)
+        assert "base_url" in sig.parameters
+        assert sig.parameters["base_url"].default is None
+
+    def test_handle_llm_qa_backward_compatible(self):
+        """Calling with just (url, query, config) should still work."""
+        from api import handle_llm_qa
+        sig = inspect.signature(handle_llm_qa)
+        # First 3 params are positional, rest have defaults
+        required = [
+            p for p in sig.parameters.values()
+            if p.default is inspect.Parameter.empty
+        ]
+        assert len(required) == 3  # url, query, config
+
+
+class TestBuildRedisUrl:
+    """Test Redis URL construction from config and env vars."""
+
+    def _build(self, config, env=None):
+        # Import the function
+        sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'deploy', 'docker'))
+        # We can't easily import from server.py without FastAPI setup,
+        # so we replicate the logic for testing
+        rc = config.get("redis", {})
+        host = (env or {}).get("REDIS_HOST", rc.get("host", "localhost"))
+        port = (env or {}).get("REDIS_PORT", rc.get("port", 6379))
+        password = (env or {}).get("REDIS_PASSWORD", rc.get("password", ""))
+        db = rc.get("db", 0)
+        scheme = "rediss" if rc.get("ssl", False) else "redis"
+        auth = f":{password}@" if password else ""
+        return f"{scheme}://{auth}{host}:{port}/{db}"
+
+    def test_default_config(self):
+        config = {"redis": {"host": "localhost", "port": 6379, "db": 0, "password": ""}}
+        assert self._build(config) == "redis://localhost:6379/0"
+
+    def test_custom_host_port(self):
+        config = {"redis": {"host": "redis-server", "port": 6380, "db": 2, "password": ""}}
+        assert self._build(config) == "redis://redis-server:6380/2"
+
+    def test_password_in_config(self):
+        config = {"redis": {"host": "localhost", "port": 6379, "db": 0, "password": "secret123"}}
+        url = self._build(config)
+        assert url == "redis://:secret123@localhost:6379/0"
+
+    def test_env_overrides_config(self):
+        config = {"redis": {"host": "localhost", "port": 6379, "db": 0, "password": ""}}
+        env = {"REDIS_HOST": "remote-redis", "REDIS_PORT": "6380", "REDIS_PASSWORD": "envpass"}
+        url = self._build(config, env)
+        assert url == "redis://:envpass@remote-redis:6380/0"
+
+    def test_ssl_uses_rediss_scheme(self):
+        config = {"redis": {"host": "localhost", "port": 6379, "db": 0, "password": "", "ssl": True}}
+        url = self._build(config)
+        assert url.startswith("rediss://")
+
+    def test_no_ssl_uses_redis_scheme(self):
+        config = {"redis": {"host": "localhost", "port": 6379, "db": 0, "password": "", "ssl": False}}
+        url = self._build(config)
+        assert url.startswith("redis://")
+
+    def test_empty_config_uses_defaults(self):
+        config = {"redis": {}}
+        url = self._build(config)
+        assert url == "redis://localhost:6379/0"
+
+    def test_missing_redis_key_uses_defaults(self):
+        config = {}
+        url = self._build(config)
+        assert url == "redis://localhost:6379/0"
+
+    def test_password_with_special_chars(self):
+        config = {"redis": {"host": "localhost", "port": 6379, "db": 0, "password": "p@ss:w0rd"}}
+        url = self._build(config)
+        assert ":p@ss:w0rd@" in url
+
+    def test_env_password_only(self):
+        config = {"redis": {"host": "localhost", "port": 6379, "db": 0, "password": ""}}
+        env = {"REDIS_PASSWORD": "fromenv"}
+        url = self._build(config, env)
+        assert ":fromenv@" in url
diff --git a/tests/test_issue_1748_screenshot_scroll_delay.py b/tests/test_issue_1748_screenshot_scroll_delay.py
new file mode 100644
index 0000000..7d93001
--- /dev/null
+++ b/tests/test_issue_1748_screenshot_scroll_delay.py
@@ -0,0 +1,261 @@
+"""
+Tests for GitHub issue #1748: scroll_delay config is now properly respected
+in take_screenshot_scroller().
+
+Three changes were made to async_crawler_strategy.py:
+  A) arun call site now passes scroll_delay from config
+  B) _generate_media_from_html call site now passes scroll_delay from config
+  C) take_screenshot_scroller reads scroll_delay from kwargs (was hardcoded 0.01)
+
+These tests verify that all three paths correctly forward and use scroll_delay.
+"""
+
+import pytest
+import asyncio
+import base64
+from io import BytesIO
+from unittest.mock import AsyncMock, MagicMock, patch, call
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_tiny_jpeg() -> bytes:
+    """Create a minimal valid JPEG image for mock screenshot returns."""
+    from PIL import Image
+
+    img = Image.new("RGB", (10, 10), color="red")
+    buf = BytesIO()
+    img.save(buf, format="JPEG")
+    return buf.getvalue()
+
+
+TINY_JPEG = _make_tiny_jpeg()
+
+# A tall HTML page that exceeds any reasonable screenshot_height_threshold
+TALL_HTML = "" + "
      Line of content
      " * 200 + ""
+
+
+def _make_mock_page(viewport_width=1280, viewport_height=200):
+    """Create a mock Playwright page with the essentials for take_screenshot_scroller."""
+    page = MagicMock()
+    page.viewport_size = {"width": viewport_width, "height": viewport_height}
+    page.set_viewport_size = AsyncMock()
+    page.evaluate = AsyncMock(return_value=None)
+    page.screenshot = AsyncMock(return_value=TINY_JPEG)
+    return page
+
+
+# ---------------------------------------------------------------------------
+# Test 1 — Unit: scroll_delay extracted from kwargs correctly
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_scroll_delay_custom_value_used():
+    """
+    When scroll_delay=1.5 is passed in kwargs, asyncio.sleep must be called
+    with 1.5 — NOT with the old hardcoded 0.01.
+    """
+    strategy = AsyncPlaywrightCrawlerStrategy.__new__(AsyncPlaywrightCrawlerStrategy)
+    # Minimal attributes needed by take_screenshot_scroller
+    strategy.logger = MagicMock()
+    strategy.adapter = MagicMock()
+
+    page = _make_mock_page()
+
+    # get_page_dimensions returns a page taller than the viewport
+    strategy.get_page_dimensions = AsyncMock(
+        return_value={"width": 1280, "height": 600}
+    )
+
+    with patch(
+        "crawl4ai.async_crawler_strategy.asyncio.sleep", new_callable=AsyncMock
+    ) as mock_sleep:
+        result = await strategy.take_screenshot_scroller(
+            page, scroll_delay=1.5, screenshot_height_threshold=100
+        )
+
+    # asyncio.sleep must have been called with our custom value
+    sleep_args = [c.args[0] for c in mock_sleep.call_args_list]
+    assert 1.5 in sleep_args, (
+        f"Expected asyncio.sleep(1.5) but got calls with: {sleep_args}"
+    )
+    # The old hardcoded 0.01 must NOT appear
+    assert 0.01 not in sleep_args, (
+        f"Old hardcoded 0.01 still present in sleep calls: {sleep_args}"
+    )
+    # Should return a base64-encoded string
+    assert isinstance(result, str)
+    assert len(result) > 0
+
+
+# ---------------------------------------------------------------------------
+# Test 2 — Unit: default scroll_delay is 0.2 when not provided
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_scroll_delay_default_value():
+    """
+    When scroll_delay is NOT provided in kwargs, asyncio.sleep must be called
+    with 0.2 (the correct default), NOT 0.01.
+    """
+    strategy = AsyncPlaywrightCrawlerStrategy.__new__(AsyncPlaywrightCrawlerStrategy)
+    strategy.logger = MagicMock()
+    strategy.adapter = MagicMock()
+
+    page = _make_mock_page()
+
+    strategy.get_page_dimensions = AsyncMock(
+        return_value={"width": 1280, "height": 600}
+    )
+
+    with patch(
+        "crawl4ai.async_crawler_strategy.asyncio.sleep", new_callable=AsyncMock
+    ) as mock_sleep:
+        result = await strategy.take_screenshot_scroller(
+            page, screenshot_height_threshold=100
+        )
+
+    sleep_args = [c.args[0] for c in mock_sleep.call_args_list]
+    assert 0.2 in sleep_args, (
+        f"Expected default asyncio.sleep(0.2) but got calls with: {sleep_args}"
+    )
+    assert 0.01 not in sleep_args, (
+        f"Old hardcoded 0.01 still present in sleep calls: {sleep_args}"
+    )
+    assert isinstance(result, str)
+
+
+# ---------------------------------------------------------------------------
+# Test 3 — Unit: take_screenshot forwards scroll_delay to take_screenshot_scroller
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_take_screenshot_forwards_scroll_delay():
+    """
+    When take_screenshot is called with scroll_delay=2.5 in kwargs and the page
+    needs scrolling, it must pass that value through to take_screenshot_scroller.
+    """
+    strategy = AsyncPlaywrightCrawlerStrategy.__new__(AsyncPlaywrightCrawlerStrategy)
+    strategy.logger = MagicMock()
+    strategy.adapter = MagicMock()
+
+    page = _make_mock_page()
+
+    # page_need_scroll returns True so the scroller path is taken
+    strategy.page_need_scroll = AsyncMock(return_value=True)
+    strategy.take_screenshot_scroller = AsyncMock(return_value="base64data")
+
+    await strategy.take_screenshot(page, scroll_delay=2.5)
+
+    # Verify take_screenshot_scroller was called with scroll_delay in kwargs
+    strategy.take_screenshot_scroller.assert_called_once()
+    call_kwargs = strategy.take_screenshot_scroller.call_args
+    # kwargs are passed through via **kwargs
+    assert call_kwargs.kwargs.get("scroll_delay") == 2.5 or (
+        len(call_kwargs.args) > 1 and False
+    ), f"scroll_delay=2.5 not forwarded. Call was: {call_kwargs}"
+
+
+# ---------------------------------------------------------------------------
+# Test 4 — Integration: full-page screenshot with custom scroll_delay
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_integration_arun_respects_scroll_delay():
+    """
+    End-to-end: use AsyncWebCrawler with a raw: tall HTML page and a very low
+    screenshot_height_threshold to force the scroller path. Verify asyncio.sleep
+    is called with the configured scroll_delay, not 0.01.
+    """
+    config = CrawlerRunConfig(
+        screenshot=True,
+        scroll_delay=0.5,
+        screenshot_height_threshold=100,  # Very low to force scroller
+    )
+
+    with patch(
+        "crawl4ai.async_crawler_strategy.asyncio.sleep", new_callable=AsyncMock
+    ) as mock_sleep:
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(f"raw:{TALL_HTML}", config=config)
+
+    assert result.success, f"Crawl failed: {result.error_message}"
+    assert result.screenshot is not None, "Expected screenshot data"
+
+    # Check that our custom scroll_delay was used during screenshot capture
+    sleep_args = [c.args[0] for c in mock_sleep.call_args_list]
+    assert 0.5 in sleep_args, (
+        f"Expected asyncio.sleep(0.5) in screenshot capture but got: {sleep_args}"
+    )
+    assert 0.01 not in sleep_args, (
+        f"Old hardcoded 0.01 still present in sleep calls: {sleep_args}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Test 5 — Integration: _generate_media_from_html respects scroll_delay
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_integration_generate_media_respects_scroll_delay():
+    """
+    Call _generate_media_from_html directly with a config that has
+    scroll_delay=0.75 and screenshot=True. Verify asyncio.sleep is called
+    with 0.75 during screenshot capture.
+    """
+    config = CrawlerRunConfig(
+        screenshot=True,
+        scroll_delay=0.75,
+        screenshot_height_threshold=100,  # Very low to force scroller
+    )
+
+    with patch(
+        "crawl4ai.async_crawler_strategy.asyncio.sleep", new_callable=AsyncMock
+    ) as mock_sleep:
+        async with AsyncWebCrawler() as crawler:
+            (
+                screenshot_data,
+                pdf_data,
+                mhtml_data,
+            ) = await crawler.crawler_strategy._generate_media_from_html(
+                TALL_HTML, config
+            )
+
+    assert screenshot_data is not None, (
+        "Expected screenshot data from _generate_media_from_html"
+    )
+
+    sleep_args = [c.args[0] for c in mock_sleep.call_args_list]
+    assert 0.75 in sleep_args, f"Expected asyncio.sleep(0.75) but got: {sleep_args}"
+    assert 0.01 not in sleep_args, (
+        f"Old hardcoded 0.01 still present in sleep calls: {sleep_args}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Test 6 — Unit: CrawlerRunConfig default scroll_delay is 0.2
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_crawler_run_config_default_scroll_delay():
+    """CrawlerRunConfig.scroll_delay defaults to 0.2."""
+    config = CrawlerRunConfig()
+    assert config.scroll_delay == 0.2, (
+        f"Expected default scroll_delay=0.2, got {config.scroll_delay}"
+    )
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])
diff --git a/tests/test_issue_1750_screenshot_scan_full_page.py b/tests/test_issue_1750_screenshot_scan_full_page.py
new file mode 100644
index 0000000..a7b7bd9
--- /dev/null
+++ b/tests/test_issue_1750_screenshot_scan_full_page.py
@@ -0,0 +1,249 @@
+"""
+Tests for issue #1750: Screenshot size should respect scan_full_page setting.
+
+When scan_full_page=False, screenshots should capture only the viewport,
+not the entire scrollable page.
+"""
+
+import asyncio
+import base64
+import pytest
+from io import BytesIO
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+
+
+# ---------------------------------------------------------------------------
+# Helper
+# ---------------------------------------------------------------------------
+
+def get_image_dimensions(screenshot_b64: str) -> tuple:
+    """Decode a base64 screenshot and return (width, height)."""
+    from PIL import Image
+    img_data = base64.b64decode(screenshot_b64)
+    img = Image.open(BytesIO(img_data))
+    return img.width, img.height
+
+
+# ---------------------------------------------------------------------------
+# Unit tests (mock-based, no browser needed)
+# ---------------------------------------------------------------------------
+
+class TestTakeScreenshotRouting:
+    """Unit tests for take_screenshot routing logic."""
+
+    @pytest.fixture
+    def strategy(self):
+        """Create a minimal AsyncPlaywrightCrawlerStrategy with mocked methods."""
+        from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+        s = object.__new__(AsyncPlaywrightCrawlerStrategy)
+        s.logger = MagicMock()
+        s.take_screenshot_naive = AsyncMock(return_value="naive_b64")
+        s.take_screenshot_scroller = AsyncMock(return_value="scroller_b64")
+        s.page_need_scroll = AsyncMock(return_value=True)
+        return s
+
+    @pytest.mark.asyncio
+    async def test_scan_full_page_false_uses_naive(self, strategy):
+        """scan_full_page=False should always use viewport (naive) screenshot."""
+        page = MagicMock()
+        result = await strategy.take_screenshot(page, scan_full_page=False)
+        assert result == "naive_b64"
+        strategy.take_screenshot_naive.assert_awaited_once_with(page)
+        strategy.take_screenshot_scroller.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_scan_full_page_true_scrollable_uses_scroller(self, strategy):
+        """scan_full_page=True on a scrollable page should use scroller."""
+        page = MagicMock()
+        result = await strategy.take_screenshot(page, scan_full_page=True)
+        assert result == "scroller_b64"
+        strategy.take_screenshot_scroller.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    async def test_scan_full_page_true_short_page_uses_naive(self, strategy):
+        """scan_full_page=True on a short page should still use naive."""
+        strategy.page_need_scroll = AsyncMock(return_value=False)
+        page = MagicMock()
+        result = await strategy.take_screenshot(page, scan_full_page=True)
+        assert result == "naive_b64"
+
+    @pytest.mark.asyncio
+    async def test_default_scan_full_page_is_true(self, strategy):
+        """When scan_full_page is not passed, default to True (full page)."""
+        page = MagicMock()
+        result = await strategy.take_screenshot(page)
+        # Should go to scroller since page_need_scroll=True and default is True
+        assert result == "scroller_b64"
+
+    @pytest.mark.asyncio
+    async def test_force_viewport_overrides_scan_full_page_true(self, strategy):
+        """force_viewport_screenshot=True should use naive even with scan_full_page=True."""
+        page = MagicMock()
+        result = await strategy.take_screenshot(
+            page, force_viewport_screenshot=True, scan_full_page=True
+        )
+        assert result == "naive_b64"
+
+    @pytest.mark.asyncio
+    async def test_scan_full_page_false_on_short_page(self, strategy):
+        """scan_full_page=False on a short page should use naive (no regression)."""
+        strategy.page_need_scroll = AsyncMock(return_value=False)
+        page = MagicMock()
+        result = await strategy.take_screenshot(page, scan_full_page=False)
+        assert result == "naive_b64"
+
+    @pytest.mark.asyncio
+    async def test_scan_full_page_false_does_not_call_page_need_scroll(self, strategy):
+        """When scan_full_page=False, we should skip the scroll check entirely."""
+        page = MagicMock()
+        await strategy.take_screenshot(page, scan_full_page=False)
+        strategy.page_need_scroll.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_force_viewport_false_scan_full_page_false(self, strategy):
+        """force_viewport=False + scan_full_page=False should still use naive."""
+        page = MagicMock()
+        result = await strategy.take_screenshot(
+            page, force_viewport_screenshot=False, scan_full_page=False
+        )
+        assert result == "naive_b64"
+
+
+# ---------------------------------------------------------------------------
+# Integration tests (real browser)
+# ---------------------------------------------------------------------------
+
+TALL_PAGE_HTML = """
+
+
+
      
+    
      Tall page for screenshot testing
      
+
      
+
+
+"""
+
+SHORT_PAGE_HTML = """
+
+
+
      
+    
      Short page
      
+
      
+
+
+"""
+
+
+@pytest.fixture(scope="module")
+def event_loop():
+    loop = asyncio.new_event_loop()
+    yield loop
+    loop.close()
+
+
+class TestScreenshotIntegration:
+    """Integration tests using real browser with raw:// HTML pages."""
+
+    VIEWPORT_W = 800
+    VIEWPORT_H = 600
+
+    @pytest.fixture(scope="class")
+    def browser_config(self):
+        return BrowserConfig(
+            viewport_width=self.VIEWPORT_W,
+            viewport_height=self.VIEWPORT_H,
+            headless=True,
+        )
+
+    @pytest.mark.asyncio
+    async def test_tall_page_scan_full_page_false(self, browser_config):
+        """Tall page + scan_full_page=False -> viewport-sized screenshot."""
+        config = CrawlerRunConfig(screenshot=True, scan_full_page=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=f"raw://{TALL_PAGE_HTML}", config=config)
+        assert result.screenshot is not None
+        w, h = get_image_dimensions(result.screenshot)
+        assert w == self.VIEWPORT_W
+        assert h == self.VIEWPORT_H, (
+            f"Expected viewport height {self.VIEWPORT_H}, got {h}"
+        )
+
+    @pytest.mark.asyncio
+    async def test_tall_page_scan_full_page_true(self, browser_config):
+        """Tall page + scan_full_page=True -> full page screenshot (taller than viewport)."""
+        config = CrawlerRunConfig(screenshot=True, scan_full_page=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=f"raw://{TALL_PAGE_HTML}", config=config)
+        assert result.screenshot is not None
+        w, h = get_image_dimensions(result.screenshot)
+        assert h > self.VIEWPORT_H, (
+            f"Expected full-page screenshot taller than {self.VIEWPORT_H}, got {h}"
+        )
+
+    @pytest.mark.asyncio
+    async def test_tall_page_default_scan_full_page(self, browser_config):
+        """Default config (scan_full_page=False by default) -> viewport-sized screenshot."""
+        config = CrawlerRunConfig(screenshot=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=f"raw://{TALL_PAGE_HTML}", config=config)
+        assert result.screenshot is not None
+        w, h = get_image_dimensions(result.screenshot)
+        # Default scan_full_page is False, so screenshot should be viewport-sized
+        assert h == self.VIEWPORT_H
+
+    @pytest.mark.asyncio
+    async def test_short_page_scan_full_page_false(self, browser_config):
+        """Short page + scan_full_page=False -> viewport-sized screenshot."""
+        config = CrawlerRunConfig(screenshot=True, scan_full_page=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=f"raw://{SHORT_PAGE_HTML}", config=config)
+        assert result.screenshot is not None
+        w, h = get_image_dimensions(result.screenshot)
+        assert h == self.VIEWPORT_H
+
+    @pytest.mark.asyncio
+    async def test_short_page_scan_full_page_true(self, browser_config):
+        """Short page + scan_full_page=True -> should still be viewport-sized (no scroll needed)."""
+        config = CrawlerRunConfig(screenshot=True, scan_full_page=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=f"raw://{SHORT_PAGE_HTML}", config=config)
+        assert result.screenshot is not None
+        w, h = get_image_dimensions(result.screenshot)
+        # Short page doesn't need scrolling, so screenshot should be viewport-sized
+        assert h <= self.VIEWPORT_H + 50  # small tolerance
+
+    @pytest.mark.asyncio
+    async def test_force_viewport_overrides_on_tall_page(self, browser_config):
+        """force_viewport_screenshot=True should give viewport size even with scan_full_page=True."""
+        config = CrawlerRunConfig(
+            screenshot=True,
+            scan_full_page=True,
+            force_viewport_screenshot=True,
+        )
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=f"raw://{TALL_PAGE_HTML}", config=config)
+        assert result.screenshot is not None
+        w, h = get_image_dimensions(result.screenshot)
+        assert h == self.VIEWPORT_H
+
+    @pytest.mark.asyncio
+    async def test_screenshot_width_always_matches_viewport(self, browser_config):
+        """Width should always match viewport regardless of scan_full_page setting."""
+        for scan_full in [True, False]:
+            config = CrawlerRunConfig(screenshot=True, scan_full_page=scan_full)
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                result = await crawler.arun(url=f"raw://{TALL_PAGE_HTML}", config=config)
+            w, h = get_image_dimensions(result.screenshot)
+            assert w == self.VIEWPORT_W, (
+                f"scan_full_page={scan_full}: width {w} != viewport {self.VIEWPORT_W}"
+            )
+
+    @pytest.mark.asyncio
+    async def test_no_screenshot_when_disabled(self, browser_config):
+        """screenshot=False should return no screenshot regardless of scan_full_page."""
+        config = CrawlerRunConfig(screenshot=False, scan_full_page=False)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(url=f"raw://{TALL_PAGE_HTML}", config=config)
+        assert not result.screenshot
diff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.py
new file mode 100644
index 0000000..45150e5
--- /dev/null
+++ b/tests/test_issue_1837_config_list.py
@@ -0,0 +1,140 @@
+"""
+Tests for issue #1837: Docker API arun_many config-list support.
+
+Verifies that the /crawl endpoint accepts crawler_configs (list of dicts)
+alongside the existing crawler_config (single dict), and that the list
+is correctly passed through to arun_many().
+"""
+
+import pytest
+from unittest.mock import AsyncMock, patch, MagicMock
+from crawl4ai import CrawlerRunConfig, CacheMode
+
+
+# -- Schema tests --
+
+class TestCrawlRequestSchema:
+    """Verify CrawlRequest schema accepts crawler_configs."""
+
+    def test_schema_has_crawler_configs_field(self):
+        """CrawlRequest should have an optional crawler_configs field."""
+        import importlib.util
+        with open("deploy/docker/schemas.py") as f:
+            source = f.read()
+        assert "crawler_configs" in source
+        assert "Optional[List[Dict]]" in source
+
+    def test_schema_backward_compatible(self):
+        """crawler_config (singular) should still work."""
+        with open("deploy/docker/schemas.py") as f:
+            source = f.read()
+        assert "crawler_config: Optional[Dict]" in source
+
+    def test_crawler_configs_default_none(self):
+        """crawler_configs should default to None."""
+        with open("deploy/docker/schemas.py") as f:
+            source = f.read()
+        assert "default=None" in source
+
+
+# -- API handler tests --
+
+class TestHandleCrawlRequestSignature:
+    """Verify handle_crawl_request accepts crawler_configs parameter."""
+
+    def test_handler_accepts_crawler_configs(self):
+        """handle_crawl_request should have crawler_configs parameter."""
+        with open("deploy/docker/api.py") as f:
+            source = f.read()
+        assert "crawler_configs: Optional[List[dict]]" in source
+
+    def test_handler_defaults_crawler_configs_none(self):
+        """crawler_configs should default to None."""
+        with open("deploy/docker/api.py") as f:
+            source = f.read()
+        assert "crawler_configs: Optional[List[dict]] = None" in source
+
+
+# -- Config list deserialization --
+
+class TestConfigListDeserialization:
+    """Verify that a list of config dicts can be deserialized."""
+
+    def test_single_config_loads(self):
+        """Single config dict should deserialize as before."""
+        data = {"type": "CrawlerRunConfig", "params": {"verbose": False}}
+        config = CrawlerRunConfig.load(data)
+        assert isinstance(config, CrawlerRunConfig)
+
+    def test_multiple_configs_load(self):
+        """Multiple config dicts should each deserialize independently."""
+        configs_data = [
+            {"type": "CrawlerRunConfig", "params": {"verbose": False}},
+            {"type": "CrawlerRunConfig", "params": {"cache_mode": {"type": "CacheMode", "params": "bypass"}}},
+        ]
+        configs = [CrawlerRunConfig.load(c) for c in configs_data]
+        assert len(configs) == 2
+        assert all(isinstance(c, CrawlerRunConfig) for c in configs)
+
+    def test_empty_config_list(self):
+        """Empty config list should produce empty list."""
+        configs = [CrawlerRunConfig.load(c) for c in []]
+        assert configs == []
+
+
+# -- Integration: config list logic in api.py --
+
+class TestConfigListLogic:
+    """Verify the branching logic for single vs list configs."""
+
+    def test_api_uses_config_list_when_provided(self):
+        """When crawler_configs is provided with multiple URLs, it should be used."""
+        with open("deploy/docker/api.py") as f:
+            source = f.read()
+        # Should check crawler_configs and build a list
+        assert "if crawler_configs and len(urls) > 1:" in source
+        assert "config_list" in source
+
+    def test_api_falls_back_to_single_config(self):
+        """When crawler_configs is None, original single-config path is used."""
+        with open("deploy/docker/api.py") as f:
+            source = f.read()
+        assert "effective_config = crawler_config" in source
+
+    def test_api_applies_base_config_to_each(self):
+        """Base config should be applied to each config in the list."""
+        with open("deploy/docker/api.py") as f:
+            source = f.read()
+        assert "for cfg in config_list:" in source
+
+
+# -- Server endpoint passes crawler_configs --
+
+class TestServerEndpoint:
+    """Verify the /crawl endpoint passes crawler_configs through."""
+
+    def test_server_passes_crawler_configs(self):
+        """The crawl endpoint should pass crawler_configs to handle_crawl_request."""
+        with open("deploy/docker/server.py") as f:
+            source = f.read()
+        assert "crawler_configs=crawl_request.crawler_configs" in source
+
+
+# -- Backward compatibility --
+
+class TestBackwardCompatibility:
+    """Ensure existing single-config requests still work."""
+
+    def test_single_url_ignores_crawler_configs(self):
+        """With a single URL, crawler_configs should be ignored (uses arun, not arun_many)."""
+        with open("deploy/docker/api.py") as f:
+            source = f.read()
+        # Single URL uses arun which only takes one config
+        assert '"arun" if len(urls) == 1 else "arun_many"' in source
+
+    def test_no_crawler_configs_uses_single(self):
+        """When crawler_configs is None, the original single config path is used."""
+        with open("deploy/docker/api.py") as f:
+            source = f.read()
+        # The else branch uses the original crawler_config
+        assert "effective_config = crawler_config" in source
diff --git a/tests/test_issue_1842_browser_none.py b/tests/test_issue_1842_browser_none.py
new file mode 100644
index 0000000..2d351ee
--- /dev/null
+++ b/tests/test_issue_1842_browser_none.py
@@ -0,0 +1,181 @@
+"""
+Tests for issue #1842: Docker Job Queue API Error: 'NoneType' object has no attribute 'new_context'
+
+Verifies that BrowserManager.create_browser_context() raises clear, correct
+RuntimeError messages when self.browser is None, distinguishing between:
+  - persistent context mode (self._launched_persistent = True)
+  - browser closed/crashed/not started (self._launched_persistent = False)
+"""
+
+import asyncio
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from crawl4ai.browser_manager import BrowserManager
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+
+@pytest.fixture
+def browser_config():
+    """Default BrowserConfig for testing."""
+    return BrowserConfig()
+
+
+@pytest.fixture
+def manager(browser_config):
+    """BrowserManager with browser=None (simulating closed/crashed state)."""
+    mgr = BrowserManager(browser_config)
+    assert mgr.browser is None  # starts as None
+    return mgr
+
+
+# ── Guard raises RuntimeError (not AttributeError) ─────────────────────
+
+class TestBrowserNoneGuard:
+    """Verify that create_browser_context raises RuntimeError when browser is None."""
+
+    @pytest.mark.asyncio
+    async def test_browser_none_raises_runtime_error(self, manager):
+        """Should raise RuntimeError, not AttributeError."""
+        with pytest.raises(RuntimeError):
+            await manager.create_browser_context()
+
+    @pytest.mark.asyncio
+    async def test_browser_none_not_attribute_error(self, manager):
+        """Must never raise AttributeError ('NoneType' has no attribute 'new_context')."""
+        with pytest.raises(RuntimeError):
+            await manager.create_browser_context()
+        # If we get here, it raised RuntimeError — not AttributeError. Good.
+
+    @pytest.mark.asyncio
+    async def test_browser_none_with_crawler_run_config(self, manager):
+        """Guard should trigger even when CrawlerRunConfig is passed."""
+        config = CrawlerRunConfig()
+        with pytest.raises(RuntimeError):
+            await manager.create_browser_context(config)
+
+
+# ── Correct error message based on cause ────────────────────────────────
+
+class TestErrorMessageAccuracy:
+    """Verify error messages correctly identify why browser is None."""
+
+    @pytest.mark.asyncio
+    async def test_persistent_context_message(self, manager):
+        """When _launched_persistent=True, message should mention persistent context."""
+        manager._launched_persistent = True
+        with pytest.raises(RuntimeError, match="use_persistent_context=True"):
+            await manager.create_browser_context()
+
+    @pytest.mark.asyncio
+    async def test_persistent_context_message_mentions_single_context(self, manager):
+        """Persistent context error should mention single shared context."""
+        manager._launched_persistent = True
+        with pytest.raises(RuntimeError, match="single shared context"):
+            await manager.create_browser_context()
+
+    @pytest.mark.asyncio
+    async def test_closed_browser_message(self, manager):
+        """When browser is closed (not persistent), message should mention closed/crashed."""
+        manager._launched_persistent = False
+        with pytest.raises(RuntimeError, match="closed.*crashed|crashed.*closed"):
+            await manager.create_browser_context()
+
+    @pytest.mark.asyncio
+    async def test_closed_browser_message_no_persistent_mention(self, manager):
+        """Non-persistent error should NOT mention use_persistent_context."""
+        manager._launched_persistent = False
+        with pytest.raises(RuntimeError) as exc_info:
+            await manager.create_browser_context()
+        assert "use_persistent_context" not in str(exc_info.value)
+
+    @pytest.mark.asyncio
+    async def test_default_state_gives_closed_message(self, manager):
+        """Default manager (never started) should give the closed/crashed message."""
+        # _launched_persistent defaults to False
+        assert manager._launched_persistent is False
+        with pytest.raises(RuntimeError, match="not available"):
+            await manager.create_browser_context()
+
+
+# ── Browser available (no guard triggered) ──────────────────────────────
+
+class TestBrowserAvailable:
+    """Verify create_browser_context works normally when browser is available."""
+
+    @pytest.mark.asyncio
+    async def test_browser_set_does_not_raise(self, manager):
+        """When browser is set, should not raise RuntimeError."""
+        mock_context = AsyncMock()
+        mock_browser = AsyncMock()
+        mock_browser.new_context = AsyncMock(return_value=mock_context)
+        manager.browser = mock_browser
+
+        context = await manager.create_browser_context()
+        assert context == mock_context
+        mock_browser.new_context.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_browser_set_with_config(self, manager):
+        """When browser is set, should work with CrawlerRunConfig."""
+        mock_context = AsyncMock()
+        mock_browser = AsyncMock()
+        mock_browser.new_context = AsyncMock(return_value=mock_context)
+        manager.browser = mock_browser
+
+        config = CrawlerRunConfig()
+        context = await manager.create_browser_context(config)
+        assert context == mock_context
+
+
+# ── Simulate Docker race condition scenario ─────────────────────────────
+
+class TestDockerRaceCondition:
+    """Simulate the scenario from issue #1842: browser becomes None during use."""
+
+    @pytest.mark.asyncio
+    async def test_browser_closed_mid_use(self, manager):
+        """Simulate browser being closed while another task tries to use it."""
+        mock_context = AsyncMock()
+        mock_browser = AsyncMock()
+        mock_browser.new_context = AsyncMock(return_value=mock_context)
+        manager.browser = mock_browser
+
+        # First call works
+        ctx = await manager.create_browser_context()
+        assert ctx == mock_context
+
+        # Simulate janitor closing the browser
+        manager.browser = None
+
+        # Second call should raise RuntimeError, not AttributeError
+        with pytest.raises(RuntimeError, match="not available"):
+            await manager.create_browser_context()
+
+    @pytest.mark.asyncio
+    async def test_concurrent_access_after_close(self, manager):
+        """Multiple concurrent calls after browser close all get RuntimeError."""
+        manager.browser = None
+        manager._launched_persistent = False
+
+        async def try_create():
+            with pytest.raises(RuntimeError, match="not available"):
+                await manager.create_browser_context()
+
+        # Run multiple concurrent attempts
+        await asyncio.gather(*[try_create() for _ in range(5)])
+
+    @pytest.mark.asyncio
+    async def test_error_type_is_not_attribute_error(self, manager):
+        """Explicitly verify the original bug (AttributeError) cannot occur."""
+        manager.browser = None
+        try:
+            await manager.create_browser_context()
+            assert False, "Should have raised"
+        except RuntimeError:
+            pass  # Expected
+        except AttributeError:
+            pytest.fail(
+                "Got AttributeError ('NoneType' has no attribute 'new_context') "
+                "— this is the original bug from issue #1842"
+            )
diff --git a/tests/test_issue_1848_logger_serialize.py b/tests/test_issue_1848_logger_serialize.py
new file mode 100644
index 0000000..283e244
--- /dev/null
+++ b/tests/test_issue_1848_logger_serialize.py
@@ -0,0 +1,222 @@
+"""
+Tests for issue #1848: ValueError on deserialization of 'Logger' type.
+
+When BFSDeepCrawlStrategy is serialized for the Docker API, its logger field
+(a logging.Logger) was serialized as {"type": "Logger", "params": {...}},
+which then failed deserialization because Logger is not in the allowlist.
+
+Fix: to_serializable_dict skips non-allowlisted types (returns None),
+and from_serializable_dict returns None for unknown types instead of raising.
+"""
+
+import logging
+import pytest
+from crawl4ai.async_configs import (
+    to_serializable_dict,
+    from_serializable_dict,
+    ALLOWED_DESERIALIZE_TYPES,
+)
+from crawl4ai import (
+    BFSDeepCrawlStrategy,
+    DFSDeepCrawlStrategy,
+    CrawlerRunConfig,
+    BrowserConfig,
+    CacheMode,
+)
+
+
+# -- Serialization: non-allowlisted types are skipped --
+
+class TestSerializationSkipsNonAllowlisted:
+    """to_serializable_dict should return None for types not in the allowlist."""
+
+    def test_logger_serialized_as_none(self):
+        logger = logging.getLogger("test")
+        result = to_serializable_dict(logger)
+        assert result is None
+
+    def test_bfs_strategy_logger_is_none_in_output(self):
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+        serialized = to_serializable_dict(strategy)
+        assert serialized["type"] == "BFSDeepCrawlStrategy"
+        # Logger should be None, not a {"type": "Logger", ...} dict
+        logger_val = serialized["params"].get("logger")
+        assert logger_val is None or logger_val is None
+
+    def test_dfs_strategy_logger_is_none_in_output(self):
+        strategy = DFSDeepCrawlStrategy(max_depth=3, max_pages=5)
+        serialized = to_serializable_dict(strategy)
+        assert serialized["type"] == "DFSDeepCrawlStrategy"
+        logger_val = serialized["params"].get("logger")
+        assert logger_val is None
+
+    def test_allowlisted_types_still_serialized(self):
+        """Types in the allowlist should serialize normally."""
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+        serialized = to_serializable_dict(strategy)
+        assert serialized["type"] == "BFSDeepCrawlStrategy"
+        assert serialized["params"]["max_depth"] == 2
+        assert serialized["params"]["max_pages"] == 10
+
+    def test_callable_serialized_as_none(self):
+        """Callables (like on_state_change) should also be skipped."""
+        async def callback(state):
+            pass
+        result = to_serializable_dict(callback)
+        assert result is None
+
+
+# -- Deserialization: unknown types return None instead of raising --
+
+class TestDeserializationSkipsUnknown:
+    """from_serializable_dict should return None for unknown types."""
+
+    def test_logger_type_returns_none(self):
+        """The exact payload from the bug report should not raise."""
+        data = {
+            "type": "Logger",
+            "params": {"name": "crawl4ai.deep_crawling.bfs_strategy"}
+        }
+        result = from_serializable_dict(data)
+        assert result is None
+
+    def test_arbitrary_unknown_type_returns_none(self):
+        data = {"type": "SomeRandomClass", "params": {"foo": "bar"}}
+        result = from_serializable_dict(data)
+        assert result is None
+
+    def test_known_types_still_deserialize(self):
+        data = {
+            "type": "BFSDeepCrawlStrategy",
+            "params": {"max_depth": 2, "max_pages": 10}
+        }
+        result = from_serializable_dict(data)
+        assert isinstance(result, BFSDeepCrawlStrategy)
+        assert result.max_depth == 2
+
+    def test_no_valueerror_raised(self):
+        """Must never raise ValueError for unknown types."""
+        data = {"type": "Logger", "params": {"name": "test"}}
+        # Should NOT raise
+        result = from_serializable_dict(data)
+        assert result is None
+
+
+# -- Roundtrip: serialize then deserialize --
+
+class TestSerializationRoundtrip:
+    """Full roundtrip should work for strategies with logger."""
+
+    def test_bfs_strategy_roundtrip(self):
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+        serialized = to_serializable_dict(strategy)
+        restored = from_serializable_dict(serialized)
+        assert isinstance(restored, BFSDeepCrawlStrategy)
+        assert restored.max_depth == 2
+        assert restored.max_pages == 10
+
+    def test_dfs_strategy_serialization_no_logger(self):
+        """DFS strategy should not serialize Logger either."""
+        strategy = DFSDeepCrawlStrategy(max_depth=3, max_pages=5)
+        serialized = to_serializable_dict(strategy)
+        assert serialized["type"] == "DFSDeepCrawlStrategy"
+        import json
+        assert '"type": "Logger"' not in json.dumps(serialized)
+
+    def test_crawler_config_with_deep_crawl_roundtrip(self):
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=strategy,
+            cache_mode=CacheMode.BYPASS,
+            verbose=False,
+        )
+        serialized = to_serializable_dict(config)
+        restored = from_serializable_dict(serialized)
+        assert isinstance(restored, CrawlerRunConfig)
+        assert isinstance(restored.deep_crawl_strategy, BFSDeepCrawlStrategy)
+        assert restored.deep_crawl_strategy.max_depth == 2
+
+    def test_browser_config_roundtrip(self):
+        config = BrowserConfig(headless=True)
+        serialized = to_serializable_dict(config)
+        restored = from_serializable_dict(serialized)
+        assert isinstance(restored, BrowserConfig)
+
+
+# -- Reporter's exact scenario --
+
+class TestReporterScenario:
+    """Reproduce the exact scenario from issue #1848."""
+
+    def test_reporter_payload_deserializes(self):
+        """The exact JSON from the bug report should work."""
+        payload = {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "deep_crawl_strategy": {
+                    "type": "BFSDeepCrawlStrategy",
+                    "params": {
+                        "max_depth": 2,
+                        "max_pages": 10,
+                        "logger": {
+                            "type": "Logger",
+                            "params": {
+                                "name": "crawl4ai.deep_crawling.bfs_strategy"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        result = from_serializable_dict(payload)
+        assert isinstance(result, CrawlerRunConfig)
+        assert isinstance(result.deep_crawl_strategy, BFSDeepCrawlStrategy)
+        assert result.deep_crawl_strategy.max_depth == 2
+        assert result.deep_crawl_strategy.max_pages == 10
+
+    def test_reporter_full_request(self):
+        """Full request payload from the bug report."""
+        crawler_config = {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "scraping_strategy": {
+                    "type": "LXMLWebScrapingStrategy",
+                    "params": {}
+                },
+                "table_extraction": {
+                    "type": "DefaultTableExtraction",
+                    "params": {}
+                },
+                "verbose": False,
+                "deep_crawl_strategy": {
+                    "type": "BFSDeepCrawlStrategy",
+                    "params": {
+                        "max_depth": 2,
+                        "max_pages": 10,
+                        "logger": {
+                            "type": "Logger",
+                            "params": {
+                                "name": "crawl4ai.deep_crawling.bfs_strategy"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        result = from_serializable_dict(crawler_config)
+        assert isinstance(result, CrawlerRunConfig)
+        assert result.deep_crawl_strategy.max_depth == 2
+
+    def test_client_side_serialization_clean(self):
+        """New client serialization should not include Logger at all."""
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=10)
+        config = CrawlerRunConfig(
+            deep_crawl_strategy=strategy,
+            verbose=False,
+        )
+        serialized = to_serializable_dict(config)
+
+        # Walk the serialized dict — no "Logger" type should appear
+        import json
+        serialized_str = json.dumps(serialized)
+        assert '"type": "Logger"' not in serialized_str
diff --git a/tests/test_issue_1850_mcp_sse.py b/tests/test_issue_1850_mcp_sse.py
new file mode 100644
index 0000000..2dd21f0
--- /dev/null
+++ b/tests/test_issue_1850_mcp_sse.py
@@ -0,0 +1,172 @@
+"""
+Tests for issue #1850: MCP SSE endpoint not working.
+
+Starlette's Route wraps async functions in request_response(), which calls
+handler(request) instead of handler(scope, receive, send). The fix uses a
+callable class to bypass this wrapping.
+
+These tests verify the ASGI routing behavior without requiring a running
+MCP server or Docker.
+"""
+
+import inspect
+import pytest
+from starlette.routing import Route
+from starlette.testclient import TestClient
+from starlette.applications import Starlette
+from starlette.responses import PlainTextResponse
+
+
+# -- Core issue: Route wrapping behavior --
+
+class TestRouteWrappingBehavior:
+    """Verify that Starlette Route wraps functions but not class instances."""
+
+    def test_async_function_is_wrapped(self):
+        """An async function endpoint gets wrapped in request_response()."""
+        async def handler(scope, receive, send):
+            pass
+
+        r = Route("/test", endpoint=handler)
+        # Route wraps it — r.app is NOT handler
+        assert r.app is not handler
+
+    def test_callable_class_is_not_wrapped(self):
+        """A callable class instance is treated as raw ASGI (not wrapped)."""
+        class Handler:
+            async def __call__(self, scope, receive, send):
+                pass
+
+        h = Handler()
+        r = Route("/test", endpoint=h)
+        # Route passes it through — r.app IS handler
+        assert r.app is h
+
+    def test_async_function_is_function(self):
+        """Confirm async def is detected as function by inspect."""
+        async def handler(scope, receive, send):
+            pass
+        assert inspect.isfunction(handler)
+
+    def test_callable_class_is_not_function(self):
+        """Confirm callable class is NOT detected as function by inspect."""
+        class Handler:
+            async def __call__(self, scope, receive, send):
+                pass
+        assert not inspect.isfunction(Handler())
+
+
+# -- ASGI handler receives correct arguments --
+
+class TestASGIHandlerArgs:
+    """Verify that the callable class receives scope/receive/send correctly."""
+
+    def test_callable_class_receives_asgi_args(self):
+        """A callable class mounted via Route should get scope, receive, send."""
+        received_args = {}
+
+        class Handler:
+            async def __call__(self, scope, receive, send):
+                received_args["scope_type"] = scope["type"]
+                response = PlainTextResponse("ok")
+                await response(scope, receive, send)
+
+        app = Starlette(routes=[Route("/test", endpoint=Handler())])
+        client = TestClient(app)
+        resp = client.get("/test")
+        assert resp.status_code == 200
+        assert received_args["scope_type"] == "http"
+
+    def test_async_function_receives_request_not_asgi(self):
+        """An async function mounted via Route gets Request, not raw ASGI."""
+        received_type = {}
+
+        async def handler(*args, **kwargs):
+            received_type["arg_count"] = len(args)
+            if args and hasattr(args[0], "url"):
+                received_type["is_request"] = True
+                return PlainTextResponse("ok")
+            received_type["is_request"] = False
+
+        app = Starlette(routes=[Route("/test", endpoint=handler)])
+        client = TestClient(app)
+        resp = client.get("/test")
+        # Starlette wraps it and passes Request object (1 arg)
+        assert received_type.get("is_request") is True
+
+
+# -- MCP bridge SSE handler structure --
+
+class TestMCPBridgeSSEHandler:
+    """Verify the mcp_bridge SSE handler is correctly structured."""
+
+    def test_mcp_bridge_uses_callable_class(self):
+        """The SSE handler in mcp_bridge should be a callable class, not a function."""
+        # Import and check the source
+        import importlib.util
+        spec = importlib.util.spec_from_file_location(
+            "mcp_bridge_check",
+            "deploy/docker/mcp_bridge.py"
+        )
+        # We can't fully import mcp_bridge (needs Docker deps), so check source
+        with open("deploy/docker/mcp_bridge.py") as f:
+            source = f.read()
+
+        # Should have a class-based handler
+        assert "class _MCPSseApp" in source
+        assert "async def __call__(self, scope, receive, send)" in source
+
+    def test_mcp_bridge_no_async_def_sse_handler(self):
+        """Should NOT have a plain async def _mcp_sse_handler."""
+        with open("deploy/docker/mcp_bridge.py") as f:
+            source = f.read()
+
+        # The old buggy pattern should be gone
+        assert "async def _mcp_sse_handler(scope, receive, send)" not in source
+
+    def test_mcp_bridge_route_uses_class_instance(self):
+        """Route should be created with _MCPSseApp() instance, not a function."""
+        with open("deploy/docker/mcp_bridge.py") as f:
+            source = f.read()
+
+        assert "_MCPSseApp()" in source
+
+
+# -- Regression: ensure Route + callable class pattern works end-to-end --
+
+class TestRouteCallableClassEndToEnd:
+    """End-to-end test that a callable class works as a Route endpoint."""
+
+    def test_sse_like_handler(self):
+        """Simulate an SSE-like raw ASGI handler via Route."""
+        class SSEHandler:
+            async def __call__(self, scope, receive, send):
+                response = PlainTextResponse(
+                    "event: endpoint\ndata: /test\n\n",
+                    media_type="text/event-stream",
+                )
+                await response(scope, receive, send)
+
+        app = Starlette(routes=[Route("/sse", endpoint=SSEHandler())])
+        client = TestClient(app)
+        resp = client.get("/sse")
+        assert resp.status_code == 200
+        assert "event: endpoint" in resp.text
+
+    def test_multiple_routes_with_mixed_handlers(self):
+        """Callable class and regular function handlers can coexist."""
+        class RawHandler:
+            async def __call__(self, scope, receive, send):
+                response = PlainTextResponse("raw")
+                await response(scope, receive, send)
+
+        async def regular_handler(request):
+            return PlainTextResponse("regular")
+
+        app = Starlette(routes=[
+            Route("/raw", endpoint=RawHandler()),
+            Route("/regular", endpoint=regular_handler),
+        ])
+        client = TestClient(app)
+        assert client.get("/raw").text == "raw"
+        assert client.get("/regular").text == "regular"
diff --git a/tests/test_link_extractor.py b/tests/test_link_extractor.py
new file mode 100644
index 0000000..1482ce0
--- /dev/null
+++ b/tests/test_link_extractor.py
@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+"""
+Test script for Link Extractor functionality
+"""
+
+from crawl4ai.models import Link
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai import LinkPreviewConfig
+import asyncio
+import sys
+import os
+
+# Add the crawl4ai directory to the path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'crawl4ai'))
+
+
+async def test_link_extractor():
+    """Test the link extractor functionality"""
+
+    print("🔗 Testing Link Extractor Functionality")
+    print("=" * 50)
+
+    # Test configuration with link extraction AND scoring enabled
+    config = CrawlerRunConfig(
+        link_preview_config=LinkPreviewConfig(
+            include_internal=True,
+            include_external=False,  # Only internal links for this test
+            # No include/exclude patterns for first test - let's see what we get
+            query="API documentation reference guide",
+            score_threshold=0.3,
+            concurrency=5,
+            timeout=10,
+            max_links=5,  # Just test with 5 links first
+            verbose=True  # Show detailed progress
+        ),
+        score_links=True,  # Enable intrinsic link scoring
+        only_text=True,
+        verbose=True
+    )
+
+    # Test URLs
+    test_urls = [
+        "https://docs.python.org/3/",  # Python docs - should have many internal links
+        "https://httpbin.org/",        # Simple site for testing
+    ]
+
+    async with AsyncWebCrawler() as crawler:
+        for url in test_urls:
+            print(f"\n🌐 Testing URL: {url}")
+            print("-" * 40)
+
+            try:
+                result = await crawler.arun(url, config=config)
+                
+                # Debug: Check if link extraction config is being passed
+                print(f"🔍 Debug - Link extraction config: {config.link_preview_config.to_dict() if config.link_preview_config else None}")
+                print(f"🔍 Debug - Score links: {config.score_links}")
+
+                if result.success:
+                    print(f"✅ Crawl successful!")
+                    print(
+                        f"📄 Page title: {result.metadata.get('title', 'No title')}")
+
+                    # Check links - handle both dict and Links object structure
+                    if isinstance(result.links, dict):
+                        internal_links = [
+                            Link(**link) for link in result.links.get('internal', [])]
+                        external_links = [
+                            Link(**link) for link in result.links.get('external', [])]
+                    else:
+                        internal_links = result.links.internal
+                        external_links = result.links.external
+
+                    print(f"🔗 Found {len(internal_links)} internal links")
+                    print(f"🌍 Found {len(external_links)} external links")
+
+                    # Show links with head data
+                    links_with_head = [link for link in internal_links + external_links
+                                       if hasattr(link, 'head_data') and link.head_data]
+
+                    print(
+                        f"🧠 Links with head data extracted: {len(links_with_head)}")
+
+                    # Show all score types for all links (first 3)
+                    all_links = internal_links + external_links
+                    if all_links:
+                        print(f"\n🔢 Sample link scores (first 3 links):")
+                        for i, link in enumerate(all_links[:3]):
+                            print(f"\n  {i+1}. {link.href}")
+                            
+                            # Show intrinsic score
+                            if hasattr(link, 'intrinsic_score') and link.intrinsic_score is not None:
+                                if link.intrinsic_score == float('inf'):
+                                    print(f"     Intrinsic Score: ∞ (scoring disabled)")
+                                else:
+                                    print(f"     Intrinsic Score: {link.intrinsic_score:.2f}/10.0")
+                            else:
+                                print(f"     Intrinsic Score: Not available")
+                            
+                            # Show contextual score (BM25)
+                            if hasattr(link, 'contextual_score') and link.contextual_score is not None:
+                                print(f"     Contextual Score: {link.contextual_score:.3f}")
+                            else:
+                                print(f"     Contextual Score: Not available")
+                            
+                            # Show total score
+                            if hasattr(link, 'total_score') and link.total_score is not None:
+                                print(f"     Total Score: {link.total_score:.3f}")
+                            else:
+                                print(f"     Total Score: Not available")
+                            
+                            print(f"     Text: '{link.text[:50]}...' " if link.text else "     Text: (no text)")
+
+                    if links_with_head:
+                        print("\n📊 Sample links with head data:")
+                        # Show top 3
+                        for i, link in enumerate(links_with_head[:3]):
+                            print(f"\n  {i+1}. {link.href}")
+                            print(
+                                f"     Status: {link.head_extraction_status}")
+                            
+                            # Show all three score types
+                            print(f"     📊 Scoring Summary:")
+                            if hasattr(link, 'intrinsic_score') and link.intrinsic_score is not None:
+                                if link.intrinsic_score == float('inf'):
+                                    print(f"       • Intrinsic Score: ∞ (scoring disabled)")
+                                else:
+                                    print(f"       • Intrinsic Score: {link.intrinsic_score:.2f}/10.0")
+                            else:
+                                print(f"       • Intrinsic Score: Not available")
+                            
+                            if hasattr(link, 'contextual_score') and link.contextual_score is not None:
+                                print(f"       • Contextual Score: {link.contextual_score:.3f}")
+                            else:
+                                print(f"       • Contextual Score: Not available")
+                            
+                            if hasattr(link, 'total_score') and link.total_score is not None:
+                                print(f"       • Total Score: {link.total_score:.3f}")
+                            else:
+                                print(f"       • Total Score: Not available")
+                            
+                            if link.head_data:
+                                title = link.head_data.get('title', 'No title')
+                                if title:
+                                    print(f"     Title: {title[:60]}...")
+
+                                meta = link.head_data.get('meta', {})
+                                if 'description' in meta and meta['description']:
+                                    desc = meta['description']
+                                    print(f"     Description: {desc[:80]}...")
+
+                                # Show link metadata keys (should now be properly formatted)
+                                link_data = link.head_data.get('link', {})
+                                if link_data:
+                                    keys = list(link_data.keys())[:3]
+                                    print(f"     Link types: {keys}")
+
+                    # Show failed extractions
+                    failed_links = [link for link in internal_links + external_links
+                                    if hasattr(link, 'head_extraction_status') and
+                                    link.head_extraction_status == 'failed']
+
+                    if failed_links:
+                        print(
+                            f"\n❌ Failed head extractions: {len(failed_links)}")
+                        for link in failed_links[:2]:  # Show first 2 failures
+                            print(f"  - {link.href}")
+                            if hasattr(link, 'head_extraction_error') and link.head_extraction_error:
+                                print(
+                                    f"    Error: {link.head_extraction_error}")
+
+                else:
+                    print(f"❌ Crawl failed: {result.error_message}")
+
+            except Exception as e:
+                print(f"💥 Error testing {url}: {str(e)}")
+                import traceback
+                traceback.print_exc()
+
+
+def test_config_examples():
+    """Show example configurations"""
+
+    print("\n📚 Example Configurations")
+    print("=" * 50)
+
+    examples = [
+        {
+            "name": "BM25 Scored Documentation Links",
+            "config": LinkPreviewConfig(
+                include_internal=True,
+                include_external=False,
+                include_patterns=["*/docs/*", "*/api/*", "*/reference/*"],
+                query="API documentation reference guide",
+                score_threshold=0.3,
+                max_links=30,
+                verbose=True
+            )
+        },
+        {
+            "name": "Internal Links Only",
+            "config": LinkPreviewConfig(
+                include_internal=True,
+                include_external=False,
+                max_links=50,
+                verbose=True
+            )
+        },
+        {
+            "name": "External Links with Patterns",
+            "config": LinkPreviewConfig(
+                include_internal=False,
+                include_external=True,
+                include_patterns=["*github.com*", "*stackoverflow.com*"],
+                max_links=20,
+                concurrency=10
+            )
+        },
+        {
+            "name": "High-Performance Mode",
+            "config": LinkPreviewConfig(
+                include_internal=True,
+                include_external=False,
+                concurrency=20,
+                timeout=3,
+                max_links=100,
+                verbose=False
+            )
+        }
+    ]
+
+    for example in examples:
+        print(f"\n📝 {example['name']}:")
+        print("   Configuration:")
+        config_dict = example['config'].to_dict()
+        for key, value in config_dict.items():
+            print(f"     {key}: {value}")
+
+        print("   Usage:")
+        print("     from crawl4ai import LinkPreviewConfig")
+        print("     config = CrawlerRunConfig(")
+        print("         link_preview_config=LinkPreviewConfig(")
+        for key, value in config_dict.items():
+            if isinstance(value, str):
+                print(f"             {key}='{value}',")
+            elif isinstance(value, list) and value:
+                print(f"             {key}={value},")
+            elif value is not None:
+                print(f"             {key}={value},")
+        print("         )")
+        print("     )")
+
+
+if __name__ == "__main__":
+    # Show configuration examples first
+    test_config_examples()
+
+    # Run the actual test
+    print("\n🚀 Running Link Extractor Tests...")
+    asyncio.run(test_link_extractor())
+
+    print("\n✨ Test completed!")
diff --git a/tests/test_llm_extraction_parallel_issue_1055.py b/tests/test_llm_extraction_parallel_issue_1055.py
new file mode 100644
index 0000000..19f1e50
--- /dev/null
+++ b/tests/test_llm_extraction_parallel_issue_1055.py
@@ -0,0 +1,220 @@
+"""
+Final verification test for Issue #1055 fix
+
+This test demonstrates that LLM extraction now runs in parallel
+when using arun_many with multiple URLs.
+"""
+
+import os
+import sys
+import time
+import asyncio
+
+grandparent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(grandparent_dir)
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    LLMExtractionStrategy,
+    LLMConfig,
+)
+
+from pydantic import BaseModel
+
+
+class SimpleData(BaseModel):
+    title: str
+    summary: str
+
+
+def print_section(title):
+    print("\n" + "=" * 80)
+    print(title)
+    print("=" * 80 + "\n")
+
+
+async def test_without_llm():
+    """Baseline: Test crawling without LLM extraction"""
+    print_section("TEST 1: Crawling WITHOUT LLM Extraction")
+
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+    )
+
+    browser_config = BrowserConfig(headless=True, verbose=False)
+
+    urls = [
+        "https://www.example.com",
+        "https://www.iana.org",
+        "https://www.wikipedia.org",
+    ]
+
+    print(f"Crawling {len(urls)} URLs without LLM extraction...")
+    print("Expected: Fast and parallel\n")
+
+    start_time = time.time()
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        results = await crawler.arun_many(urls=urls, config=config)
+
+    duration = time.time() - start_time
+
+    print(f"\n✅ Completed in {duration:.2f}s")
+    print(f"   Successful: {sum(1 for r in results if r.success)}/{len(urls)}")
+    print(f"   Average: {duration/len(urls):.2f}s per URL")
+
+    return duration
+
+
+async def test_with_llm_before_fix():
+    """Demonstrate the problem: Sequential execution with LLM"""
+    print_section("TEST 2: What Issue #1055 Reported (LLM Sequential Behavior)")
+
+    print("The issue reported that with LLM extraction, URLs would crawl")
+    print("one after another instead of in parallel.")
+    print("\nWithout our fix, this would show:")
+    print("  - URL 1 fetches → extracts → completes")
+    print("  - URL 2 fetches → extracts → completes")
+    print("  - URL 3 fetches → extracts → completes")
+    print("\nTotal time would be approximately sum of all individual times.")
+
+
+async def test_with_llm_after_fix():
+    """Demonstrate the fix: Parallel execution with LLM"""
+    print_section("TEST 3: After Fix - LLM Extraction in Parallel")
+
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+            schema=SimpleData.model_json_schema(),
+            extraction_type="schema",
+            instruction="Extract title and summary",
+        )
+    )
+
+    browser_config = BrowserConfig(headless=True, verbose=False)
+
+    urls = [
+        "https://www.example.com",
+        "https://www.iana.org",
+        "https://www.wikipedia.org",
+    ]
+
+    print(f"Crawling {len(urls)} URLs WITH LLM extraction...")
+    print("Expected: Parallel execution with our fix\n")
+
+    completion_times = {}
+    start_time = time.time()
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        results = await crawler.arun_many(urls=urls, config=config)
+        for result in results:
+            elapsed = time.time() - start_time
+            completion_times[result.url] = elapsed
+            print(f"  [{elapsed:5.2f}s] ✓ {result.url[:50]}")
+
+    duration = time.time() - start_time
+
+    print(f"\n✅ Total time: {duration:.2f}s")
+    print(f"   Successful: {sum(1 for url in urls if url in completion_times)}/{len(urls)}")
+
+    # Analyze parallelism
+    times = list(completion_times.values())
+    if len(times) >= 2:
+        # If parallel, completion times should be staggered, not evenly spaced
+        time_diffs = [times[i+1] - times[i] for i in range(len(times)-1)]
+        avg_diff = sum(time_diffs) / len(time_diffs)
+
+        print(f"\nParallelism Analysis:")
+        print(f"   Completion time differences: {[f'{d:.2f}s' for d in time_diffs]}")
+        print(f"   Average difference: {avg_diff:.2f}s")
+
+        # In parallel mode, some tasks complete close together
+        # In sequential mode, they're evenly spaced (avg ~2-3s apart)
+        if avg_diff < duration / len(urls):
+            print(f"   ✅ PARALLEL: Tasks completed with overlapping execution")
+        else:
+            print(f"   ⚠️  SEQUENTIAL: Tasks completed one after another")
+
+    return duration
+
+
+async def test_multiple_arun_calls():
+    """Test multiple individual arun() calls in parallel"""
+    print_section("TEST 4: Multiple arun() Calls with asyncio.gather")
+
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+            schema=SimpleData.model_json_schema(),
+            extraction_type="schema",
+            instruction="Extract title and summary",
+        )
+    )
+
+    browser_config = BrowserConfig(headless=True, verbose=False)
+
+    urls = [
+        "https://www.example.com",
+        "https://www.iana.org",
+        "https://www.wikipedia.org",
+    ]
+
+    print(f"Running {len(urls)} arun() calls with asyncio.gather()...")
+    print("Expected: True parallel execution\n")
+
+    start_time = time.time()
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        tasks = [crawler.arun(url, config=config) for url in urls]
+        results = await asyncio.gather(*tasks)
+
+    duration = time.time() - start_time
+
+    print(f"\n✅ Completed in {duration:.2f}s")
+    print(f"   Successful: {sum(1 for r in results if r.success)}/{len(urls)}")
+    print(f"   This proves the async LLM extraction works correctly")
+
+    return duration
+
+
+async def main():
+    print("\n" + "🚀" * 40)
+    print("ISSUE #1055 FIX VERIFICATION")
+    print("Testing: Sequential → Parallel LLM Extraction")
+    print("🚀" * 40)
+
+    # Run tests
+    await test_without_llm()
+
+    await test_with_llm_before_fix()
+
+    time_with_llm = await test_with_llm_after_fix()
+
+    time_gather = await test_multiple_arun_calls()
+
+    # Final summary
+    print_section("FINAL VERDICT")
+
+    print("✅ Fix Verified!")
+    print("\nWhat changed:")
+    print("  • Created aperform_completion_with_backoff() using litellm.acompletion")
+    print("  • Added arun() method to ExtractionStrategy base class")
+    print("  • Implemented parallel arun() in LLMExtractionStrategy")
+    print("  • Updated AsyncWebCrawler to use arun() when available")
+    print("\nResult:")
+    print("  • LLM extraction now runs in parallel across multiple URLs")
+    print("  • Backward compatible - existing strategies still work")
+    print("  • No breaking changes to the API")
+    print("\n✨ Issue #1055 is RESOLVED!")
+
+    print("\n" + "=" * 80 + "\n")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/test_llm_simple_url.py b/tests/test_llm_simple_url.py
new file mode 100644
index 0000000..bb31434
--- /dev/null
+++ b/tests/test_llm_simple_url.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""
+Test LLMTableExtraction with controlled HTML
+"""
+
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    LLMConfig,
+    LLMTableExtraction,
+    DefaultTableExtraction,
+    CacheMode
+)
+
+async def test_controlled_html():
+    """Test with controlled HTML content."""
+    print("\n" + "=" * 60)
+    print("LLM TABLE EXTRACTION TEST")
+    print("=" * 60)
+    
+    url = "https://en.wikipedia.org/wiki/List_of_chemical_elements"
+    # url = "https://en.wikipedia.org/wiki/List_of_prime_ministers_of_India"
+    
+    # Configure LLM
+    llm_config = LLMConfig(
+        # provider="openai/gpt-4.1-mini",
+        # api_token=os.getenv("OPENAI_API_KEY"),
+        provider="groq/llama-3.3-70b-versatile",
+        api_token="GROQ_API_TOKEN",
+        temperature=0.1,
+        max_tokens=32000
+    )
+    
+    print("\n1. Testing LLMTableExtraction:")
+    
+    # Create LLM extraction strategy
+    llm_strategy = LLMTableExtraction(
+        llm_config=llm_config,
+        verbose=True,
+        # css_selector="div.w3-example"
+        css_selector="div.mw-content-ltr",
+        # css_selector="table.wikitable",
+        max_tries=2,
+        
+        enable_chunking=True,
+        chunk_token_threshold=5000,  # Lower threshold to force chunking
+        min_rows_per_chunk=10,
+        max_parallel_chunks=3
+    )
+    
+    config_llm = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        table_extraction=llm_strategy
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        # Test with LLM extraction
+        result_llm = await crawler.arun(
+            # url=f"raw:{test_html}",
+            url=url,
+            config=config_llm
+        )
+        
+        if result_llm.success:
+            print(f"\n   ✓ LLM Extraction: Found {len(result_llm.tables)} table(s)")
+            
+            for i, table in enumerate(result_llm.tables, 1):
+                print(f"\n   Table {i}:")
+                print(f"   - Caption: {table.get('caption', 'No caption')}")
+                print(f"   - Headers: {table['headers']}")
+                print(f"   - Rows: {len(table['rows'])}")
+                
+                # Show how colspan/rowspan were handled
+                print(f"   - Sample rows:")
+                for j, row in enumerate(table['rows'][:2], 1):
+                    print(f"     Row {j}: {row}")
+                
+                metadata = table.get('metadata', {})
+                print(f"   - Metadata:")
+                print(f"     • Has merged cells: {metadata.get('has_merged_cells', False)}")
+                print(f"     • Table type: {metadata.get('table_type', 'unknown')}")
+        
+        # # Compare with default extraction
+        # print("\n2. Comparing with DefaultTableExtraction:")
+        
+        # default_strategy = DefaultTableExtraction(
+        #     table_score_threshold=3,
+        #     verbose=False
+        # )
+        
+        # config_default = CrawlerRunConfig(
+        #     cache_mode=CacheMode.BYPASS,
+        #     table_extraction=default_strategy
+        # )
+        
+        # result_default = await crawler.arun(
+        #     # url=f"raw:{test_html}",
+        #     url=url,
+        #     config=config_default
+        # )
+        
+        # if result_default.success:
+        #     print(f"   ✓ Default Extraction: Found {len(result_default.tables)} table(s)")
+            
+        #     # Compare handling of complex structures
+        #     print("\n3. Comparison Summary:")
+        #     print(f"   LLM found: {len(result_llm.tables)} tables")
+        #     print(f"   Default found: {len(result_default.tables)} tables")
+            
+        #     if result_llm.tables and result_default.tables:
+        #         llm_first = result_llm.tables[0]
+        #         default_first = result_default.tables[0]
+                
+        #         print(f"\n   First table comparison:")
+        #         print(f"   LLM headers: {len(llm_first['headers'])} columns")
+        #         print(f"   Default headers: {len(default_first['headers'])} columns")
+                
+        #         # Check if LLM better handled the complex structure
+        #         if llm_first.get('metadata', {}).get('has_merged_cells'):
+        #             print("   ✓ LLM correctly identified merged cells")
+                
+        #         # Test pandas compatibility
+        #         try:
+        #             import pandas as pd
+                    
+        #             print("\n4. Testing Pandas compatibility:")
+                    
+        #             # Create DataFrame from LLM extraction
+        #             df_llm = pd.DataFrame(
+        #                 llm_first['rows'],
+        #                 columns=llm_first['headers']
+        #             )
+        #             print(f"   ✓ LLM table -> DataFrame: Shape {df_llm.shape}")
+                    
+        #             # Create DataFrame from default extraction
+        #             df_default = pd.DataFrame(
+        #                 default_first['rows'],
+        #                 columns=default_first['headers']
+        #             )
+        #             print(f"   ✓ Default table -> DataFrame: Shape {df_default.shape}")
+                    
+        #             print("\n   LLM DataFrame preview:")
+        #             print(df_llm.head(2).to_string())
+                    
+        #         except ImportError:
+        #             print("\n4. Pandas not installed, skipping DataFrame test")
+        
+        print("\n✅ Test completed successfully!")
+
+async def main():
+    """Run the test."""
+    
+    # Check for API key
+    if not os.getenv("OPENAI_API_KEY"):
+        print("⚠️  OPENAI_API_KEY not set. Please set it to test LLM extraction.")
+        print("   You can set it with: export OPENAI_API_KEY='your-key-here'")
+        return
+    
+    await test_controlled_html()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+    
+    
+    
\ No newline at end of file
diff --git a/tests/test_llmtxt.py b/tests/test_llmtxt.py
new file mode 100644
index 0000000..2cdb027
--- /dev/null
+++ b/tests/test_llmtxt.py
@@ -0,0 +1,52 @@
+from crawl4ai.llmtxt import AsyncLLMTextManager  # Changed to AsyncLLMTextManager
+from crawl4ai.async_logger import AsyncLogger
+from pathlib import Path
+import asyncio
+
+
+async def main():
+    current_file = Path(__file__).resolve()
+    # base_dir = current_file.parent.parent / "local/_docs/llm.txt/test_docs"
+    base_dir = current_file.parent.parent / "local/_docs/llm.txt"
+    docs_dir = base_dir
+
+    # Create directory if it doesn't exist
+    docs_dir.mkdir(parents=True, exist_ok=True)
+
+    # Initialize logger
+    logger = AsyncLogger()
+    # Updated initialization with default batching params
+    # manager = AsyncLLMTextManager(docs_dir, logger, max_concurrent_calls=3, batch_size=2)
+    manager = AsyncLLMTextManager(docs_dir, logger, batch_size=2)
+
+    # Let's first check what files we have
+    print("\nAvailable files:")
+    for f in docs_dir.glob("*.md"):
+        print(f"- {f.name}")
+
+    # Generate index files
+    print("\nGenerating index files...")
+    await manager.generate_index_files(
+        force_generate_facts=False, clear_bm25_cache=False
+    )
+
+    # Test some relevant queries about Crawl4AI
+    test_queries = [
+        "How is using the `arun_many` method?",
+    ]
+
+    print("\nTesting search functionality:")
+    for query in test_queries:
+        print(f"\nQuery: {query}")
+        results = manager.search(query, top_k=2)
+        print(f"Results length: {len(results)} characters")
+        if results:
+            print(
+                "First 200 chars of results:", results[:200].replace("\n", " "), "..."
+            )
+        else:
+            print("No results found")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/test_main.py b/tests/test_main.py
new file mode 100644
index 0000000..b32b68f
--- /dev/null
+++ b/tests/test_main.py
@@ -0,0 +1,276 @@
+import asyncio
+import aiohttp
+import json
+import time
+import os
+from typing import Dict, Any
+
+
+class NBCNewsAPITest:
+    def __init__(self, base_url: str = "http://localhost:8000"):
+        self.base_url = base_url
+        self.session = None
+
+    async def __aenter__(self):
+        self.session = aiohttp.ClientSession()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+
+    async def submit_crawl(self, request_data: Dict[str, Any]) -> str:
+        async with self.session.post(
+            f"{self.base_url}/crawl", json=request_data
+        ) as response:
+            result = await response.json()
+            return result["task_id"]
+
+    async def get_task_status(self, task_id: str) -> Dict[str, Any]:
+        async with self.session.get(f"{self.base_url}/task/{task_id}") as response:
+            return await response.json()
+
+    async def wait_for_task(
+        self, task_id: str, timeout: int = 300, poll_interval: int = 2
+    ) -> Dict[str, Any]:
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Task {task_id} did not complete within {timeout} seconds"
+                )
+
+            status = await self.get_task_status(task_id)
+            if status["status"] in ["completed", "failed"]:
+                return status
+
+            await asyncio.sleep(poll_interval)
+
+    async def check_health(self) -> Dict[str, Any]:
+        async with self.session.get(f"{self.base_url}/health") as response:
+            return await response.json()
+
+
+async def test_basic_crawl():
+    print("\n=== Testing Basic Crawl ===")
+    async with NBCNewsAPITest() as api:
+        request = {"urls": ["https://www.nbcnews.com/business"], "priority": 10}
+        task_id = await api.submit_crawl(request)
+        result = await api.wait_for_task(task_id)
+        print(f"Basic crawl result length: {len(result['result']['markdown'])}")
+        assert result["status"] == "completed"
+        assert "result" in result
+        assert result["result"]["success"]
+
+
+async def test_js_execution():
+    print("\n=== Testing JS Execution ===")
+    async with NBCNewsAPITest() as api:
+        request = {
+            "urls": ["https://www.nbcnews.com/business"],
+            "priority": 8,
+            "js_code": [
+                "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+            ],
+            "wait_for": "article.tease-card:nth-child(10)",
+            "crawler_params": {"headless": True},
+        }
+        task_id = await api.submit_crawl(request)
+        result = await api.wait_for_task(task_id)
+        print(f"JS execution result length: {len(result['result']['markdown'])}")
+        assert result["status"] == "completed"
+        assert result["result"]["success"]
+
+
+async def test_css_selector():
+    print("\n=== Testing CSS Selector ===")
+    async with NBCNewsAPITest() as api:
+        request = {
+            "urls": ["https://www.nbcnews.com/business"],
+            "priority": 7,
+            "css_selector": ".wide-tease-item__description",
+        }
+        task_id = await api.submit_crawl(request)
+        result = await api.wait_for_task(task_id)
+        print(f"CSS selector result length: {len(result['result']['markdown'])}")
+        assert result["status"] == "completed"
+        assert result["result"]["success"]
+
+
+async def test_structured_extraction():
+    print("\n=== Testing Structured Extraction ===")
+    async with NBCNewsAPITest() as api:
+        schema = {
+            "name": "NBC News Articles",
+            "baseSelector": "article.tease-card",
+            "fields": [
+                {"name": "title", "selector": "h2", "type": "text"},
+                {
+                    "name": "description",
+                    "selector": ".tease-card__description",
+                    "type": "text",
+                },
+                {
+                    "name": "link",
+                    "selector": "a",
+                    "type": "attribute",
+                    "attribute": "href",
+                },
+            ],
+        }
+
+        request = {
+            "urls": ["https://www.nbcnews.com/business"],
+            "priority": 9,
+            "extraction_config": {"type": "json_css", "params": {"schema": schema}},
+        }
+        task_id = await api.submit_crawl(request)
+        result = await api.wait_for_task(task_id)
+        extracted = json.loads(result["result"]["extracted_content"])
+        print(f"Extracted {len(extracted)} articles")
+        assert result["status"] == "completed"
+        assert result["result"]["success"]
+        assert len(extracted) > 0
+
+
+async def test_batch_crawl():
+    print("\n=== Testing Batch Crawl ===")
+    async with NBCNewsAPITest() as api:
+        request = {
+            "urls": [
+                "https://www.nbcnews.com/business",
+                "https://www.nbcnews.com/business/consumer",
+                "https://www.nbcnews.com/business/economy",
+            ],
+            "priority": 6,
+            "crawler_params": {"headless": True},
+        }
+        task_id = await api.submit_crawl(request)
+        result = await api.wait_for_task(task_id)
+        print(f"Batch crawl completed, got {len(result['results'])} results")
+        assert result["status"] == "completed"
+        assert "results" in result
+        assert len(result["results"]) == 3
+
+
+async def test_llm_extraction():
+    print("\n=== Testing LLM Extraction with Ollama ===")
+    async with NBCNewsAPITest() as api:
+        schema = {
+            "type": "object",
+            "properties": {
+                "article_title": {
+                    "type": "string",
+                    "description": "The main title of the news article",
+                },
+                "summary": {
+                    "type": "string",
+                    "description": "A brief summary of the article content",
+                },
+                "main_topics": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Main topics or themes discussed in the article",
+                },
+            },
+            "required": ["article_title", "summary", "main_topics"],
+        }
+
+        request = {
+            "urls": ["https://www.nbcnews.com/business"],
+            "priority": 8,
+            "extraction_config": {
+                "type": "llm",
+                "params": {
+                    "provider": "openai/gpt-4o-mini",
+                    "api_key": os.getenv("OLLAMA_API_KEY"),
+                    "schema": schema,
+                    "extraction_type": "schema",
+                    "instruction": """Extract the main article information including title, a brief summary, and main topics discussed. 
+                    Focus on the primary business news article on the page.""",
+                },
+            },
+            "crawler_params": {"headless": True, "word_count_threshold": 1},
+        }
+
+        task_id = await api.submit_crawl(request)
+        result = await api.wait_for_task(task_id)
+
+        if result["status"] == "completed":
+            extracted = json.loads(result["result"]["extracted_content"])
+            print("Extracted article analysis:")
+            print(json.dumps(extracted, indent=2))
+
+        assert result["status"] == "completed"
+        assert result["result"]["success"]
+
+
+async def test_screenshot():
+    print("\n=== Testing Screenshot ===")
+    async with NBCNewsAPITest() as api:
+        request = {
+            "urls": ["https://www.nbcnews.com/business"],
+            "priority": 5,
+            "screenshot": True,
+            "crawler_params": {"headless": True},
+        }
+        task_id = await api.submit_crawl(request)
+        result = await api.wait_for_task(task_id)
+        print("Screenshot captured:", bool(result["result"]["screenshot"]))
+        assert result["status"] == "completed"
+        assert result["result"]["success"]
+        assert result["result"]["screenshot"] is not None
+
+
+async def test_priority_handling():
+    print("\n=== Testing Priority Handling ===")
+    async with NBCNewsAPITest() as api:
+        # Submit low priority task first
+        low_priority = {
+            "urls": ["https://www.nbcnews.com/business"],
+            "priority": 1,
+            "crawler_params": {"headless": True},
+        }
+        low_task_id = await api.submit_crawl(low_priority)
+
+        # Submit high priority task
+        high_priority = {
+            "urls": ["https://www.nbcnews.com/business/consumer"],
+            "priority": 10,
+            "crawler_params": {"headless": True},
+        }
+        high_task_id = await api.submit_crawl(high_priority)
+
+        # Get both results
+        high_result = await api.wait_for_task(high_task_id)
+        low_result = await api.wait_for_task(low_task_id)
+
+        print("Both tasks completed")
+        assert high_result["status"] == "completed"
+        assert low_result["status"] == "completed"
+
+
+async def main():
+    try:
+        # Start with health check
+        async with NBCNewsAPITest() as api:
+            health = await api.check_health()
+            print("Server health:", health)
+
+        # Run all tests
+        # await test_basic_crawl()
+        # await test_js_execution()
+        # await test_css_selector()
+        # await test_structured_extraction()
+        await test_llm_extraction()
+        # await test_batch_crawl()
+        # await test_screenshot()
+        # await test_priority_handling()
+
+    except Exception as e:
+        print(f"Test failed: {str(e)}")
+        raise
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/test_markdown_generator_validation_1880.py b/tests/test_markdown_generator_validation_1880.py
new file mode 100644
index 0000000..124079a
--- /dev/null
+++ b/tests/test_markdown_generator_validation_1880.py
@@ -0,0 +1,158 @@
+"""
+Tests for #1880: markdown_generator deserialization validation in CrawlerRunConfig
+
+Ensures that:
+1. Correct {"type": ..., "params": {...}} format deserializes properly
+2. Wrong key names ("options") raise a clear ValueError, not a cryptic AttributeError
+3. Nested content_filter deserializes correctly
+"""
+import pytest
+
+
+class TestMarkdownGeneratorDeserialization:
+    """Test CrawlerRunConfig.load() with markdown_generator configs."""
+
+    def test_params_key_deserializes_correctly(self):
+        """{"type": ..., "params": {...}} should produce a real object."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+
+        data = {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {},
+            }
+        }
+        config = CrawlerRunConfig.load(data)
+        from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+        assert isinstance(config.markdown_generator, DefaultMarkdownGenerator)
+
+    def test_params_with_content_filter(self):
+        """Nested BM25ContentFilter should deserialize inside markdown_generator."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+        data = {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "BM25ContentFilter",
+                        "params": {
+                            "user_query": "example",
+                            "bm25_threshold": 0.9,
+                        },
+                    }
+                },
+            }
+        }
+        config = CrawlerRunConfig.load(data)
+        assert isinstance(config.markdown_generator.content_filter, BM25ContentFilter)
+        assert config.markdown_generator.content_filter.user_query == "example"
+        assert config.markdown_generator.content_filter.bm25_threshold == 0.9
+
+    def test_params_with_pruning_filter(self):
+        """PruningContentFilter should also work."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        from crawl4ai.content_filter_strategy import PruningContentFilter
+
+        data = {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {},
+                    }
+                },
+            }
+        }
+        config = CrawlerRunConfig.load(data)
+        assert isinstance(config.markdown_generator.content_filter, PruningContentFilter)
+
+    def test_options_key_raises_clear_error(self):
+        """Using "options" instead of "params" should raise ValueError with hint."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+
+        data = {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "options": {"content_filter": {}},
+            }
+        }
+        with pytest.raises(ValueError, match="params.*required"):
+            CrawlerRunConfig.load(data)
+
+    def test_arbitrary_key_raises_clear_error(self):
+        """Any non-"params" key should raise ValueError."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+
+        data = {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "settings": {},
+            }
+        }
+        with pytest.raises(ValueError, match="markdown_generator must be an instance"):
+            CrawlerRunConfig.load(data)
+
+    def test_plain_dict_raises_clear_error(self):
+        """A dict without type/params structure should raise ValueError."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+
+        data = {
+            "markdown_generator": {"foo": "bar"}
+        }
+        with pytest.raises(ValueError, match="got dict"):
+            CrawlerRunConfig.load(data)
+
+    def test_error_message_mentions_params_key(self):
+        """Error message should specifically mention that 'params' is required."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+
+        data = {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "options": {},
+            }
+        }
+        with pytest.raises(ValueError) as exc_info:
+            CrawlerRunConfig.load(data)
+        msg = str(exc_info.value)
+        assert "params" in msg
+        assert "options" in msg or "not recognized" in msg
+
+    def test_none_markdown_generator_uses_default(self):
+        """None should use the default (DefaultMarkdownGenerator)."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+        config = CrawlerRunConfig(markdown_generator=None)
+        # None is allowed — the crawler falls back to default behavior
+        assert config.markdown_generator is None
+
+    def test_valid_instance_passes_validation(self):
+        """Passing an actual instance should work fine."""
+        from crawl4ai.async_configs import CrawlerRunConfig
+        from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+        from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+        gen = DefaultMarkdownGenerator(
+            content_filter=BM25ContentFilter(user_query="test")
+        )
+        config = CrawlerRunConfig(markdown_generator=gen)
+        assert config.markdown_generator is gen
+        assert config.markdown_generator.content_filter.user_query == "test"
+
+
+class TestExistingValidationStillWorks:
+    """Ensure existing extraction_strategy/chunking_strategy validation unchanged."""
+
+    def test_extraction_strategy_dict_raises(self):
+        from crawl4ai.async_configs import CrawlerRunConfig
+        with pytest.raises(ValueError, match="extraction_strategy"):
+            CrawlerRunConfig(extraction_strategy={"type": "bad"})
+
+    def test_chunking_strategy_dict_raises(self):
+        from crawl4ai.async_configs import CrawlerRunConfig
+        with pytest.raises(ValueError, match="chunking_strategy"):
+            CrawlerRunConfig(chunking_strategy={"type": "bad"})
diff --git a/tests/test_memory_macos.py b/tests/test_memory_macos.py
new file mode 100755
index 0000000..7019ff0
--- /dev/null
+++ b/tests/test_memory_macos.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Test script to verify macOS memory calculation accuracy."""
+
+import psutil
+import platform
+import time
+from crawl4ai.utils import get_true_memory_usage_percent, get_memory_stats, get_true_available_memory_gb
+
+
+def test_memory_calculation():
+    """Test and compare memory calculations."""
+    print(f"Platform: {platform.system()}")
+    print(f"Python version: {platform.python_version()}")
+    print("-" * 60)
+    
+    # Get psutil's view
+    vm = psutil.virtual_memory()
+    psutil_percent = vm.percent
+    psutil_available_gb = vm.available / (1024**3)
+    total_gb = vm.total / (1024**3)
+    
+    # Get our corrected view
+    true_percent = get_true_memory_usage_percent()
+    true_available_gb = get_true_available_memory_gb()
+    true_percent_calc, available_calc, total_calc = get_memory_stats()
+    
+    print("Memory Statistics Comparison:")
+    print(f"Total Memory: {total_gb:.2f} GB")
+    print()
+    
+    print("PSUtil (Standard) Calculation:")
+    print(f"  - Memory Used: {psutil_percent:.1f}%")
+    print(f"  - Available: {psutil_available_gb:.2f} GB")
+    print()
+    
+    print("Platform-Aware Calculation:")
+    print(f"  - Memory Used: {true_percent:.1f}%")
+    print(f"  - Available: {true_available_gb:.2f} GB")
+    print(f"  - Difference: {true_available_gb - psutil_available_gb:.2f} GB of reclaimable memory")
+    print()
+    
+    # Show the impact on dispatcher behavior
+    print("Impact on MemoryAdaptiveDispatcher:")
+    thresholds = {
+        "Normal": 90.0,
+        "Critical": 95.0,
+        "Recovery": 85.0
+    }
+    
+    for name, threshold in thresholds.items():
+        psutil_triggered = psutil_percent >= threshold
+        true_triggered = true_percent >= threshold
+        print(f"  - {name} Threshold ({threshold}%):")
+        print(f"    PSUtil: {'TRIGGERED' if psutil_triggered else 'OK'}")
+        print(f"    Platform-Aware: {'TRIGGERED' if true_triggered else 'OK'}")
+        if psutil_triggered != true_triggered:
+            print(f"    → Difference: Platform-aware prevents false {'pressure' if psutil_triggered else 'recovery'}")
+    print()
+    
+    # Monitor for a few seconds
+    print("Monitoring memory for 10 seconds...")
+    for i in range(10):
+        vm = psutil.virtual_memory()
+        true_pct = get_true_memory_usage_percent()
+        print(f"  {i+1}s - PSUtil: {vm.percent:.1f}% | Platform-Aware: {true_pct:.1f}%", end="\r")
+        time.sleep(1)
+    print("\n")
+
+
+if __name__ == "__main__":
+    test_memory_calculation()
\ No newline at end of file
diff --git a/tests/test_merge_head_data_scoring.py b/tests/test_merge_head_data_scoring.py
new file mode 100644
index 0000000..65aad04
--- /dev/null
+++ b/tests/test_merge_head_data_scoring.py
@@ -0,0 +1,183 @@
+"""
+Unit tests for _merge_head_data() total_score calculation.
+
+Verifies that total_score is computed for all links, including those
+that fail head extraction and only have an intrinsic_score.
+
+Regression tests for https://github.com/unclecode/crawl4ai/issues/1749
+"""
+
+import pytest
+from unittest.mock import MagicMock
+
+from crawl4ai.models import Link, Links
+from crawl4ai.link_preview import LinkPreview
+from crawl4ai.utils import calculate_total_score
+
+
+class TestCalculateTotalScore:
+    """Test the calculate_total_score utility function."""
+
+    def test_intrinsic_only(self):
+        """When only intrinsic_score is available, total_score should equal intrinsic_score."""
+        score = calculate_total_score(
+            intrinsic_score=5.0,
+            contextual_score=None,
+            score_links_enabled=True,
+            query_provided=True,
+        )
+        assert score == 5.0
+
+    def test_no_scoring_enabled(self):
+        """When scoring is disabled, total_score should be neutral (5.0)."""
+        score = calculate_total_score(
+            intrinsic_score=8.0,
+            contextual_score=0.5,
+            score_links_enabled=False,
+            query_provided=True,
+        )
+        assert score == 5.0
+
+    def test_both_scores(self):
+        """When both scores are available, total_score should be a weighted combination."""
+        score = calculate_total_score(
+            intrinsic_score=8.0,
+            contextual_score=0.5,
+            score_links_enabled=True,
+            query_provided=True,
+        )
+        # 70% intrinsic + 30% contextual_scaled: (8.0 * 0.7) + (0.5 * 10.0 * 0.3) = 5.6 + 1.5 = 7.1
+        assert score == pytest.approx(7.1, abs=0.01)
+
+    def test_no_scores_at_all(self):
+        """When no scores are available, total_score should be 0."""
+        score = calculate_total_score(
+            intrinsic_score=None,
+            contextual_score=None,
+            score_links_enabled=True,
+            query_provided=False,
+        )
+        assert score == 0.0
+
+
+class TestMergeHeadDataScoring:
+    """Test _merge_head_data() calculates total_score for all links."""
+
+    def _make_config(self, score_links=True, query="test query"):
+        """Create a mock CrawlerRunConfig."""
+        config = MagicMock()
+        config.score_links = score_links
+        config.link_preview_config.query = query
+        return config
+
+    def test_internal_link_with_head_data_gets_total_score(self):
+        """Internal link with successful head extraction should have total_score."""
+        link = Link(href="https://example.com/page1", text="Page 1", intrinsic_score=6.0)
+        links = Links(internal=[link], external=[])
+
+        head_results = [
+            {
+                "url": "https://example.com/page1",
+                "head_data": {"title": "Page 1"},
+                "status": "valid",
+                "relevance_score": 0.8,
+            }
+        ]
+
+        preview = LinkPreview()
+        config = self._make_config()
+        updated = preview._merge_head_data(links, head_results, config)
+
+        assert updated.internal[0].total_score is not None
+        assert updated.internal[0].total_score > 0
+
+    def test_internal_link_without_head_data_gets_total_score(self):
+        """Internal link that failed head extraction should still get total_score from intrinsic_score."""
+        link = Link(href="https://example.com/doc.pdf", text="PDF Doc", intrinsic_score=5.0)
+        links = Links(internal=[link], external=[])
+
+        # No head results for this URL (simulates failed extraction)
+        head_results = []
+
+        preview = LinkPreview()
+        config = self._make_config()
+        updated = preview._merge_head_data(links, head_results, config)
+
+        assert updated.internal[0].total_score is not None
+        assert updated.internal[0].total_score == 5.0
+
+    def test_external_link_without_head_data_gets_total_score(self):
+        """External link that failed head extraction should still get total_score from intrinsic_score."""
+        link = Link(href="https://external.com/page", text="External", intrinsic_score=4.0)
+        links = Links(internal=[], external=[link])
+
+        head_results = []
+
+        preview = LinkPreview()
+        config = self._make_config()
+        updated = preview._merge_head_data(links, head_results, config)
+
+        assert updated.external[0].total_score is not None
+        assert updated.external[0].total_score == 4.0
+
+    def test_mixed_links_all_get_total_score(self):
+        """Mix of successful and failed head extractions should all have total_score."""
+        internal_success = Link(href="https://example.com/page1", text="Page 1", intrinsic_score=7.0)
+        internal_fail = Link(href="https://example.com/doc.pdf", text="PDF", intrinsic_score=5.0)
+        external_success = Link(href="https://other.com/page", text="Other", intrinsic_score=6.0)
+        external_fail = Link(href="https://other.com/timeout", text="Timeout", intrinsic_score=3.0)
+
+        links = Links(
+            internal=[internal_success, internal_fail],
+            external=[external_success, external_fail],
+        )
+
+        head_results = [
+            {
+                "url": "https://example.com/page1",
+                "head_data": {"title": "Page 1"},
+                "status": "valid",
+                "relevance_score": 0.9,
+            },
+            {
+                "url": "https://other.com/page",
+                "head_data": {"title": "Other Page"},
+                "status": "valid",
+                "relevance_score": 0.7,
+            },
+        ]
+
+        preview = LinkPreview()
+        config = self._make_config()
+        updated = preview._merge_head_data(links, head_results, config)
+
+        # All 4 links should have total_score set
+        for link in updated.internal + updated.external:
+            assert link.total_score is not None, f"total_score is None for {link.href}"
+            assert link.total_score > 0, f"total_score is 0 for {link.href}"
+
+    def test_link_without_intrinsic_score_and_no_head_data(self):
+        """Link with no intrinsic_score and no head data should still get a total_score (0.0)."""
+        link = Link(href="https://example.com/unknown", text="Unknown")
+        links = Links(internal=[link], external=[])
+
+        head_results = []
+
+        preview = LinkPreview()
+        config = self._make_config(query="test")
+        updated = preview._merge_head_data(links, head_results, config)
+
+        assert updated.internal[0].total_score is not None
+
+    def test_scoring_disabled_returns_neutral_score(self):
+        """When score_links is disabled, total_score should be neutral (5.0) for all links."""
+        link = Link(href="https://example.com/page", text="Page", intrinsic_score=8.0)
+        links = Links(internal=[link], external=[])
+
+        head_results = []
+
+        preview = LinkPreview()
+        config = self._make_config(score_links=False)
+        updated = preview._merge_head_data(links, head_results, config)
+
+        assert updated.internal[0].total_score == 5.0
diff --git a/tests/test_multi_config.py b/tests/test_multi_config.py
new file mode 100644
index 0000000..09dd528
--- /dev/null
+++ b/tests/test_multi_config.py
@@ -0,0 +1,117 @@
+"""
+Test example for multiple crawler configs feature
+"""
+import asyncio
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, MatchMode, CacheMode
+
+async def test_multi_config():
+    # Create different configs for different URL patterns
+    
+    # Config for PDF files
+    pdf_config = CrawlerRunConfig(
+        url_matcher="*.pdf",
+    )
+    
+    # Config for articles (using multiple patterns with OR logic)
+    article_config = CrawlerRunConfig(
+        url_matcher=["*/news/*", "*blog*", "*/article/*"],
+        match_mode=MatchMode.OR,
+        screenshot=True,
+    )
+    
+    # Config using custom matcher function
+    api_config = CrawlerRunConfig(
+        url_matcher=lambda url: 'api' in url or 'json' in url,
+    )
+    
+    # Config combining patterns and functions with AND logic
+    secure_docs_config = CrawlerRunConfig(
+        url_matcher=[
+            "*.doc*",  # Matches .doc, .docx
+            lambda url: url.startswith('https://')  # Must be HTTPS
+        ],
+        match_mode=MatchMode.AND,
+    )
+    
+    # Default config (no url_matcher means it won't match anything unless it's the fallback)
+    default_config = CrawlerRunConfig(
+        # cache_mode=CacheMode.BYPASS,
+    )
+    
+    # List of configs - order matters! First match wins
+    configs = [
+        pdf_config,
+        article_config, 
+        api_config,
+        secure_docs_config,
+        default_config  # Fallback
+    ]
+    
+    # Test URLs - using real URLs that exist
+    test_urls = [
+        "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",  # Real PDF
+        "https://www.bbc.com/news/articles/c5y3e3glnldo",  # News article
+        "https://blog.python.org/",  # Blog URL  
+        "https://api.github.com/users/github",  # GitHub API (returns JSON)
+        "https://httpbin.org/json",  # API endpoint that returns JSON
+        "https://www.python.org/",  # Generic HTTPS page
+        "http://info.cern.ch/",  # HTTP (not HTTPS) page
+        "https://example.com/",  # → Default config
+    ]
+    
+    # Test the matching logic
+    print("Config matching test:")
+    print("-" * 50)
+    for url in test_urls:
+        for i, config in enumerate(configs):
+            if config.is_match(url):
+                print(f"{url} -> Config {i} matches")
+                break
+        else:
+            print(f"{url} -> No match, will use fallback (first config)")
+    
+    print("\n" + "=" * 50 + "\n")
+    
+    # Now test with actual crawler
+    async with AsyncWebCrawler() as crawler:
+        # Single config - traditional usage still works
+        print("Test 1: Single config (backwards compatible)")
+        result = await crawler.arun_many(
+            urls=["https://www.python.org/"],
+            config=default_config
+        )
+        print(f"Crawled {len(result)} URLs with single config\n")
+        
+        # Multiple configs - new feature
+        print("Test 2: Multiple configs")
+        # Just test with 2 URLs to avoid timeout
+        results = await crawler.arun_many(
+            urls=test_urls[:2],  # Just test first 2 URLs
+            config=configs  # Pass list of configs
+        )
+        print(f"Crawled {len(results)} URLs with multiple configs")
+        
+        # Using custom matcher inline
+        print("\nTest 3: Inline custom matcher")
+        custom_config = CrawlerRunConfig(
+            url_matcher=lambda url: len(url) > 50 and 'python' in url.lower(),
+            verbose=False
+        )
+        results = await crawler.arun_many(
+            urls=[
+                "https://docs.python.org/3/library/asyncio.html",  # Long URL with 'python'
+                "https://python.org/",  # Short URL with 'python' - won't match
+                "https://www.google.com/"  # No 'python' - won't match
+            ],
+            config=[custom_config, default_config]
+        )
+        print(f"Crawled {len(results)} URLs with custom matcher")
+
+if __name__ == "__main__":
+    asyncio.run(test_multi_config())
\ No newline at end of file
diff --git a/tests/test_normalize_url.py b/tests/test_normalize_url.py
new file mode 100644
index 0000000..b1f1cc7
--- /dev/null
+++ b/tests/test_normalize_url.py
@@ -0,0 +1,91 @@
+import unittest
+from crawl4ai.utils import normalize_url
+
+class TestNormalizeUrl(unittest.TestCase):
+
+    def test_basic_relative_path(self):
+        self.assertEqual(normalize_url("path/to/page.html", "http://example.com/base/"), "http://example.com/base/path/to/page.html")
+
+    def test_base_url_with_trailing_slash(self):
+        self.assertEqual(normalize_url("page.html", "http://example.com/base/"), "http://example.com/base/page.html")
+
+    def test_base_url_without_trailing_slash(self):
+        # If normalize_url correctly uses urljoin, "base" is treated as a file.
+        self.assertEqual(normalize_url("page.html", "http://example.com/base"), "http://example.com/page.html")
+
+    def test_absolute_url_as_href(self):
+        self.assertEqual(normalize_url("http://another.com/page.html", "http://example.com/"), "http://another.com/page.html")
+
+    def test_href_with_leading_trailing_spaces(self):
+        self.assertEqual(normalize_url("  page.html  ", "http://example.com/"), "http://example.com/page.html")
+
+    def test_empty_href(self):
+        # urljoin with an empty href and base ending in '/' returns the base.
+        self.assertEqual(normalize_url("", "http://example.com/base/"), "http://example.com/base/")
+        # urljoin with an empty href and base not ending in '/' also returns base.
+        self.assertEqual(normalize_url("", "http://example.com/base"), "http://example.com/base")
+
+    def test_href_with_query_parameters(self):
+        self.assertEqual(normalize_url("page.html?query=test", "http://example.com/"), "http://example.com/page.html?query=test")
+
+    def test_href_with_fragment(self):
+        self.assertEqual(normalize_url("page.html#section", "http://example.com/"), "http://example.com/page.html#section")
+
+    def test_different_scheme_in_href(self):
+        self.assertEqual(normalize_url("https://secure.example.com/page.html", "http://example.com/"), "https://secure.example.com/page.html")
+
+    def test_parent_directory_in_href(self):
+        self.assertEqual(normalize_url("../otherpage.html", "http://example.com/base/current/"), "http://example.com/base/otherpage.html")
+
+    def test_root_relative_href(self):
+        self.assertEqual(normalize_url("/otherpage.html", "http://example.com/base/current/"), "http://example.com/otherpage.html")
+
+    def test_base_url_with_path_and_no_trailing_slash(self):
+        # If normalize_url correctly uses urljoin, "path" is treated as a file.
+        self.assertEqual(normalize_url("file.html", "http://example.com/path"), "http://example.com/file.html")
+
+    def test_base_url_is_just_domain(self):
+        self.assertEqual(normalize_url("page.html", "http://example.com"), "http://example.com/page.html")
+
+    def test_href_is_only_query(self):
+        self.assertEqual(normalize_url("?query=true", "http://example.com/page.html"), "http://example.com/page.html?query=true")
+
+    def test_href_is_only_fragment(self):
+        self.assertEqual(normalize_url("#fragment", "http://example.com/page.html"), "http://example.com/page.html#fragment")
+
+    def test_relative_link_from_base_file_url(self):
+        """
+        Tests the specific bug report: relative links from a base URL that is a file.
+        Example:
+        Page URL: http://example.com/path/to/document.html
+        Link on page: 
+        Expected: http://example.com/path/to/file.xlsx
+        """
+        base_url_file = "http://example.com/zwgk/fdzdgk/zdxx/spaq/t19360680.shtml"
+        href_relative_current_dir = "./P020241203375994691134.xlsx"
+        expected_url1 = "http://example.com/zwgk/fdzdgk/zdxx/spaq/P020241203375994691134.xlsx"
+        self.assertEqual(normalize_url(href_relative_current_dir, base_url_file), expected_url1)
+
+        # Test with a relative link that doesn't start with "./"
+        href_relative_no_dot_slash = "another.doc"
+        expected_url2 = "http://example.com/zwgk/fdzdgk/zdxx/spaq/another.doc"
+        self.assertEqual(normalize_url(href_relative_no_dot_slash, base_url_file), expected_url2)
+
+    def test_invalid_base_url_scheme(self):
+        with self.assertRaises(ValueError) as context:
+            normalize_url("page.html", "ftp://example.com/")
+        self.assertIn("Invalid base URL format", str(context.exception))
+
+    def test_invalid_base_url_netloc(self):
+        with self.assertRaises(ValueError) as context:
+            normalize_url("page.html", "http:///path/")
+        self.assertIn("Invalid base URL format", str(context.exception))
+        
+    def test_base_url_with_port(self):
+        self.assertEqual(normalize_url("path/file.html", "http://example.com:8080/base/"), "http://example.com:8080/base/path/file.html")
+
+    def test_href_with_special_characters(self):
+        self.assertEqual(normalize_url("path%20with%20spaces/file.html", "http://example.com/"), "http://example.com/path%20with%20spaces/file.html")
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file
diff --git a/tests/test_pr_1290_1668.py b/tests/test_pr_1290_1668.py
new file mode 100644
index 0000000..6b6afb5
--- /dev/null
+++ b/tests/test_pr_1290_1668.py
@@ -0,0 +1,167 @@
+"""
+Tests for PR #1290 and #1668
+
+- #1290: Type-list pipeline in JsonCssExtractionStrategy._extract_single_field
+- #1668: --json-ensure-ascii CLI flag and JSON_ENSURE_ASCII config
+"""
+import json
+import pytest
+from bs4 import BeautifulSoup
+
+
+# ── PR #1290: Type-list pipeline in _extract_single_field ─────────────────
+
+
+class TestTypePipeline:
+    """Test that field type can be a list for chained extraction."""
+
+    @pytest.fixture
+    def strategy(self):
+        from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+        schema = {"name": "test", "baseSelector": "div", "fields": []}
+        return JsonCssExtractionStrategy(schema)
+
+    @pytest.fixture
+    def element(self):
+        html = ''
+        soup = BeautifulSoup(html, "html.parser")
+        return soup.find("div")
+
+    def test_single_type_text_still_works(self, strategy, element):
+        """Single string type 'text' should still work as before."""
+        field = {"selector": "a", "type": "text"}
+        result = strategy._extract_single_field(element, field)
+        assert result == "Product Name"
+
+    def test_single_type_attribute_still_works(self, strategy, element):
+        """Single string type 'attribute' should still work."""
+        field = {"selector": "a", "type": "attribute", "attribute": "href"}
+        result = strategy._extract_single_field(element, field)
+        assert result == "/product/12345?ref=home"
+
+    def test_single_type_html_still_works(self, strategy, element):
+        """Single string type 'html' should still work."""
+        field = {"selector": "a", "type": "html"}
+        result = strategy._extract_single_field(element, field)
+        assert "Product Name" in result
+        assert "href" in result
+
+    def test_single_type_regex_still_works(self, strategy, element):
+        """Single string type 'regex' should still work (reads text, applies pattern)."""
+        field = {"selector": "a", "type": "regex", "pattern": r"Product (\w+)"}
+        result = strategy._extract_single_field(element, field)
+        assert result == "Name"
+
+    def test_pipeline_attribute_then_regex(self, strategy, element):
+        """Pipeline: get attribute, then regex-extract from it."""
+        field = {
+            "selector": "a",
+            "type": ["attribute", "regex"],
+            "attribute": "href",
+            "pattern": r"/product/(\d+)",
+        }
+        result = strategy._extract_single_field(element, field)
+        assert result == "12345"
+
+    def test_pipeline_html_then_regex(self, strategy, element):
+        """Pipeline: get HTML, then regex-extract from it."""
+        field = {
+            "selector": "a",
+            "type": ["html", "regex"],
+            "pattern": r'href="([^"]+)"',
+        }
+        result = strategy._extract_single_field(element, field)
+        assert result == "/product/12345?ref=home"
+
+    def test_pipeline_text_then_regex(self, strategy, element):
+        """Pipeline: get text, then regex — same as single 'regex' type."""
+        field = {
+            "selector": "a",
+            "type": ["text", "regex"],
+            "pattern": r"Product (\w+)",
+        }
+        result = strategy._extract_single_field(element, field)
+        assert result == "Name"
+
+    def test_pipeline_stops_on_none(self, strategy, element):
+        """Pipeline should stop and return default when a step yields None."""
+        field = {
+            "selector": "a",
+            "type": ["attribute", "regex"],
+            "attribute": "href",
+            "pattern": r"NOMATCH(\d+)",
+            "default": "N/A",
+        }
+        result = strategy._extract_single_field(element, field)
+        assert result == "N/A"
+
+    def test_pipeline_custom_group(self, strategy):
+        """Pipeline with custom regex group number."""
+        html = '
      text
      '
+        soup = BeautifulSoup(html, "html.parser")
+        doc = soup.find("div")
+        field = {
+            "selector": "span",
+            "type": ["attribute", "regex"],
+            "attribute": "data-info",
+            "pattern": r"(\w+):(\w+)",
+            "group": 2,
+        }
+        result = strategy._extract_single_field(doc, field)
+        assert result == "value123"
+
+    def test_single_element_list_same_as_string(self, strategy, element):
+        """A list with one element should behave identically to a string."""
+        field_str = {"selector": "a", "type": "text"}
+        field_list = {"selector": "a", "type": ["text"]}
+        assert strategy._extract_single_field(element, field_str) == \
+               strategy._extract_single_field(element, field_list)
+
+
+# ── PR #1668: JSON_ENSURE_ASCII config setting ────────────────────────────
+
+
+class TestJsonEnsureAsciiConfig:
+    """Test the JSON_ENSURE_ASCII configuration setting."""
+
+    def test_user_settings_has_json_ensure_ascii(self):
+        """USER_SETTINGS should include JSON_ENSURE_ASCII."""
+        from crawl4ai.config import USER_SETTINGS
+        assert "JSON_ENSURE_ASCII" in USER_SETTINGS
+        assert USER_SETTINGS["JSON_ENSURE_ASCII"]["default"] is True
+        assert USER_SETTINGS["JSON_ENSURE_ASCII"]["type"] == "boolean"
+
+    def test_ensure_ascii_true_escapes_unicode(self):
+        """With ensure_ascii=True, non-ASCII chars should be escaped."""
+        data = {"name": "Ján Kováč"}
+        output = json.dumps(data, ensure_ascii=True)
+        assert "\\u" in output
+        assert "Ján" not in output
+
+    def test_ensure_ascii_false_preserves_unicode(self):
+        """With ensure_ascii=False, non-ASCII chars should be preserved."""
+        data = {"name": "Ján Kováč"}
+        output = json.dumps(data, ensure_ascii=False)
+        assert "Ján Kováč" in output
+        assert "\\u" not in output
+
+    def test_cli_has_json_ensure_ascii_option(self):
+        """The crawl_cmd should accept --json-ensure-ascii flag."""
+        import click
+        from crawl4ai.cli import crawl_cmd
+        # Get the click command's params
+        param_names = [p.name for p in crawl_cmd.params]
+        assert "json_ensure_ascii" in param_names
+
+    def test_default_cmd_has_json_ensure_ascii_option(self):
+        """The default command should accept --json-ensure-ascii flag."""
+        from crawl4ai.cli import default
+        param_names = [p.name for p in default.params]
+        assert "json_ensure_ascii" in param_names
+
+    def test_cli_source_uses_ensure_ascii_in_dumps(self):
+        """cli.py should pass ensure_ascii to json.dumps calls."""
+        import inspect
+        from crawl4ai import cli
+        source = inspect.getsource(cli)
+        assert "ensure_ascii=ensure_ascii" in source
diff --git a/tests/test_pr_1435_redirected_status_code.py b/tests/test_pr_1435_redirected_status_code.py
new file mode 100644
index 0000000..f87d1e8
--- /dev/null
+++ b/tests/test_pr_1435_redirected_status_code.py
@@ -0,0 +1,72 @@
+"""Tests for PR #1435: redirected_status_code in CrawlResult."""
+
+import pytest
+import pytest_asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.models import CrawlResult, AsyncCrawlResponse
+
+
+class TestRedirectedStatusCodeModel:
+    """Test that the field exists and defaults correctly on both models."""
+
+    def test_crawl_result_default_none(self):
+        result = CrawlResult(url="http://example.com", html="", success=True)
+        assert result.redirected_status_code is None
+
+    def test_crawl_result_set_value(self):
+        result = CrawlResult(url="http://example.com", html="", success=True, redirected_status_code=200)
+        assert result.redirected_status_code == 200
+
+    def test_async_crawl_response_default_none(self):
+        resp = AsyncCrawlResponse(html="", response_headers={}, status_code=200)
+        assert resp.redirected_status_code is None
+
+    def test_async_crawl_response_set_value(self):
+        resp = AsyncCrawlResponse(html="", response_headers={}, status_code=200, redirected_status_code=301)
+        assert resp.redirected_status_code == 301
+
+
+@pytest.mark.asyncio
+async def test_redirected_status_code_on_direct_request():
+    """A non-redirected request should have redirected_status_code equal to the final status."""
+    browser_config = BrowserConfig(headless=True)
+    run_config = CrawlerRunConfig()
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun("https://httpbin.org/get", config=run_config)
+
+    assert result.success
+    # Direct request — redirected_status_code should be the final response status (200)
+    assert result.redirected_status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_redirected_status_code_on_redirect():
+    """A redirected request should capture the final destination's status code."""
+    browser_config = BrowserConfig(headless=True)
+    run_config = CrawlerRunConfig()
+
+    # httpbin /redirect/1 does a 302 redirect to /get (which returns 200)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun("https://httpbin.org/redirect/1", config=run_config)
+
+    assert result.success
+    # status_code should be 302 (the first hop, per crawl4ai's redirect chain walking)
+    assert result.status_code == 302
+    # redirected_status_code should be 200 (the final destination)
+    assert result.redirected_status_code == 200
+    # redirected_url should point to the final destination
+    assert "/get" in (result.redirected_url or "")
+
+
+@pytest.mark.asyncio
+async def test_redirected_status_code_on_raw_html():
+    """Raw HTML input should have redirected_status_code = None (no network request)."""
+    browser_config = BrowserConfig(headless=True)
+    run_config = CrawlerRunConfig()
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun("raw:test", config=run_config)
+
+    assert result.success
+    assert result.redirected_status_code is None
diff --git a/tests/test_pr_1463_device_scale_factor.py b/tests/test_pr_1463_device_scale_factor.py
new file mode 100644
index 0000000..7fbc717
--- /dev/null
+++ b/tests/test_pr_1463_device_scale_factor.py
@@ -0,0 +1,62 @@
+"""Tests for PR #1463: configurable device_scale_factor in BrowserConfig."""
+
+import pytest
+import pytest_asyncio
+import base64
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+
+class TestDeviceScaleFactorConfig:
+    """Test that device_scale_factor flows correctly through BrowserConfig."""
+
+    def test_default_value(self):
+        config = BrowserConfig()
+        assert config.device_scale_factor == 1.0
+
+    def test_custom_value(self):
+        config = BrowserConfig(device_scale_factor=2.0)
+        assert config.device_scale_factor == 2.0
+
+    def test_to_dict_includes_field(self):
+        config = BrowserConfig(device_scale_factor=3.0)
+        d = config.to_dict()
+        assert d["device_scale_factor"] == 3.0
+
+    def test_clone_preserves(self):
+        config = BrowserConfig(device_scale_factor=2.5)
+        cloned = config.clone()
+        assert cloned.device_scale_factor == 2.5
+
+    def test_from_kwargs(self):
+        config = BrowserConfig.from_kwargs({"device_scale_factor": 1.5})
+        assert config.device_scale_factor == 1.5
+
+    def test_from_kwargs_default(self):
+        config = BrowserConfig.from_kwargs({})
+        assert config.device_scale_factor == 1.0
+
+
+@pytest.mark.asyncio
+async def test_device_scale_factor_produces_larger_screenshot():
+    """Integration test: higher device_scale_factor should produce a larger screenshot."""
+    html = "
      Scale Test
      "
+    raw_url = f"raw:{html}"
+    run_config = CrawlerRunConfig(screenshot=True)
+
+    # Take screenshot at scale 1.0
+    browser_1x = BrowserConfig(headless=True, device_scale_factor=1.0, viewport_width=800, viewport_height=600)
+    async with AsyncWebCrawler(config=browser_1x) as crawler:
+        result_1x = await crawler.arun(raw_url, config=run_config)
+
+    # Take screenshot at scale 2.0
+    browser_2x = BrowserConfig(headless=True, device_scale_factor=2.0, viewport_width=800, viewport_height=600)
+    async with AsyncWebCrawler(config=browser_2x) as crawler:
+        result_2x = await crawler.arun(raw_url, config=run_config)
+
+    assert result_1x.screenshot is not None
+    assert result_2x.screenshot is not None
+
+    # 2x scale should produce more pixel data (larger base64 string)
+    size_1x = len(base64.b64decode(result_1x.screenshot))
+    size_2x = len(base64.b64decode(result_2x.screenshot))
+    assert size_2x > size_1x, f"2x screenshot ({size_2x} bytes) should be larger than 1x ({size_1x} bytes)"
diff --git a/tests/test_pr_1795_1798_1734.py b/tests/test_pr_1795_1798_1734.py
new file mode 100644
index 0000000..7b79919
--- /dev/null
+++ b/tests/test_pr_1795_1798_1734.py
@@ -0,0 +1,222 @@
+"""
+Tests for PR #1795, #1798, #1734
+
+- #1795: /token endpoint requires api_token when configured
+- #1798: Deep-crawl streaming branches to arun() for single URL
+- #1734: GitHub Actions versions bumped to latest
+"""
+import pytest
+import yaml
+import ast
+from unittest.mock import AsyncMock, MagicMock, patch
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+# ── PR #1795: api_token protection on /token endpoint ────────────────────
+
+
+class TestTokenEndpointAuth:
+    """Test the api_token gating logic added to the /token endpoint."""
+
+    def test_token_request_model_has_api_token_field(self):
+        """auth.py TokenRequest should have an api_token field in its source."""
+        source = (ROOT / "deploy" / "docker" / "auth.py").read_text()
+        # Parse the AST to verify the field exists on the class
+        tree = ast.parse(source)
+        token_request = None
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef) and node.name == "TokenRequest":
+                token_request = node
+                break
+        assert token_request is not None, "TokenRequest class not found"
+        field_names = [
+            stmt.target.id
+            for stmt in token_request.body
+            if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name)
+        ]
+        assert "email" in field_names, "TokenRequest missing email field"
+        assert "api_token" in field_names, "TokenRequest missing api_token field"
+
+    def test_server_token_check_logic_no_config(self):
+        """When api_token is empty in config, any request should pass."""
+        config = {"security": {"api_token": ""}}
+        expected_token = config.get("security", {}).get("api_token", "")
+        # Empty string is falsy, so check should be skipped
+        assert not expected_token
+
+    def test_server_token_check_logic_with_config_match(self):
+        """When api_token is set and request matches, should pass."""
+        config = {"security": {"api_token": "my-secret"}}
+        expected_token = config.get("security", {}).get("api_token", "")
+        req_token = "my-secret"
+        assert expected_token and req_token == expected_token
+
+    def test_server_token_check_logic_with_config_mismatch(self):
+        """When api_token is set and request doesn't match, should reject."""
+        config = {"security": {"api_token": "my-secret"}}
+        expected_token = config.get("security", {}).get("api_token", "")
+        req_token = "wrong-token"
+        assert expected_token and req_token != expected_token
+
+    def test_server_token_check_logic_with_config_none(self):
+        """When api_token is set and request sends None, should reject."""
+        config = {"security": {"api_token": "my-secret"}}
+        expected_token = config.get("security", {}).get("api_token", "")
+        req_token = None
+        assert expected_token and req_token != expected_token
+
+    def test_config_yml_has_api_token_field(self):
+        """config.yml should include api_token under security."""
+        with open(ROOT / "deploy" / "docker" / "config.yml") as f:
+            cfg = yaml.safe_load(f)
+        assert "api_token" in cfg["security"]
+        # Default should be empty (disabled)
+        assert cfg["security"]["api_token"] == ""
+
+    def test_server_py_contains_token_check(self):
+        """server.py get_token function should check api_token."""
+        source = (ROOT / "deploy" / "docker" / "server.py").read_text()
+        assert "api_token" in source
+        assert 'config.get("security", {}).get("api_token"' in source
+        assert "401" in source  # HTTPException 401
+
+
+# ── PR #1798: Deep-crawl streaming branches on strategy ──────────────────
+
+
+class TestDeepCrawlStreamBranching:
+    """Test the branching logic in handle_stream_crawl_request."""
+
+    def test_api_py_has_deep_crawl_branch(self):
+        """api.py should branch on deep_crawl_strategy for streaming."""
+        source = (ROOT / "deploy" / "docker" / "api.py").read_text()
+        assert "deep_crawl_strategy is not None" in source
+        assert "crawler.arun(" in source  # single-URL deep crawl path
+        assert "crawler.arun_many(" in source  # multi-URL path
+
+    def test_api_py_rejects_multi_url_deep_crawl(self):
+        """api.py should raise 400 for deep crawl with multiple URLs."""
+        source = (ROOT / "deploy" / "docker" / "api.py").read_text()
+        assert "exactly one URL per request" in source
+        assert "HTTP_400_BAD_REQUEST" in source
+
+    @pytest.mark.asyncio
+    async def test_deep_crawl_single_url_uses_arun(self):
+        """With deep_crawl_strategy + 1 URL, should call crawler.arun()."""
+        from crawl4ai import CrawlerRunConfig, BrowserConfig
+        from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+        cfg = CrawlerRunConfig(
+            deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=1, max_pages=5),
+            stream=True,
+        )
+        # Verify the config has deep_crawl_strategy set
+        assert cfg.deep_crawl_strategy is not None
+        assert cfg.stream is True
+
+        # Simulate the branching logic from api.py
+        urls = ["https://example.com"]
+        if cfg.deep_crawl_strategy is not None and len(urls) == 1:
+            path = "arun"
+        else:
+            path = "arun_many"
+        assert path == "arun"
+
+    @pytest.mark.asyncio
+    async def test_no_deep_crawl_uses_arun_many(self):
+        """Without deep_crawl_strategy, should use arun_many()."""
+        from crawl4ai import CrawlerRunConfig
+
+        cfg = CrawlerRunConfig(stream=True)
+        assert cfg.deep_crawl_strategy is None
+
+        urls = ["https://a.com", "https://b.com"]
+        if cfg.deep_crawl_strategy is not None and len(urls) == 1:
+            path = "arun"
+        else:
+            path = "arun_many"
+        assert path == "arun_many"
+
+    @pytest.mark.asyncio
+    async def test_deep_crawl_multi_url_rejected(self):
+        """Deep crawl + multiple URLs should be rejected."""
+        from crawl4ai import CrawlerRunConfig
+        from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+        cfg = CrawlerRunConfig(
+            deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=1, max_pages=5),
+            stream=True,
+        )
+        urls = ["https://a.com", "https://b.com"]
+
+        # This is what api.py does — raise before crawling
+        should_reject = cfg.deep_crawl_strategy is not None and len(urls) != 1
+        assert should_reject
+
+
+# ── PR #1734: GitHub Actions version bumps ────────────────────────────────
+
+
+class TestGitHubActionsVersions:
+    """Verify all GitHub Actions are on current major versions."""
+
+    EXPECTED_VERSIONS = {
+        "actions/checkout": "v6",
+        "actions/setup-python": "v6",
+        "docker/build-push-action": "v6",
+        "docker/setup-buildx-action": "v4",
+        "docker/login-action": "v4",
+        "softprops/action-gh-release": "v2",
+    }
+
+    def _extract_actions(self, workflow_path):
+        """Extract action@version pairs from a workflow file."""
+        with open(workflow_path) as f:
+            data = yaml.safe_load(f)
+        actions = {}
+        for job_name, job in data.get("jobs", {}).items():
+            for step in job.get("steps", []):
+                uses = step.get("uses", "")
+                if "@" in uses:
+                    name, version = uses.rsplit("@", 1)
+                    actions[name] = version
+        return actions
+
+    def test_docker_release_workflow(self):
+        """docker-release.yml should use latest action versions."""
+        actions = self._extract_actions(
+            ROOT / ".github" / "workflows" / "docker-release.yml"
+        )
+        for name, expected in self.EXPECTED_VERSIONS.items():
+            if name in actions:
+                assert actions[name] == expected, (
+                    f"{name} should be @{expected}, got @{actions[name]}"
+                )
+
+    def test_release_workflow(self):
+        """release.yml should use latest action versions."""
+        actions = self._extract_actions(
+            ROOT / ".github" / "workflows" / "release.yml"
+        )
+        for name, expected in self.EXPECTED_VERSIONS.items():
+            if name in actions:
+                assert actions[name] == expected, (
+                    f"{name} should be @{expected}, got @{actions[name]}"
+                )
+
+    def test_no_v4_or_v5_checkout_remaining(self):
+        """No workflow should still reference checkout@v4 or v5."""
+        for wf in (ROOT / ".github" / "workflows").glob("*.yml"):
+            content = wf.read_text()
+            assert "actions/checkout@v4" not in content, f"{wf.name} still uses checkout@v4"
+            assert "actions/checkout@v5" not in content, f"{wf.name} still uses checkout@v5"
+
+    def test_no_old_build_push_action(self):
+        """No workflow should still reference build-push-action@v5."""
+        for wf in (ROOT / ".github" / "workflows").glob("*.yml"):
+            content = wf.read_text()
+            assert "build-push-action@v5" not in content, (
+                f"{wf.name} still uses build-push-action@v5"
+            )
diff --git a/tests/test_prefetch_integration.py b/tests/test_prefetch_integration.py
new file mode 100644
index 0000000..77ed942
--- /dev/null
+++ b/tests/test_prefetch_integration.py
@@ -0,0 +1,236 @@
+"""Integration tests for prefetch mode with the crawler."""
+
+import pytest
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
+
+# Use crawl4ai docs as test domain
+TEST_DOMAIN = "https://docs.crawl4ai.com"
+
+
+class TestPrefetchModeIntegration:
+    """Integration tests for prefetch mode."""
+
+    @pytest.mark.asyncio
+    async def test_prefetch_returns_html_and_links(self):
+        """Test that prefetch mode returns HTML and links only."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(prefetch=True)
+            result = await crawler.arun(TEST_DOMAIN, config=config)
+
+            # Should have HTML
+            assert result.html is not None
+            assert len(result.html) > 0
+            assert "= 1
+
+    @pytest.mark.asyncio
+    async def test_prefetch_then_process_with_raw(self):
+        """Test the full two-phase workflow: prefetch then process."""
+        async with AsyncWebCrawler() as crawler:
+            # Phase 1: Prefetch
+            prefetch_config = CrawlerRunConfig(prefetch=True)
+            prefetch_result = await crawler.arun(TEST_DOMAIN, config=prefetch_config)
+
+            stored_html = prefetch_result.html
+
+            assert stored_html is not None
+            assert len(stored_html) > 0
+
+            # Phase 2: Process with raw: URL
+            process_config = CrawlerRunConfig(
+                # No prefetch - full processing
+                base_url=TEST_DOMAIN  # Provide base URL for link resolution
+            )
+            processed_result = await crawler.arun(
+                f"raw:{stored_html}",
+                config=process_config
+            )
+
+            # Should now have full processing
+            assert processed_result.html is not None
+            assert processed_result.success is True
+            # Note: cleaned_html and markdown depend on the content
+
+    @pytest.mark.asyncio
+    async def test_prefetch_links_structure(self):
+        """Test that links have the expected structure."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(prefetch=True)
+            result = await crawler.arun(TEST_DOMAIN, config=config)
+
+            assert result.links is not None
+
+            # Check internal links structure
+            if result.links["internal"]:
+                link = result.links["internal"][0]
+                assert "href" in link
+                assert "text" in link
+                assert link["href"].startswith("http")
+
+            # Check external links structure (if any)
+            if result.links["external"]:
+                link = result.links["external"][0]
+                assert "href" in link
+                assert "text" in link
+                assert link["href"].startswith("http")
+
+    @pytest.mark.asyncio
+    async def test_prefetch_config_clone(self):
+        """Test that config.clone() preserves prefetch setting."""
+        config = CrawlerRunConfig(prefetch=True)
+        cloned = config.clone()
+
+        assert cloned.prefetch == True
+
+        # Clone with override
+        cloned_false = config.clone(prefetch=False)
+        assert cloned_false.prefetch == False
+
+    @pytest.mark.asyncio
+    async def test_prefetch_to_dict(self):
+        """Test that to_dict() includes prefetch."""
+        config = CrawlerRunConfig(prefetch=True)
+        config_dict = config.to_dict()
+
+        assert "prefetch" in config_dict
+        assert config_dict["prefetch"] == True
+
+    @pytest.mark.asyncio
+    async def test_prefetch_default_false(self):
+        """Test that prefetch defaults to False."""
+        config = CrawlerRunConfig()
+        assert config.prefetch == False
+
+    @pytest.mark.asyncio
+    async def test_prefetch_explicit_false(self):
+        """Test explicit prefetch=False works like default."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(prefetch=False)
+            result = await crawler.arun(TEST_DOMAIN, config=config)
+
+            # Should have full processing
+            assert result.html is not None
+            # cleaned_html should be populated in normal mode
+            assert result.cleaned_html is not None
+
+
+class TestPrefetchPerformance:
+    """Performance-related tests for prefetch mode."""
+
+    @pytest.mark.asyncio
+    async def test_prefetch_returns_quickly(self):
+        """Test that prefetch mode returns results quickly."""
+        import time
+
+        async with AsyncWebCrawler() as crawler:
+            # Prefetch mode
+            start = time.time()
+            prefetch_config = CrawlerRunConfig(prefetch=True)
+            await crawler.arun(TEST_DOMAIN, config=prefetch_config)
+            prefetch_time = time.time() - start
+
+            # Full mode
+            start = time.time()
+            full_config = CrawlerRunConfig()
+            await crawler.arun(TEST_DOMAIN, config=full_config)
+            full_time = time.time() - start
+
+            # Log times for debugging
+            print(f"\nPrefetch: {prefetch_time:.3f}s, Full: {full_time:.3f}s")
+
+            # Prefetch should not be significantly slower
+            # (may be same or slightly faster depending on content)
+            # This is a soft check - mostly for logging
+
+
+class TestPrefetchWithRawHTML:
+    """Test prefetch mode with raw HTML input."""
+
+    @pytest.mark.asyncio
+    async def test_prefetch_with_raw_html(self):
+        """Test prefetch mode works with raw: URL scheme."""
+        sample_html = """
+        
+            Test Page
+            
+                
      Hello World
      
+                Link 1
+                Link 2
+                External
+            
+        
+        """
+
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(
+                prefetch=True,
+                base_url="https://example.com"
+            )
+            result = await crawler.arun(f"raw:{sample_html}", config=config)
+
+            assert result.success is True
+            assert result.html is not None
+            assert result.links is not None
+
+            # Should have extracted links
+            assert len(result.links["internal"]) >= 2
+            assert len(result.links["external"]) >= 1
diff --git a/tests/test_prefetch_mode.py b/tests/test_prefetch_mode.py
new file mode 100644
index 0000000..fdbaa96
--- /dev/null
+++ b/tests/test_prefetch_mode.py
@@ -0,0 +1,275 @@
+"""Unit tests for the quick_extract_links function used in prefetch mode."""
+
+import pytest
+from crawl4ai.utils import quick_extract_links
+
+
+class TestQuickExtractLinks:
+    """Unit tests for the quick_extract_links function."""
+
+    def test_basic_internal_links(self):
+        """Test extraction of internal links."""
+        html = '''
+        
+            
+                Page 1
+                Page 2
+                Page 3
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 3
+        assert result["internal"][0]["href"] == "https://example.com/page1"
+        assert result["internal"][0]["text"] == "Page 1"
+
+    def test_external_links(self):
+        """Test extraction and classification of external links."""
+        html = '''
+        
+            
+                External
+                Internal
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 1
+        assert len(result["external"]) == 1
+        assert result["external"][0]["href"] == "https://other.com/page"
+
+    def test_ignores_javascript_and_mailto(self):
+        """Test that javascript: and mailto: links are ignored."""
+        html = '''
+        
+            
+                Click
+                Email
+                Call
+                Valid
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 1
+        assert result["internal"][0]["href"] == "https://example.com/valid"
+
+    def test_ignores_anchor_only_links(self):
+        """Test that anchor-only links (#section) are ignored."""
+        html = '''
+        
+            
+                Section 1
+                Section 2
+                Page with anchor
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        # Only the page link should be included, anchor-only links are skipped
+        assert len(result["internal"]) == 1
+        assert "/page" in result["internal"][0]["href"]
+
+    def test_deduplication(self):
+        """Test that duplicate URLs are deduplicated."""
+        html = '''
+        
+            
+                Link 1
+                Link 2
+                Link 3
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 1
+
+    def test_handles_malformed_html(self):
+        """Test graceful handling of malformed HTML."""
+        html = "not valid html at all <><><"
+        result = quick_extract_links(html, "https://example.com")
+
+        # Should not raise, should return empty
+        assert result["internal"] == []
+        assert result["external"] == []
+
+    def test_empty_html(self):
+        """Test handling of empty HTML."""
+        result = quick_extract_links("", "https://example.com")
+        assert result == {"internal": [], "external": []}
+
+    def test_relative_url_resolution(self):
+        """Test that relative URLs are resolved correctly."""
+        html = '''
+        
+            
+                Relative
+                Dot Relative
+                Parent Relative
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com/docs/")
+
+        assert len(result["internal"]) >= 1
+        # All should be internal and properly resolved
+        for link in result["internal"]:
+            assert link["href"].startswith("https://example.com")
+
+    def test_text_truncation(self):
+        """Test that long link text is truncated to 200 chars."""
+        long_text = "A" * 300
+        html = f'''
+        
+            
+                {long_text}
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 1
+        assert len(result["internal"][0]["text"]) == 200
+
+    def test_empty_href_ignored(self):
+        """Test that empty href attributes are ignored."""
+        html = '''
+        
+            
+                Empty
+                Whitespace
+                Valid
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 1
+        assert result["internal"][0]["href"] == "https://example.com/valid"
+
+    def test_mixed_internal_external(self):
+        """Test correct classification of mixed internal and external links."""
+        html = '''
+        
+            
+                Internal 1
+                Internal 2
+                Google
+                GitHub
+                Internal 3
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 3
+        assert len(result["external"]) == 2
+
+    def test_subdomain_handling(self):
+        """Test that subdomains are handled correctly."""
+        html = '''
+        
+            
+                Docs subdomain
+                API subdomain
+                Main domain
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        # All should be internal (same base domain)
+        total_links = len(result["internal"]) + len(result["external"])
+        assert total_links == 3
+
+
+class TestQuickExtractLinksEdgeCases:
+    """Edge case tests for quick_extract_links."""
+
+    def test_no_links_in_page(self):
+        """Test page with no links."""
+        html = '''
+        
+            
+                
      No Links Here
      
+                
      Just some text content.
      
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert result["internal"] == []
+        assert result["external"] == []
+
+    def test_links_in_nested_elements(self):
+        """Test links nested in various elements."""
+        html = '''
+        
+            
+                
+                
      
+                    
      Check out our products.
      
+                
      
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 3
+
+    def test_link_with_nested_elements(self):
+        """Test links containing nested elements."""
+        html = '''
+        
+            
+                Nested Text
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        assert len(result["internal"]) == 1
+        assert "Nested" in result["internal"][0]["text"]
+        assert "Text" in result["internal"][0]["text"]
+
+    def test_protocol_relative_urls(self):
+        """Test handling of protocol-relative URLs (//example.com)."""
+        html = '''
+        
+            
+                CDN Link
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        # Should be resolved with https:
+        total = len(result["internal"]) + len(result["external"])
+        assert total >= 1
+
+    def test_whitespace_in_href(self):
+        """Test handling of whitespace around href values."""
+        html = '''
+        
+            
+                Padded
+                Multiline
+            
+        
+        '''
+        result = quick_extract_links(html, "https://example.com")
+
+        # Both should be extracted and normalized
+        assert len(result["internal"]) >= 1
diff --git a/tests/test_prefetch_regression.py b/tests/test_prefetch_regression.py
new file mode 100644
index 0000000..515e90c
--- /dev/null
+++ b/tests/test_prefetch_regression.py
@@ -0,0 +1,232 @@
+"""Regression tests to ensure prefetch mode doesn't break existing functionality."""
+
+import pytest
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+TEST_URL = "https://docs.crawl4ai.com"
+
+
+class TestNoRegressions:
+    """Ensure prefetch mode doesn't break existing functionality."""
+
+    @pytest.mark.asyncio
+    async def test_default_mode_unchanged(self):
+        """Test that default mode (prefetch=False) works exactly as before."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig()  # Default config
+            result = await crawler.arun(TEST_URL, config=config)
+
+            # All standard fields should be populated
+            assert result.html is not None
+            assert result.cleaned_html is not None
+            assert result.links is not None
+            assert result.success is True
+
+    @pytest.mark.asyncio
+    async def test_explicit_prefetch_false(self):
+        """Test explicit prefetch=False works like default."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(prefetch=False)
+            result = await crawler.arun(TEST_URL, config=config)
+
+            assert result.cleaned_html is not None
+
+    @pytest.mark.asyncio
+    async def test_config_clone_preserves_prefetch(self):
+        """Test that config.clone() preserves prefetch setting."""
+        config = CrawlerRunConfig(prefetch=True)
+        cloned = config.clone()
+
+        assert cloned.prefetch == True
+
+        # Clone with override
+        cloned_false = config.clone(prefetch=False)
+        assert cloned_false.prefetch == False
+
+    @pytest.mark.asyncio
+    async def test_config_to_dict_includes_prefetch(self):
+        """Test that to_dict() includes prefetch."""
+        config_true = CrawlerRunConfig(prefetch=True)
+        config_false = CrawlerRunConfig(prefetch=False)
+
+        assert config_true.to_dict()["prefetch"] == True
+        assert config_false.to_dict()["prefetch"] == False
+
+    @pytest.mark.asyncio
+    async def test_existing_extraction_still_works(self):
+        """Test that extraction strategies still work in normal mode."""
+        from crawl4ai import JsonCssExtractionStrategy
+
+        schema = {
+            "name": "Links",
+            "baseSelector": "a",
+            "fields": [
+                {"name": "href", "selector": "", "type": "attribute", "attribute": "href"},
+                {"name": "text", "selector": "", "type": "text"}
+            ]
+        }
+
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(
+                extraction_strategy=JsonCssExtractionStrategy(schema=schema)
+            )
+            result = await crawler.arun(TEST_URL, config=config)
+
+            assert result.extracted_content is not None
+
+    @pytest.mark.asyncio
+    async def test_existing_deep_crawl_still_works(self):
+        """Test that deep crawl without prefetch still does full processing."""
+        from crawl4ai import BFSDeepCrawlStrategy
+
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(
+                deep_crawl_strategy=BFSDeepCrawlStrategy(
+                    max_depth=1,
+                    max_pages=2
+                )
+                # No prefetch - should do full processing
+            )
+
+            result_container = await crawler.arun(TEST_URL, config=config)
+
+            # Handle both list and iterator results
+            if hasattr(result_container, '__aiter__'):
+                results = [r async for r in result_container]
+            else:
+                results = list(result_container) if hasattr(result_container, '__iter__') else [result_container]
+
+            # Each result should have full processing
+            for result in results:
+                assert result.cleaned_html is not None
+
+            assert len(results) >= 1
+
+    @pytest.mark.asyncio
+    async def test_raw_url_scheme_still_works(self):
+        """Test that raw: URL scheme works for processing stored HTML."""
+        sample_html = """
+        
+            Test Page
+            
+                
      Hello World
      
+                
      This is a test paragraph.
      
+                Link 1
+            
+        
+        """
+
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig()
+            result = await crawler.arun(f"raw:{sample_html}", config=config)
+
+            assert result.success is True
+            assert result.html is not None
+            assert "Hello World" in result.html
+            assert result.cleaned_html is not None
+
+    @pytest.mark.asyncio
+    async def test_screenshot_still_works(self):
+        """Test that screenshot option still works in normal mode."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(screenshot=True)
+            result = await crawler.arun(TEST_URL, config=config)
+
+            assert result.success is True
+            # Screenshot data should be present
+            assert result.screenshot is not None or result.screenshot_data is not None
+
+    @pytest.mark.asyncio
+    async def test_js_execution_still_works(self):
+        """Test that JavaScript execution still works in normal mode."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(
+                js_code="document.querySelector('h1')?.textContent"
+            )
+            result = await crawler.arun(TEST_URL, config=config)
+
+            assert result.success is True
+            assert result.html is not None
+
+
+class TestPrefetchDoesNotAffectOtherModes:
+    """Test that prefetch doesn't interfere with other configurations."""
+
+    @pytest.mark.asyncio
+    async def test_prefetch_with_other_options_ignored(self):
+        """Test that other options are properly ignored in prefetch mode."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(
+                prefetch=True,
+                # These should be ignored in prefetch mode
+                screenshot=True,
+                pdf=True,
+                only_text=True,
+                word_count_threshold=100
+            )
+            result = await crawler.arun(TEST_URL, config=config)
+
+            # Should still return HTML and links
+            assert result.html is not None
+            assert result.links is not None
+
+            # But should NOT have processed content
+            assert result.cleaned_html is None
+            assert result.extracted_content is None
+
+    @pytest.mark.asyncio
+    async def test_stream_mode_still_works(self):
+        """Test that stream mode still works normally."""
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(stream=True)
+            result = await crawler.arun(TEST_URL, config=config)
+
+            assert result.success is True
+            assert result.html is not None
+
+    @pytest.mark.asyncio
+    async def test_cache_mode_still_works(self):
+        """Test that cache mode still works normally."""
+        from crawl4ai import CacheMode
+
+        async with AsyncWebCrawler() as crawler:
+            # First request - bypass cache
+            config1 = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+            result1 = await crawler.arun(TEST_URL, config=config1)
+            assert result1.success is True
+
+            # Second request - should work
+            config2 = CrawlerRunConfig(cache_mode=CacheMode.ENABLED)
+            result2 = await crawler.arun(TEST_URL, config=config2)
+            assert result2.success is True
+
+
+class TestBackwardsCompatibility:
+    """Test backwards compatibility with existing code patterns."""
+
+    @pytest.mark.asyncio
+    async def test_config_without_prefetch_works(self):
+        """Test that configs created without prefetch parameter work."""
+        # Simulating old code that doesn't know about prefetch
+        config = CrawlerRunConfig(
+            word_count_threshold=50,
+            css_selector="body"
+        )
+
+        # Should default to prefetch=False
+        assert config.prefetch == False
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(TEST_URL, config=config)
+            assert result.success is True
+            assert result.cleaned_html is not None
+
+    @pytest.mark.asyncio
+    async def test_from_kwargs_without_prefetch(self):
+        """Test CrawlerRunConfig.from_kwargs works without prefetch."""
+        config = CrawlerRunConfig.from_kwargs({
+            "word_count_threshold": 50,
+            "verbose": False
+        })
+
+        assert config.prefetch == False
diff --git a/tests/test_preserve_https_for_internal_links.py b/tests/test_preserve_https_for_internal_links.py
new file mode 100644
index 0000000..8988f1c
--- /dev/null
+++ b/tests/test_preserve_https_for_internal_links.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""
+Final test and demo for HTTPS preservation feature (Issue #1410)
+
+This demonstrates how the preserve_https_for_internal_links flag
+prevents HTTPS downgrade when servers redirect to HTTP.
+"""
+
+import sys
+import os
+from urllib.parse import urljoin, urlparse
+
+def demonstrate_issue():
+    """Show the problem: HTTPS -> HTTP redirect causes HTTP links"""
+    
+    print("=" * 60)
+    print("DEMONSTRATING THE ISSUE")
+    print("=" * 60)
+    
+    # Simulate what happens during crawling
+    original_url = "https://quotes.toscrape.com/tag/deep-thoughts"
+    redirected_url = "http://quotes.toscrape.com/tag/deep-thoughts/"  # Server redirects to HTTP
+    
+    # Extract a relative link
+    relative_link = "/author/Albert-Einstein"
+    
+    # Standard URL joining uses the redirected (HTTP) base
+    resolved_url = urljoin(redirected_url, relative_link)
+    
+    print(f"Original URL:    {original_url}")
+    print(f"Redirected to:   {redirected_url}")
+    print(f"Relative link:   {relative_link}")
+    print(f"Resolved link:   {resolved_url}")
+    print(f"\n❌ Problem: Link is now HTTP instead of HTTPS!")
+    
+    return resolved_url
+
+def demonstrate_solution():
+    """Show the solution: preserve HTTPS for internal links"""
+    
+    print("\n" + "=" * 60)
+    print("DEMONSTRATING THE SOLUTION")
+    print("=" * 60)
+    
+    # Our normalize_url with HTTPS preservation
+    def normalize_url_with_preservation(href, base_url, preserve_https=False, original_scheme=None):
+        """Normalize URL with optional HTTPS preservation"""
+        
+        # Standard resolution
+        full_url = urljoin(base_url, href.strip())
+        
+        # Preserve HTTPS if requested
+        if preserve_https and original_scheme == 'https':
+            parsed_full = urlparse(full_url)
+            parsed_base = urlparse(base_url)
+            
+            # Only for same-domain links
+            if parsed_full.scheme == 'http' and parsed_full.netloc == parsed_base.netloc:
+                full_url = full_url.replace('http://', 'https://', 1)
+                print(f"  → Preserved HTTPS for {parsed_full.netloc}")
+        
+        return full_url
+    
+    # Same scenario as before
+    original_url = "https://quotes.toscrape.com/tag/deep-thoughts"
+    redirected_url = "http://quotes.toscrape.com/tag/deep-thoughts/"
+    relative_link = "/author/Albert-Einstein"
+    
+    # Without preservation (current behavior)
+    resolved_without = normalize_url_with_preservation(
+        relative_link, redirected_url,
+        preserve_https=False, original_scheme='https'
+    )
+    
+    print(f"\nWithout preservation:")
+    print(f"  Result: {resolved_without}")
+    
+    # With preservation (new feature)
+    resolved_with = normalize_url_with_preservation(
+        relative_link, redirected_url,
+        preserve_https=True, original_scheme='https'
+    )
+    
+    print(f"\nWith preservation (preserve_https_for_internal_links=True):")
+    print(f"  Result: {resolved_with}")
+    print(f"\n✅ Solution: Internal link stays HTTPS!")
+    
+    return resolved_with
+
+def test_edge_cases():
+    """Test important edge cases"""
+    
+    print("\n" + "=" * 60)
+    print("EDGE CASES")
+    print("=" * 60)
+    
+    from urllib.parse import urljoin, urlparse
+    
+    def preserve_https(href, base_url, original_scheme):
+        """Helper to test preservation logic"""
+        full_url = urljoin(base_url, href)
+        
+        if original_scheme == 'https':
+            parsed_full = urlparse(full_url)
+            parsed_base = urlparse(base_url)
+            # Fixed: check for protocol-relative URLs
+            if (parsed_full.scheme == 'http' and 
+                parsed_full.netloc == parsed_base.netloc and
+                not href.strip().startswith('//')):
+                full_url = full_url.replace('http://', 'https://', 1)
+        
+        return full_url
+    
+    test_cases = [
+        # (description, href, base_url, original_scheme, should_be_https)
+        ("External link", "http://other.com/page", "http://example.com", "https", False),
+        ("Already HTTPS", "/page", "https://example.com", "https", True),
+        ("No original HTTPS", "/page", "http://example.com", "http", False),
+        ("Subdomain", "/page", "http://sub.example.com", "https", True),
+        ("Protocol-relative", "//example.com/page", "http://example.com", "https", False),
+    ]
+    
+    for desc, href, base_url, orig_scheme, should_be_https in test_cases:
+        result = preserve_https(href, base_url, orig_scheme)
+        is_https = result.startswith('https://')
+        status = "✅" if is_https == should_be_https else "❌"
+        
+        print(f"\n{status} {desc}:")
+        print(f"  Input: {href} + {base_url}")
+        print(f"  Result: {result}")
+        print(f"  Expected HTTPS: {should_be_https}, Got: {is_https}")
+
+def usage_example():
+    """Show how to use the feature in crawl4ai"""
+    
+    print("\n" + "=" * 60)
+    print("USAGE IN CRAWL4AI")
+    print("=" * 60)
+    
+    print("""
+To enable HTTPS preservation in your crawl4ai code:
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async with AsyncWebCrawler() as crawler:
+    config = CrawlerRunConfig(
+        preserve_https_for_internal_links=True  # Enable HTTPS preservation
+    )
+    
+    result = await crawler.arun(
+        url="https://example.com",
+        config=config
+    )
+    
+    # All internal links will maintain HTTPS even if 
+    # the server redirects to HTTP
+```
+
+This is especially useful for:
+- Sites that redirect HTTPS to HTTP but still support HTTPS
+- Security-conscious crawling where you want to stay on HTTPS
+- Avoiding mixed content issues in downstream processing
+""")
+
+if __name__ == "__main__":
+    # Run all demonstrations
+    demonstrate_issue()
+    demonstrate_solution() 
+    test_edge_cases()
+    usage_example()
+    
+    print("\n" + "=" * 60)
+    print("✅ All tests complete!")
+    print("=" * 60)
\ No newline at end of file
diff --git a/tests/test_pruning_preserve_whitelist_1900.py b/tests/test_pruning_preserve_whitelist_1900.py
new file mode 100644
index 0000000..01a4681
--- /dev/null
+++ b/tests/test_pruning_preserve_whitelist_1900.py
@@ -0,0 +1,258 @@
+"""
+Tests for #1900: PruningContentFilter preserve_classes and preserve_tags.
+
+Verifies that whitelisted classes/tags are always kept regardless of
+pruning score, while non-whitelisted content is still pruned normally.
+"""
+import pytest
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+
+# ── HTML fixtures ────────────────────────────────────────────────────────
+
+GITHUB_COMMENT_HTML = """
+
+
      
+    
      Discussion: Feature Request
      
+    
      This is a long paragraph about the feature request with enough words to
+    pass the pruning threshold easily. The feature would add support for document
+    extraction in the crawl pipeline, enabling binary documents like PDFs and
+    DOCX files to be processed alongside HTML pages.
      
+
+    
      
+        
      
+            alice
+            
+        
      
+        
      
+            
      I think this is a great idea. We should implement it using a
+            pluggable strategy pattern so users can bring their own extraction
+            backend. This would keep the core library lean while supporting
+            many document types.
      
+        
      
+    
      
+
+    
      
+        
      
+            bob
+            
+        
      
+        
      
+            
      Agreed with alice. The abstract base class approach makes sense.
+            We could also add a built-in implementation for PDFs since crawl4ai
+            already has PDFContentScrapingStrategy that could be wrapped.
      
+        
      
+    
      
+
      
+
+
+
      
+    
      Copyright 2026
      
+
      
+
+"""
+
+ATTRIBUTION_HTML = """
+
+
      
+    
      Long article content that should definitely pass the threshold because it
+    contains enough words and text density to score well in the pruning algorithm.
+    This paragraph discusses the implementation details of the feature.
      
+    
      By Jane Smith
      
+    
      Jane is a senior engineer at Example Corp.
      
+
      
+
+"""
+
+SIMPLE_HTML = """
+
+
      
+    
      Main content paragraph with enough text to pass pruning easily. This
+    discusses important topics that should be preserved in the output.
      
+    Source: Example Research Paper, 2026
+
      
+
+"""
+
+
+# ── Default behavior (no whitelist) ──────────────────────────────────────
+
+class TestDefaultBehavior:
+
+    def test_default_no_preserve(self):
+        """Without whitelist, default pruning behavior is unchanged."""
+        f = PruningContentFilter()
+        assert f.preserve_classes == set()
+        assert f.preserve_tags == set()
+
+    def test_main_content_kept(self):
+        """Long paragraphs should still be kept."""
+        f = PruningContentFilter()
+        result = f.filter_content(GITHUB_COMMENT_HTML)
+        combined = " ".join(result)
+        assert "feature request" in combined.lower()
+
+    def test_nav_footer_still_removed(self):
+        """Nav and footer should still be removed even with whitelist on other things."""
+        f = PruningContentFilter(preserve_classes=["author"])
+        result = f.filter_content(GITHUB_COMMENT_HTML)
+        combined = " ".join(result)
+        assert "site-nav" not in combined
+        assert "Copyright 2026" not in combined
+
+
+# ── preserve_classes ─────────────────────────────────────────────────────
+
+class TestPreserveClasses:
+
+    def test_preserve_author_class(self):
+        """Elements with 'author' class should be kept when whitelisted."""
+        f = PruningContentFilter(preserve_classes=["author"])
+        result = f.filter_content(GITHUB_COMMENT_HTML)
+        combined = " ".join(result)
+        assert "alice" in combined
+        assert "bob" in combined
+
+    def test_without_preserve_author_may_be_stripped(self):
+        """Without whitelist, short author spans may be stripped."""
+        f = PruningContentFilter()
+        result = f.filter_content(ATTRIBUTION_HTML)
+        combined = " ".join(result)
+        # The main content should be there
+        assert "article content" in combined.lower()
+        # byline might be stripped (short, low density)
+
+    def test_preserve_byline(self):
+        """Preserving 'byline' class keeps attribution."""
+        f = PruningContentFilter(preserve_classes=["byline"])
+        result = f.filter_content(ATTRIBUTION_HTML)
+        combined = " ".join(result)
+        assert "Jane Smith" in combined
+
+    def test_preserve_multiple_classes(self):
+        """Multiple classes can be preserved."""
+        f = PruningContentFilter(
+            preserve_classes=["author", "comment-header", "byline"]
+        )
+        result = f.filter_content(GITHUB_COMMENT_HTML)
+        combined = " ".join(result)
+        assert "alice" in combined
+        assert "bob" in combined
+
+    def test_preserve_class_not_in_html(self):
+        """Preserving a class that doesn't exist in the HTML is harmless."""
+        f = PruningContentFilter(preserve_classes=["nonexistent-class"])
+        result = f.filter_content(GITHUB_COMMENT_HTML)
+        # Should work normally, no crash
+        assert len(result) > 0
+
+    def test_empty_preserve_classes(self):
+        """Empty list should behave like no whitelist."""
+        f = PruningContentFilter(preserve_classes=[])
+        assert f.preserve_classes == set()
+
+
+# ── preserve_tags ────────────────────────────────────────────────────────
+
+class TestPreserveTags:
+
+    def test_preserve_cite_tag(self):
+        """Preserving 'cite' tag keeps source attribution."""
+        f = PruningContentFilter(preserve_tags=["cite"])
+        result = f.filter_content(SIMPLE_HTML)
+        combined = " ".join(result)
+        assert "Example Research Paper" in combined
+
+    def test_preserve_time_tag(self):
+        """Preserving 'time' tag keeps timestamps."""
+        f = PruningContentFilter(preserve_tags=["time"])
+        result = f.filter_content(GITHUB_COMMENT_HTML)
+        combined = " ".join(result)
+        assert "Apr 6, 2026" in combined
+
+    def test_preserve_multiple_tags(self):
+        """Multiple tags can be preserved."""
+        f = PruningContentFilter(preserve_tags=["cite", "time"])
+        result = f.filter_content(SIMPLE_HTML)
+        combined = " ".join(result)
+        assert "Example Research Paper" in combined
+
+    def test_empty_preserve_tags(self):
+        """Empty list should behave like no whitelist."""
+        f = PruningContentFilter(preserve_tags=[])
+        assert f.preserve_tags == set()
+
+
+# ── Combined ─────────────────────────────────────────────────────────────
+
+class TestCombined:
+
+    def test_both_classes_and_tags(self):
+        """Both preserve_classes and preserve_tags work together."""
+        f = PruningContentFilter(
+            preserve_classes=["author"],
+            preserve_tags=["time"],
+        )
+        result = f.filter_content(GITHUB_COMMENT_HTML)
+        combined = " ".join(result)
+        assert "alice" in combined
+        assert "Apr 6, 2026" in combined
+
+    def test_whitelist_does_not_override_excluded_tags(self):
+        """Nav/footer/header are removed before pruning — whitelist can't save them."""
+        f = PruningContentFilter(preserve_tags=["nav"])
+        result = f.filter_content(GITHUB_COMMENT_HTML)
+        combined = " ".join(result)
+        # nav is in excluded_tags and removed before pruning runs
+        # preserve_tags only affects the pruning phase
+        # This is expected — excluded_tags are structural boilerplate
+
+
+# ── _is_preserved method ─────────────────────────────────────────────────
+
+class TestIsPreserved:
+
+    def test_is_preserved_by_class(self):
+        from bs4 import BeautifulSoup
+        f = PruningContentFilter(preserve_classes=["author"])
+        soup = BeautifulSoup('Alice', "html.parser")
+        node = soup.find("span")
+        assert f._is_preserved(node) is True
+
+    def test_not_preserved_without_match(self):
+        from bs4 import BeautifulSoup
+        f = PruningContentFilter(preserve_classes=["author"])
+        soup = BeautifulSoup('2026', "html.parser")
+        node = soup.find("span")
+        assert f._is_preserved(node) is False
+
+    def test_is_preserved_by_tag(self):
+        from bs4 import BeautifulSoup
+        f = PruningContentFilter(preserve_tags=["cite"])
+        soup = BeautifulSoup('Source', "html.parser")
+        node = soup.find("cite")
+        assert f._is_preserved(node) is True
+
+    def test_not_preserved_empty_whitelist(self):
+        from bs4 import BeautifulSoup
+        f = PruningContentFilter()
+        soup = BeautifulSoup('Alice', "html.parser")
+        node = soup.find("span")
+        assert f._is_preserved(node) is False
+
+
+# ── Serialization (for Docker API) ───────────────────────────────────────
+
+class TestSerialization:
+
+    def test_params_stored(self):
+        """preserve_classes and preserve_tags should be stored as attributes."""
+        f = PruningContentFilter(
+            preserve_classes=["author", "byline"],
+            preserve_tags=["time", "cite"],
+        )
+        assert f.preserve_classes == {"author", "byline"}
+        assert f.preserve_tags == {"time", "cite"}
diff --git a/tests/test_pyopenssl_security_fix.py b/tests/test_pyopenssl_security_fix.py
new file mode 100644
index 0000000..493dbd9
--- /dev/null
+++ b/tests/test_pyopenssl_security_fix.py
@@ -0,0 +1,168 @@
+"""
+Lightweight test to verify pyOpenSSL security fix (Issue #1545).
+
+This test verifies the security requirements are met:
+1. pyOpenSSL >= 25.3.0 is installed
+2. cryptography >= 45.0.7 is installed (above vulnerable range)
+3. SSL/TLS functionality works correctly
+
+This test can run without full crawl4ai dependencies installed.
+"""
+
+import sys
+from packaging import version
+
+
+def test_package_versions():
+    """Test that package versions meet security requirements."""
+    print("=" * 70)
+    print("TEST: Package Version Security Requirements (Issue #1545)")
+    print("=" * 70)
+
+    all_passed = True
+
+    # Test pyOpenSSL version
+    try:
+        import OpenSSL
+        pyopenssl_version = OpenSSL.__version__
+        print(f"\n✓ pyOpenSSL is installed: {pyopenssl_version}")
+
+        if version.parse(pyopenssl_version) >= version.parse("25.3.0"):
+            print(f"  ✓ PASS: pyOpenSSL {pyopenssl_version} >= 25.3.0 (required)")
+        else:
+            print(f"  ✗ FAIL: pyOpenSSL {pyopenssl_version} < 25.3.0 (required)")
+            all_passed = False
+
+    except ImportError as e:
+        print(f"\n✗ FAIL: pyOpenSSL not installed - {e}")
+        all_passed = False
+
+    # Test cryptography version
+    try:
+        import cryptography
+        crypto_version = cryptography.__version__
+        print(f"\n✓ cryptography is installed: {crypto_version}")
+
+        # The vulnerable range is >=37.0.0 & <43.0.1
+        # We need >= 45.0.7 to be safe
+        if version.parse(crypto_version) >= version.parse("45.0.7"):
+            print(f"  ✓ PASS: cryptography {crypto_version} >= 45.0.7 (secure)")
+            print(f"  ✓ NOT in vulnerable range (37.0.0 to 43.0.0)")
+        elif version.parse(crypto_version) >= version.parse("37.0.0") and version.parse(crypto_version) < version.parse("43.0.1"):
+            print(f"  ✗ FAIL: cryptography {crypto_version} is VULNERABLE")
+            print(f"  ✗ Version is in vulnerable range (>=37.0.0 & <43.0.1)")
+            all_passed = False
+        else:
+            print(f"  ⚠ WARNING: cryptography {crypto_version} < 45.0.7")
+            print(f"  ⚠ May not meet security requirements")
+
+    except ImportError as e:
+        print(f"\n✗ FAIL: cryptography not installed - {e}")
+        all_passed = False
+
+    return all_passed
+
+
+def test_ssl_basic_functionality():
+    """Test that SSL/TLS basic functionality works."""
+    print("\n" + "=" * 70)
+    print("TEST: SSL/TLS Basic Functionality")
+    print("=" * 70)
+
+    try:
+        import OpenSSL.SSL
+
+        # Create a basic SSL context to verify functionality
+        context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_2_METHOD)
+        print("\n✓ SSL Context created successfully")
+        print("  ✓ PASS: SSL/TLS functionality is working")
+        return True
+
+    except Exception as e:
+        print(f"\n✗ FAIL: SSL functionality test failed - {e}")
+        return False
+
+
+def test_pyopenssl_crypto_integration():
+    """Test that pyOpenSSL and cryptography integration works."""
+    print("\n" + "=" * 70)
+    print("TEST: pyOpenSSL <-> cryptography Integration")
+    print("=" * 70)
+
+    try:
+        from OpenSSL import crypto
+
+        # Generate a simple key pair to test integration
+        key = crypto.PKey()
+        key.generate_key(crypto.TYPE_RSA, 2048)
+
+        print("\n✓ Generated RSA key pair successfully")
+        print("  ✓ PASS: pyOpenSSL and cryptography are properly integrated")
+        return True
+
+    except Exception as e:
+        print(f"\n✗ FAIL: Integration test failed - {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+def main():
+    """Run all security tests."""
+    print("\n")
+    print("╔" + "=" * 68 + "╗")
+    print("║  pyOpenSSL Security Fix Verification - Issue #1545               ║")
+    print("╚" + "=" * 68 + "╝")
+    print("\nVerifying that the pyOpenSSL update resolves the security vulnerability")
+    print("in the cryptography package (CVE: versions >=37.0.0 & <43.0.1)\n")
+
+    results = []
+
+    # Test 1: Package versions
+    results.append(("Package Versions", test_package_versions()))
+
+    # Test 2: SSL functionality
+    results.append(("SSL Functionality", test_ssl_basic_functionality()))
+
+    # Test 3: Integration
+    results.append(("pyOpenSSL-crypto Integration", test_pyopenssl_crypto_integration()))
+
+    # Summary
+    print("\n" + "=" * 70)
+    print("TEST SUMMARY")
+    print("=" * 70)
+
+    all_passed = True
+    for test_name, passed in results:
+        status = "✓ PASS" if passed else "✗ FAIL"
+        print(f"{status}: {test_name}")
+        all_passed = all_passed and passed
+
+    print("=" * 70)
+
+    if all_passed:
+        print("\n✓✓✓ ALL TESTS PASSED ✓✓✓")
+        print("✓ Security vulnerability is resolved")
+        print("✓ pyOpenSSL >= 25.3.0 is working correctly")
+        print("✓ cryptography >= 45.0.7 (not vulnerable)")
+        print("\nThe dependency update is safe to merge.\n")
+        return True
+    else:
+        print("\n✗✗✗ SOME TESTS FAILED ✗✗✗")
+        print("✗ Security requirements not met")
+        print("\nDo NOT merge until all tests pass.\n")
+        return False
+
+
+if __name__ == "__main__":
+    try:
+        success = main()
+        sys.exit(0 if success else 1)
+    except KeyboardInterrupt:
+        print("\n\nTest interrupted by user")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n✗ Unexpected error: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
diff --git a/tests/test_pyopenssl_update.py b/tests/test_pyopenssl_update.py
new file mode 100644
index 0000000..fa37bee
--- /dev/null
+++ b/tests/test_pyopenssl_update.py
@@ -0,0 +1,184 @@
+"""
+Test script to verify pyOpenSSL update doesn't break crawl4ai functionality.
+
+This test verifies:
+1. pyOpenSSL and cryptography versions are correct and secure
+2. Basic crawling functionality still works
+3. HTTPS/SSL connections work properly
+4. Stealth mode integration works (uses playwright-stealth internally)
+
+Issue: #1545 - Security vulnerability in cryptography package
+Fix: Updated pyOpenSSL from >=24.3.0 to >=25.3.0
+Expected: cryptography package should be >=45.0.7 (above vulnerable range)
+"""
+
+import asyncio
+import sys
+from packaging import version
+
+
+def check_versions():
+    """Verify pyOpenSSL and cryptography versions meet security requirements."""
+    print("=" * 60)
+    print("STEP 1: Checking Package Versions")
+    print("=" * 60)
+
+    try:
+        import OpenSSL
+        pyopenssl_version = OpenSSL.__version__
+        print(f"✓ pyOpenSSL version: {pyopenssl_version}")
+
+        # Check pyOpenSSL >= 25.3.0
+        if version.parse(pyopenssl_version) >= version.parse("25.3.0"):
+            print(f"  ✓ Version check passed: {pyopenssl_version} >= 25.3.0")
+        else:
+            print(f"  ✗ Version check FAILED: {pyopenssl_version} < 25.3.0")
+            return False
+
+    except ImportError as e:
+        print(f"✗ Failed to import pyOpenSSL: {e}")
+        return False
+
+    try:
+        import cryptography
+        crypto_version = cryptography.__version__
+        print(f"✓ cryptography version: {crypto_version}")
+
+        # Check cryptography >= 45.0.7 (above vulnerable range)
+        if version.parse(crypto_version) >= version.parse("45.0.7"):
+            print(f"  ✓ Security check passed: {crypto_version} >= 45.0.7 (not vulnerable)")
+        else:
+            print(f"  ✗ Security check FAILED: {crypto_version} < 45.0.7 (potentially vulnerable)")
+            return False
+
+    except ImportError as e:
+        print(f"✗ Failed to import cryptography: {e}")
+        return False
+
+    print("\n✓ All version checks passed!\n")
+    return True
+
+
+async def test_basic_crawl():
+    """Test basic crawling functionality with HTTPS site."""
+    print("=" * 60)
+    print("STEP 2: Testing Basic HTTPS Crawling")
+    print("=" * 60)
+
+    try:
+        from crawl4ai import AsyncWebCrawler
+
+        async with AsyncWebCrawler(verbose=True) as crawler:
+            # Test with a simple HTTPS site (requires SSL/TLS)
+            print("Crawling example.com (HTTPS)...")
+            result = await crawler.arun(
+                url="https://www.example.com",
+                bypass_cache=True
+            )
+
+            if result.success:
+                print(f"✓ Crawl successful!")
+                print(f"  - Status code: {result.status_code}")
+                print(f"  - Content length: {len(result.html)} bytes")
+                print(f"  - SSL/TLS connection: ✓ Working")
+                return True
+            else:
+                print(f"✗ Crawl failed: {result.error_message}")
+                return False
+
+    except Exception as e:
+        print(f"✗ Test failed with error: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+async def test_stealth_mode():
+    """Test stealth mode functionality (depends on playwright-stealth)."""
+    print("\n" + "=" * 60)
+    print("STEP 3: Testing Stealth Mode Integration")
+    print("=" * 60)
+
+    try:
+        from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+        # Create browser config with stealth mode
+        browser_config = BrowserConfig(
+            headless=True,
+            verbose=False
+        )
+
+        async with AsyncWebCrawler(config=browser_config, verbose=True) as crawler:
+            print("Crawling with stealth mode enabled...")
+            result = await crawler.arun(
+                url="https://www.example.com",
+                bypass_cache=True
+            )
+
+            if result.success:
+                print(f"✓ Stealth crawl successful!")
+                print(f"  - Stealth mode: ✓ Working")
+                return True
+            else:
+                print(f"✗ Stealth crawl failed: {result.error_message}")
+                return False
+
+    except Exception as e:
+        print(f"✗ Stealth test failed with error: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+async def main():
+    """Run all tests."""
+    print("\n")
+    print("╔" + "=" * 58 + "╗")
+    print("║  pyOpenSSL Security Update Verification Test (Issue #1545) ║")
+    print("╚" + "=" * 58 + "╝")
+    print("\n")
+
+    # Step 1: Check versions
+    versions_ok = check_versions()
+    if not versions_ok:
+        print("\n✗ FAILED: Version requirements not met")
+        return False
+
+    # Step 2: Test basic crawling
+    crawl_ok = await test_basic_crawl()
+    if not crawl_ok:
+        print("\n✗ FAILED: Basic crawling test failed")
+        return False
+
+    # Step 3: Test stealth mode
+    stealth_ok = await test_stealth_mode()
+    if not stealth_ok:
+        print("\n✗ FAILED: Stealth mode test failed")
+        return False
+
+    # All tests passed
+    print("\n" + "=" * 60)
+    print("FINAL RESULT")
+    print("=" * 60)
+    print("✓ All tests passed successfully!")
+    print("✓ pyOpenSSL update is working correctly")
+    print("✓ No breaking changes detected")
+    print("✓ Security vulnerability resolved")
+    print("=" * 60)
+    print("\n")
+
+    return True
+
+
+if __name__ == "__main__":
+    try:
+        success = asyncio.run(main())
+        sys.exit(0 if success else 1)
+    except KeyboardInterrupt:
+        print("\n\nTest interrupted by user")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n✗ Unexpected error: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
diff --git a/tests/test_raw_html_browser.py b/tests/test_raw_html_browser.py
new file mode 100644
index 0000000..0f8648a
--- /dev/null
+++ b/tests/test_raw_html_browser.py
@@ -0,0 +1,172 @@
+"""
+Tests for raw:/file:// URL browser pipeline support.
+
+Tests the new feature that allows js_code, wait_for, and other browser operations
+to work with raw: and file:// URLs by routing them through _crawl_web() with
+set_content() instead of goto().
+"""
+
+import pytest
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+
+@pytest.mark.asyncio
+async def test_raw_html_fast_path():
+    """Test that raw: without browser params returns HTML directly (fast path)."""
+    html = "
      Original Content
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig()  # No browser params
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Original Content" in result.html
+    # Fast path should not modify the HTML
+    assert result.html == html
+
+
+@pytest.mark.asyncio
+async def test_js_code_on_raw_html():
+    """Test that js_code executes on raw: HTML and modifies the DOM."""
+    html = "
      Original
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code="document.getElementById('test').innerText = 'Modified by JS'"
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Modified by JS" in result.html
+    assert "Original" not in result.html or "Modified by JS" in result.html
+
+
+@pytest.mark.asyncio
+async def test_js_code_adds_element_to_raw_html():
+    """Test that js_code can add new elements to raw: HTML."""
+    html = "
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code='document.getElementById("container").innerHTML = "Custom Content"'
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "injected" in result.html
+    assert "Custom Content" in result.html
+
+
+@pytest.mark.asyncio
+async def test_screenshot_on_raw_html():
+    """Test that screenshots work on raw: HTML."""
+    html = "
      Screenshot Test
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(screenshot=True)
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert result.screenshot is not None
+    assert len(result.screenshot) > 100  # Should have substantial screenshot data
+
+
+@pytest.mark.asyncio
+async def test_process_in_browser_flag():
+    """Test that process_in_browser=True forces browser path even without other params."""
+    html = "
      Test
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(process_in_browser=True)
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    # Browser path normalizes HTML, so it may be slightly different
+    assert "Test" in result.html
+
+
+@pytest.mark.asyncio
+async def test_raw_prefix_variations():
+    """Test both raw: and raw:// prefix formats."""
+    html = "Content"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code='document.body.innerHTML += "
      Added
      "'
+        )
+
+        # Test raw: prefix
+        result1 = await crawler.arun(f"raw:{html}", config=config)
+        assert result1.success
+        assert "Added" in result1.html
+
+        # Test raw:// prefix
+        result2 = await crawler.arun(f"raw://{html}", config=config)
+        assert result2.success
+        assert "Added" in result2.html
+
+
+@pytest.mark.asyncio
+async def test_wait_for_on_raw_html():
+    """Test that wait_for works with raw: HTML after js_code modifies DOM."""
+    html = "
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code='''
+                setTimeout(() => {
+                    document.getElementById('container').innerHTML = '
      Delayed Content
      ';
+                }, 100);
+            ''',
+            wait_for="#delayed",
+            wait_for_timeout=5000
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Delayed Content" in result.html
+
+
+@pytest.mark.asyncio
+async def test_multiple_js_code_scripts():
+    """Test that multiple js_code scripts execute in order."""
+    html = "
      0
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code=[
+                "document.getElementById('counter').innerText = '1'",
+                "document.getElementById('counter').innerText = parseInt(document.getElementById('counter').innerText) + 1",
+                "document.getElementById('counter').innerText = parseInt(document.getElementById('counter').innerText) + 1",
+            ]
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert ">3<" in result.html  # Counter should be 3 after all scripts run
+
+
+if __name__ == "__main__":
+    # Run a quick manual test
+    async def quick_test():
+        html = "
      Original
      "
+
+        async with AsyncWebCrawler(verbose=True) as crawler:
+            # Test 1: Fast path
+            print("\n=== Test 1: Fast path (no browser params) ===")
+            result1 = await crawler.arun(f"raw:{html}")
+            print(f"Success: {result1.success}")
+            print(f"HTML contains 'Original': {'Original' in result1.html}")
+
+            # Test 2: js_code modifies DOM
+            print("\n=== Test 2: js_code modifies DOM ===")
+            config = CrawlerRunConfig(
+                js_code="document.getElementById('test').innerText = 'Modified by JS'"
+            )
+            result2 = await crawler.arun(f"raw:{html}", config=config)
+            print(f"Success: {result2.success}")
+            print(f"HTML contains 'Modified by JS': {'Modified by JS' in result2.html}")
+            print(f"HTML snippet: {result2.html[:500]}...")
+
+    asyncio.run(quick_test())
diff --git a/tests/test_raw_html_edge_cases.py b/tests/test_raw_html_edge_cases.py
new file mode 100644
index 0000000..8517d19
--- /dev/null
+++ b/tests/test_raw_html_edge_cases.py
@@ -0,0 +1,563 @@
+"""
+BRUTAL edge case tests for raw:/file:// URL browser pipeline.
+
+These tests try to break the system with tricky inputs, edge cases,
+and compatibility checks to ensure we didn't break existing functionality.
+"""
+
+import pytest
+import asyncio
+import tempfile
+import os
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+
+# ============================================================================
+# EDGE CASE: Hash characters in HTML (previously broke urlparse - Issue #283)
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_with_hash_in_css():
+    """Test that # in CSS colors doesn't break HTML parsing (regression for #283)."""
+    html = """
+    
+    
+        
+    
+    
+        
      Content with hash colors
      
+    
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(js_code="document.body.innerHTML += '
      Added
      '")
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "#ff5733" in result.html or "ff5733" in result.html  # Color should be preserved
+    assert "Added" in result.html  # JS executed
+    assert "Content with hash colors" in result.html  # Original content preserved
+
+
+@pytest.mark.asyncio
+async def test_raw_html_with_fragment_links():
+    """Test HTML with # fragment links doesn't break."""
+    html = """
+    
+        Go to section 1
+        Go to section 2
+        
      Section 1
      
+        
      Section 2
      
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(js_code="document.getElementById('section1').innerText = 'Modified Section 1'")
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Modified Section 1" in result.html
+    assert "#section2" in result.html  # Fragment link preserved
+
+
+# ============================================================================
+# EDGE CASE: Special characters and unicode
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_with_unicode():
+    """Test raw HTML with various unicode characters."""
+    html = """
+    
+        
      日本語 中文 한국어 العربية 🎉 💻 🚀
      
+        
      & < > " '
      
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(js_code="document.getElementById('unicode').innerText += ' ✅ Modified'")
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "✅ Modified" in result.html or "Modified" in result.html
+    # Check unicode is preserved
+    assert "日本語" in result.html or "&#" in result.html  # Either preserved or encoded
+
+
+@pytest.mark.asyncio
+async def test_raw_html_with_script_tags():
+    """Test raw HTML with existing script tags doesn't interfere with js_code."""
+    html = """
+    
+        
      0
      
+        
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        # Our js_code runs AFTER the page scripts
+        config = CrawlerRunConfig(
+            js_code="document.getElementById('counter').innerText = parseInt(document.getElementById('counter').innerText) + 5"
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    # The embedded script sets it to 10, then our js_code adds 5
+    assert ">15<" in result.html or "15" in result.html
+
+
+# ============================================================================
+# EDGE CASE: Empty and malformed HTML
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_empty():
+    """Test empty raw HTML."""
+    html = ""
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(js_code="document.body.innerHTML = '
      Added to empty
      '")
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Added to empty" in result.html
+
+
+@pytest.mark.asyncio
+async def test_raw_html_minimal():
+    """Test minimal HTML (just text, no tags)."""
+    html = "Just plain text, no HTML tags"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(js_code="document.body.innerHTML += '
      Injected
      '")
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    # Browser should wrap it in proper HTML
+    assert "Injected" in result.html
+
+
+@pytest.mark.asyncio
+async def test_raw_html_malformed():
+    """Test malformed HTML with unclosed tags."""
+    html = "
      Unclosed tags
      More content"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(js_code="document.body.innerHTML += '
      Valid Added
      '")
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Valid Added" in result.html
+    # Browser should have fixed the malformed HTML
+
+
+# ============================================================================
+# EDGE CASE: Very large HTML
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_large():
+    """Test large raw HTML (100KB+)."""
+    # Generate 100KB of HTML
+    items = "".join([f'
      Item {i} content here with some text
      \n' for i in range(2000)])
+    html = f"{items}"
+
+    assert len(html) > 100000  # Verify it's actually large
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code="document.getElementById('item-999').innerText = 'MODIFIED ITEM 999'"
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "MODIFIED ITEM 999" in result.html
+    assert "item-1999" in result.html  # Last item should still exist
+
+
+# ============================================================================
+# EDGE CASE: JavaScript errors and timeouts
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_js_error_doesnt_crash():
+    """Test that JavaScript errors in js_code don't crash the crawl."""
+    html = "
      Original
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code=[
+                "nonExistentFunction();",  # This will throw an error
+                "document.getElementById('test').innerText = 'Still works'"  # This should still run
+            ]
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    # Crawl should succeed even with JS errors
+    assert result.success
+
+
+@pytest.mark.asyncio
+async def test_raw_html_wait_for_timeout():
+    """Test wait_for with element that never appears times out gracefully."""
+    html = "
      Original
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            wait_for="#never-exists",
+            wait_for_timeout=1000  # 1 second timeout
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    # Should timeout but still return the HTML we have
+    # The behavior might be success=False or success=True with partial content
+    # Either way, it shouldn't hang or crash
+    assert result is not None
+
+
+# ============================================================================
+# COMPATIBILITY: Normal HTTP URLs still work
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_http_urls_still_work():
+    """Ensure we didn't break normal HTTP URL crawling."""
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com")
+
+    assert result.success
+    assert "Example Domain" in result.html
+
+
+@pytest.mark.asyncio
+async def test_http_with_js_code_still_works():
+    """Ensure HTTP URLs with js_code still work."""
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code="document.body.innerHTML += '
      Injected via JS
      '"
+        )
+        result = await crawler.arun("https://example.com", config=config)
+
+    assert result.success
+    assert "Injected via JS" in result.html
+
+
+# ============================================================================
+# COMPATIBILITY: File URLs
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_file_url_with_js_code():
+    """Test file:// URLs with js_code execution."""
+    # Create a temp file
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.html', delete=False) as f:
+        f.write("
      File Content
      ")
+        temp_path = f.name
+
+    try:
+        async with AsyncWebCrawler() as crawler:
+            config = CrawlerRunConfig(
+                js_code="document.getElementById('file-content').innerText = 'Modified File Content'"
+            )
+            result = await crawler.arun(f"file://{temp_path}", config=config)
+
+        assert result.success
+        assert "Modified File Content" in result.html
+    finally:
+        os.unlink(temp_path)
+
+
+@pytest.mark.asyncio
+async def test_file_url_fast_path():
+    """Test file:// fast path (no browser params)."""
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.html', delete=False) as f:
+        f.write("Fast path file content")
+        temp_path = f.name
+
+    try:
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(f"file://{temp_path}")
+
+        assert result.success
+        assert "Fast path file content" in result.html
+    finally:
+        os.unlink(temp_path)
+
+
+# ============================================================================
+# COMPATIBILITY: Extraction strategies with raw HTML
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_with_css_extraction():
+    """Test CSS extraction on raw HTML after js_code modifies it."""
+    from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+    html = """
+    
+        
      
+            
      Original Product
      
+        
      
+    
+    """
+
+    schema = {
+        "name": "Products",
+        "baseSelector": ".product",
+        "fields": [
+            {"name": "name", "selector": ".name", "type": "text"}
+        ]
+    }
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code="""
+                document.querySelector('.products').innerHTML +=
+                '
      JS Added Product
      ';
+            """,
+            extraction_strategy=JsonCssExtractionStrategy(schema)
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    # Check that extraction found both products
+    import json
+    extracted = json.loads(result.extracted_content)
+    names = [p.get('name', '') for p in extracted]
+    assert any("JS Added Product" in name for name in names)
+
+
+# ============================================================================
+# EDGE CASE: Concurrent raw: requests
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_concurrent_raw_requests():
+    """Test multiple concurrent raw: requests don't interfere."""
+    htmls = [
+        f"
      Request {i}
      "
+        for i in range(5)
+    ]
+
+    async with AsyncWebCrawler() as crawler:
+        configs = [
+            CrawlerRunConfig(
+                js_code=f"document.getElementById('test').innerText += ' Modified {i}'"
+            )
+            for i in range(5)
+        ]
+
+        # Run concurrently
+        tasks = [
+            crawler.arun(f"raw:{html}", config=config)
+            for html, config in zip(htmls, configs)
+        ]
+        results = await asyncio.gather(*tasks)
+
+    for i, result in enumerate(results):
+        assert result.success
+        assert f"Request {i}" in result.html
+        assert f"Modified {i}" in result.html
+
+
+# ============================================================================
+# EDGE CASE: raw: with base_url for link resolution
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_with_base_url():
+    """Test that base_url is used for link resolution in markdown."""
+    html = """
+    
+        Page 1
+        Page 2
+        Logo
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            base_url="https://example.com",
+            process_in_browser=True  # Force browser to test base_url handling
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    # Check markdown has absolute URLs
+    if result.markdown:
+        # Links should be absolute
+        md = result.markdown.raw_markdown if hasattr(result.markdown, 'raw_markdown') else str(result.markdown)
+        assert "example.com" in md or "/page1" in md
+
+
+# ============================================================================
+# EDGE CASE: raw: with screenshot of complex page
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_screenshot_complex_page():
+    """Test screenshot of complex raw HTML with CSS and JS modifications."""
+    html = """
+    
+    
+        
+    
+    
+        
      
+            
      Original Title
      
+            
      This is a test card with styling.
      
+        
      
+    
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code="document.getElementById('title').innerText = 'Modified Title'",
+            screenshot=True
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert result.screenshot is not None
+    assert len(result.screenshot) > 1000  # Should be substantial
+    assert "Modified Title" in result.html
+
+
+# ============================================================================
+# EDGE CASE: JavaScript that tries to navigate away
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_js_navigation_blocked():
+    """Test that JS trying to navigate doesn't break the crawl."""
+    html = """
+    
+        
      Original Content
      
+        
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            # Try to navigate via js_code
+            js_code=[
+                "document.getElementById('content').innerText = 'Before navigation attempt'",
+                # Actual navigation attempt commented - would cause issues
+                # "window.location.href = 'https://example.com'",
+            ]
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Before navigation attempt" in result.html
+
+
+# ============================================================================
+# EDGE CASE: Raw HTML with iframes
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_with_iframes():
+    """Test raw HTML containing iframes."""
+    html = """
+    
+        
      Main content
      
+        
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code="document.getElementById('main').innerText = 'Modified main'",
+            process_iframes=True
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Modified main" in result.html
+
+
+# ============================================================================
+# TRICKY: Protocol inside raw content
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_with_urls_inside():
+    """Test raw: with http:// URLs inside the content."""
+    html = """
+    
+        Example
+        Google
+        Cat
+        
      Test content with URL: https://test.com
      
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code="document.getElementById('test').innerText += ' - Modified'"
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Modified" in result.html
+    assert "http://example.com" in result.html or "example.com" in result.html
+
+
+# ============================================================================
+# TRICKY: Double raw: prefix
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_double_raw_prefix():
+    """Test what happens with double raw: prefix (edge case)."""
+    html = "Content"
+
+    async with AsyncWebCrawler() as crawler:
+        # raw:raw:... - the second raw: becomes part of content
+        result = await crawler.arun(f"raw:raw:{html}")
+
+    # Should either handle gracefully or return "raw:..." as content
+    assert result is not None
+
+
+if __name__ == "__main__":
+    import sys
+
+    async def run_tests():
+        # Run a few key tests manually
+        tests = [
+            ("Hash in CSS", test_raw_html_with_hash_in_css),
+            ("Unicode", test_raw_html_with_unicode),
+            ("Large HTML", test_raw_html_large),
+            ("HTTP still works", test_http_urls_still_work),
+            ("Concurrent requests", test_concurrent_raw_requests),
+            ("Complex screenshot", test_raw_html_screenshot_complex_page),
+        ]
+
+        for name, test_fn in tests:
+            print(f"\n=== Running: {name} ===")
+            try:
+                await test_fn()
+                print(f"✅ {name} PASSED")
+            except Exception as e:
+                print(f"❌ {name} FAILED: {e}")
+                import traceback
+                traceback.print_exc()
+
+    asyncio.run(run_tests())
diff --git a/tests/test_raw_html_redirected_url.py b/tests/test_raw_html_redirected_url.py
new file mode 100644
index 0000000..c7dd062
--- /dev/null
+++ b/tests/test_raw_html_redirected_url.py
@@ -0,0 +1,299 @@
+"""
+Tests for redirected_url handling with raw: URLs.
+
+This test file verifies the fix for the issue where redirected_url was incorrectly
+set to the entire raw HTML content (potentially 300KB+) instead of None or base_url.
+
+Issue: In raw: mode, async_crawler_strategy.py was setting redirected_url = config.base_url or url,
+which fell back to the raw HTML string when base_url wasn't provided.
+"""
+
+try:
+    import pytest
+    HAS_PYTEST = True
+except ImportError:
+    HAS_PYTEST = False
+    # Create a dummy decorator
+    class pytest:
+        class mark:
+            @staticmethod
+            def asyncio(fn):
+                return fn
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+
+# ============================================================================
+# Core fix tests: redirected_url should NOT be the raw HTML string
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_redirected_url_is_none_without_base_url():
+    """Test that redirected_url is None for raw: URLs when no base_url is provided."""
+    html = "
      Test Content
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig()
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    # Key assertion: redirected_url should be None, NOT the raw HTML string
+    assert result.redirected_url is None, (
+        f"redirected_url should be None for raw: URLs without base_url, "
+        f"but got: {result.redirected_url[:100] if result.redirected_url else None}..."
+    )
+
+
+@pytest.mark.asyncio
+async def test_raw_html_redirected_url_not_huge():
+    """Test that redirected_url is not a huge string (the raw HTML content)."""
+    # Create a large HTML (100KB+)
+    items = "".join([f'
      Item {i} with some content
      \n' for i in range(2000)])
+    large_html = f"{items}"
+    assert len(large_html) > 100000, "Test HTML should be >100KB"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig()
+        result = await crawler.arun(f"raw:{large_html}", config=config)
+
+    assert result.success
+    # Key assertion: redirected_url should NOT be the huge HTML string
+    if result.redirected_url is not None:
+        assert len(result.redirected_url) < 1000, (
+            f"redirected_url should not be the raw HTML! "
+            f"Got {len(result.redirected_url)} chars: {result.redirected_url[:100]}..."
+        )
+
+
+@pytest.mark.asyncio
+async def test_raw_html_with_base_url_sets_redirected_url():
+    """Test that redirected_url is set to base_url when provided."""
+    html = "
      Test Content
      "
+    base_url = "https://example.com/page"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(base_url=base_url)
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    # Key assertion: redirected_url should be the base_url
+    assert result.redirected_url == base_url, (
+        f"redirected_url should be '{base_url}', got: {result.redirected_url}"
+    )
+
+
+@pytest.mark.asyncio
+async def test_raw_double_slash_prefix_redirected_url():
+    """Test redirected_url handling with raw:// prefix."""
+    html = "Content"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig()
+        result = await crawler.arun(f"raw://{html}", config=config)
+
+    assert result.success
+    # Should be None, not the HTML
+    assert result.redirected_url is None
+
+
+# ============================================================================
+# Browser path tests (with js_code, screenshot, etc.)
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_browser_path_redirected_url_none():
+    """Test redirected_url is None for raw: URLs in browser path (with js_code)."""
+    html = "
      Original
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            js_code="document.getElementById('test').innerText = 'Modified'"
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Modified" in result.html
+    # Key assertion: even with browser path, redirected_url should be None
+    assert result.redirected_url is None, (
+        f"redirected_url should be None, got: {result.redirected_url}"
+    )
+
+
+@pytest.mark.asyncio
+async def test_raw_html_browser_path_with_base_url():
+    """Test redirected_url is base_url for raw: URLs in browser path."""
+    html = "
      Original
      "
+    base_url = "https://mysite.com/processed"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(
+            base_url=base_url,
+            js_code="document.getElementById('test').innerText = 'Modified'"
+        )
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert "Modified" in result.html
+    assert result.redirected_url == base_url
+
+
+@pytest.mark.asyncio
+async def test_raw_html_screenshot_redirected_url():
+    """Test redirected_url with screenshot (browser path)."""
+    html = "
      Screenshot Test
      "
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(screenshot=True)
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert result.screenshot is not None
+    # redirected_url should still be None
+    assert result.redirected_url is None
+
+
+# ============================================================================
+# Compatibility tests: HTTP URLs should still work correctly
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_http_url_redirected_url_still_works():
+    """Ensure HTTP URLs still set redirected_url correctly."""
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com")
+
+    assert result.success
+    # For HTTP URLs, redirected_url should be the final URL (or original if no redirect)
+    assert result.redirected_url is not None
+    assert "example.com" in result.redirected_url
+
+
+@pytest.mark.asyncio
+async def test_http_url_with_redirect_preserves_redirected_url():
+    """Test that HTTP redirects still capture the final URL."""
+    # httpbin.org/redirect-to redirects to the specified URL
+    async with AsyncWebCrawler() as crawler:
+        # Use a URL that redirects
+        result = await crawler.arun("https://httpbin.org/redirect-to?url=https://example.com")
+
+    assert result.success
+    # Should capture the final redirected URL
+    assert result.redirected_url is not None
+    assert "example.com" in result.redirected_url
+
+
+# ============================================================================
+# Edge cases
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_raw_html_with_url_like_content():
+    """Test raw HTML containing URLs doesn't confuse redirected_url."""
+    html = """
+    
+        Link
+        
      Visit https://google.com for more
      
+        
      raw:https://fake.com
      
+    
+    """
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(f"raw:{html}")
+
+    assert result.success
+    # redirected_url should be None, not any URL from the content
+    assert result.redirected_url is None
+
+
+@pytest.mark.asyncio
+async def test_raw_html_empty_base_url():
+    """Test raw HTML with empty string base_url."""
+    html = "Content"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(base_url="")
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    # Empty string is falsy, so redirected_url should be None
+    assert result.redirected_url is None or result.redirected_url == ""
+
+
+@pytest.mark.asyncio
+async def test_raw_html_process_in_browser_redirected_url():
+    """Test redirected_url with process_in_browser=True."""
+    html = "Test"
+
+    async with AsyncWebCrawler() as crawler:
+        config = CrawlerRunConfig(process_in_browser=True)
+        result = await crawler.arun(f"raw:{html}", config=config)
+
+    assert result.success
+    assert result.redirected_url is None
+
+
+# ============================================================================
+# Regression test: specific issue scenario
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_regression_321kb_html_redirected_url():
+    """
+    Regression test for the specific issue:
+    - raw:{321KB HTML} should NOT have redirected_url = "raw:{321KB HTML}"
+    - This was causing massive memory/logging issues
+    """
+    # Create ~321KB of HTML content
+    content = "X" * 300000  # ~300KB of content
+    html = f"
      {content}
      "
+    assert len(html) > 300000, "Should be >300KB"
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(f"raw:{html}")
+
+    assert result.success
+
+    # The bug was: redirected_url = "raw:{321KB HTML}"
+    # After fix: redirected_url = None
+    assert result.redirected_url is None, (
+        "REGRESSION: redirected_url contains the raw HTML! "
+        f"Length: {len(result.redirected_url) if result.redirected_url else 0}"
+    )
+
+
+if __name__ == "__main__":
+    async def run_tests():
+        tests = [
+            ("redirected_url None without base_url", test_raw_html_redirected_url_is_none_without_base_url),
+            ("redirected_url not huge", test_raw_html_redirected_url_not_huge),
+            ("redirected_url with base_url", test_raw_html_with_base_url_sets_redirected_url),
+            ("raw:// prefix", test_raw_double_slash_prefix_redirected_url),
+            ("browser path None", test_raw_html_browser_path_redirected_url_none),
+            ("browser path with base_url", test_raw_html_browser_path_with_base_url),
+            ("HTTP URL still works", test_http_url_redirected_url_still_works),
+            ("321KB regression", test_regression_321kb_html_redirected_url),
+        ]
+
+        passed = 0
+        failed = 0
+
+        for name, test_fn in tests:
+            print(f"\n=== {name} ===")
+            try:
+                await test_fn()
+                print(f"PASSED")
+                passed += 1
+            except Exception as e:
+                print(f"FAILED: {e}")
+                import traceback
+                traceback.print_exc()
+                failed += 1
+
+        print(f"\n{'='*50}")
+        print(f"Results: {passed} passed, {failed} failed")
+        return failed == 0
+
+    import sys
+    success = asyncio.run(run_tests())
+    sys.exit(0 if success else 1)
diff --git a/tests/test_scraping_strategy.py b/tests/test_scraping_strategy.py
new file mode 100644
index 0000000..df46285
--- /dev/null
+++ b/tests/test_scraping_strategy.py
@@ -0,0 +1,26 @@
+import nest_asyncio
+
+nest_asyncio.apply()
+
+import asyncio
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    LXMLWebScrapingStrategy,
+    CacheMode,
+)
+
+
+async def main():
+    config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        scraping_strategy=LXMLWebScrapingStrategy(),  # Faster alternative to default BeautifulSoup
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url="https://example.com", config=config)
+        print(f"Success: {result.success}")
+        print(f"Markdown length: {len(result.markdown.raw_markdown)}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/tests/test_source_sibling_selector.py b/tests/test_source_sibling_selector.py
new file mode 100644
index 0000000..7c65313
--- /dev/null
+++ b/tests/test_source_sibling_selector.py
@@ -0,0 +1,396 @@
+"""Tests for the `source` (sibling selector) support in JSON extraction strategies."""
+
+import pytest
+from crawl4ai.extraction_strategy import (
+    JsonCssExtractionStrategy,
+    JsonXPathExtractionStrategy,
+)
+
+# ---------------------------------------------------------------------------
+# Shared HTML fixture — mimics Hacker News sibling-row layout
+# ---------------------------------------------------------------------------
+HN_HTML = """\
+
+  
+    
+    
+  
+  
+    
+    
+  
+  
+
+  
+    
+    
+  
+  
+    
+    
+  
+  
+
      1.Alpha
      
+      100 points
+      alice
+      2 hours ago
+    
      2.Beta
      
+      42 points
+      bob
+      5 hours ago
+    
      
+"""
+
+
+# ---------------------------------------------------------------------------
+# CSS Strategy Tests
+# ---------------------------------------------------------------------------
+class TestCssSourceField:
+    """JsonCssExtractionStrategy with source field."""
+
+    def _extract(self, schema):
+        strategy = JsonCssExtractionStrategy(schema)
+        return strategy.extract(None, HN_HTML)
+
+    def test_basic_source_extraction(self):
+        """Fields with source='+ tr' should extract data from the next sibling row."""
+        schema = {
+            "name": "HN",
+            "baseSelector": "tr.athing.submission",
+            "fields": [
+                {"name": "rank", "selector": "span.rank", "type": "text"},
+                {"name": "title", "selector": "span.titleline a", "type": "text"},
+                {"name": "url", "selector": "span.titleline a", "type": "attribute", "attribute": "href"},
+                {"name": "score", "selector": "span.score", "type": "text", "source": "+ tr"},
+                {"name": "author", "selector": "a.hnuser", "type": "text", "source": "+ tr"},
+            ],
+        }
+        results = self._extract(schema)
+        assert len(results) == 2
+
+        assert results[0]["rank"] == "1."
+        assert results[0]["title"] == "Alpha"
+        assert results[0]["url"] == "https://example.com/a"
+        assert results[0]["score"] == "100 points"
+        assert results[0]["author"] == "alice"
+
+        assert results[1]["rank"] == "2."
+        assert results[1]["title"] == "Beta"
+        assert results[1]["score"] == "42 points"
+        assert results[1]["author"] == "bob"
+
+    def test_backward_compat_no_source(self):
+        """Schema without source key should work exactly as before."""
+        schema = {
+            "name": "HN titles only",
+            "baseSelector": "tr.athing.submission",
+            "fields": [
+                {"name": "title", "selector": "span.titleline a", "type": "text"},
+            ],
+        }
+        results = self._extract(schema)
+        assert len(results) == 2
+        assert results[0]["title"] == "Alpha"
+        assert results[1]["title"] == "Beta"
+
+    def test_source_missing_sibling_returns_default(self):
+        """When source points to a non-existent sibling, field returns its default."""
+        schema = {
+            "name": "HN",
+            "baseSelector": "tr.athing.submission",
+            "fields": [
+                {"name": "title", "selector": "span.titleline a", "type": "text"},
+                {
+                    "name": "missing",
+                    "selector": "span.nope",
+                    "type": "text",
+                    "source": "+ div.nonexistent",
+                    "default": "N/A",
+                },
+            ],
+        }
+        results = self._extract(schema)
+        assert len(results) == 2
+        assert results[0]["missing"] == "N/A"
+
+    def test_source_with_class_filter(self):
+        """source='+ tr.spacer' should skip the subtext row and match the spacer."""
+        schema = {
+            "name": "HN spacer",
+            "baseSelector": "tr.athing.submission",
+            "fields": [
+                {"name": "title", "selector": "span.titleline a", "type": "text"},
+                # The spacer  has no content, so score should be empty/default
+                {
+                    "name": "score_from_spacer",
+                    "selector": "span.score",
+                    "type": "text",
+                    "source": "+ tr.spacer",
+                    "default": "none",
+                },
+            ],
+        }
+        results = self._extract(schema)
+        # The spacer has no span.score, so should fall back to default
+        # But note: "+ tr.spacer" should skip the immediate sibling (no class spacer)
+        # and find the spacer tr. Actually BS4 find_next_sibling finds the FIRST matching sibling.
+        # The immediate next sibling is  (no class), then .
+        # find_next_sibling("tr", class_="spacer") should skip the first and find the spacer.
+        assert results[0]["score_from_spacer"] == "none"
+
+    def test_source_on_attribute_field(self):
+        """source should work with attribute field type."""
+        schema = {
+            "name": "HN",
+            "baseSelector": "tr.athing.submission",
+            "fields": [
+                {
+                    "name": "author_href",
+                    "selector": "a.hnuser",
+                    "type": "attribute",
+                    "attribute": "href",
+                    "source": "+ tr",
+                    "default": "no-href",
+                },
+            ],
+        }
+        results = self._extract(schema)
+        assert len(results) == 2
+        # The  has no href in our test HTML, so attribute returns None -> default
+        assert results[0]["author_href"] == "no-href"
+
+
+# ---------------------------------------------------------------------------
+# XPath Strategy Tests
+# ---------------------------------------------------------------------------
+class TestXPathSourceField:
+    """JsonXPathExtractionStrategy with source field."""
+
+    def _extract(self, schema):
+        strategy = JsonXPathExtractionStrategy(schema)
+        return strategy.extract(None, HN_HTML)
+
+    def test_basic_source_extraction(self):
+        """Fields with source='+ tr' should extract data from the next sibling row."""
+        schema = {
+            "name": "HN",
+            "baseSelector": "//tr[contains(@class, 'athing') and contains(@class, 'submission')]",
+            "fields": [
+                {"name": "rank", "selector": ".//span[@class='rank']", "type": "text"},
+                {"name": "title", "selector": ".//span[@class='titleline']/a", "type": "text"},
+                {"name": "url", "selector": ".//span[@class='titleline']/a", "type": "attribute", "attribute": "href"},
+                {"name": "score", "selector": ".//span[@class='score']", "type": "text", "source": "+ tr"},
+                {"name": "author", "selector": ".//a[@class='hnuser']", "type": "text", "source": "+ tr"},
+            ],
+        }
+        results = self._extract(schema)
+        assert len(results) == 2
+
+        assert results[0]["rank"] == "1."
+        assert results[0]["title"] == "Alpha"
+        assert results[0]["url"] == "https://example.com/a"
+        assert results[0]["score"] == "100 points"
+        assert results[0]["author"] == "alice"
+
+        assert results[1]["rank"] == "2."
+        assert results[1]["title"] == "Beta"
+        assert results[1]["score"] == "42 points"
+        assert results[1]["author"] == "bob"
+
+    def test_backward_compat_no_source(self):
+        """Schema without source key should work exactly as before."""
+        schema = {
+            "name": "HN titles only",
+            "baseSelector": "//tr[contains(@class, 'athing') and contains(@class, 'submission')]",
+            "fields": [
+                {"name": "title", "selector": ".//span[@class='titleline']/a", "type": "text"},
+            ],
+        }
+        results = self._extract(schema)
+        assert len(results) == 2
+        assert results[0]["title"] == "Alpha"
+        assert results[1]["title"] == "Beta"
+
+    def test_source_missing_sibling_returns_default(self):
+        """When source points to a non-existent sibling, field returns its default."""
+        schema = {
+            "name": "HN",
+            "baseSelector": "//tr[contains(@class, 'athing') and contains(@class, 'submission')]",
+            "fields": [
+                {"name": "title", "selector": ".//span[@class='titleline']/a", "type": "text"},
+                {
+                    "name": "missing",
+                    "selector": ".//span",
+                    "type": "text",
+                    "source": "+ div",
+                    "default": "N/A",
+                },
+            ],
+        }
+        results = self._extract(schema)
+        assert len(results) == 2
+        assert results[0]["missing"] == "N/A"
+
+    def test_source_with_class_filter(self):
+        """source='+ tr.spacer' should find the sibling with class 'spacer'."""
+        schema = {
+            "name": "HN spacer",
+            "baseSelector": "//tr[contains(@class, 'athing') and contains(@class, 'submission')]",
+            "fields": [
+                {"name": "title", "selector": ".//span[@class='titleline']/a", "type": "text"},
+                {
+                    "name": "score_from_spacer",
+                    "selector": ".//span[@class='score']",
+                    "type": "text",
+                    "source": "+ tr.spacer",
+                    "default": "none",
+                },
+            ],
+        }
+        results = self._extract(schema)
+        assert results[0]["score_from_spacer"] == "none"
+
+
+# ---------------------------------------------------------------------------
+# Edge case: source on nested/list field types
+# ---------------------------------------------------------------------------
+NESTED_SIBLING_HTML = """\
+
+  
      
+    Item A
+  
      
+  
      
+    $10
+    In Stock
+  
      
+
+  
      
+    Item B
+  
      
+  
      
+    $20
+    Out of Stock
+  
      
+
+"""
+
+
+class TestCssSourceNested:
+    """Test source with nested field types (CSS)."""
+
+    def test_source_on_nested_field(self):
+        """source should work with nested field type — element swap before dispatch."""
+        schema = {
+            "name": "Items",
+            "baseSelector": "div.item",
+            "fields": [
+                {"name": "name", "selector": "span.name", "type": "text"},
+                {
+                    "name": "info",
+                    "type": "nested",
+                    "selector": "div.details",
+                    "source": "+ div.details",
+                    "fields": [
+                        {"name": "price", "selector": "span.price", "type": "text"},
+                        {"name": "stock", "selector": "span.stock", "type": "text"},
+                    ],
+                },
+            ],
+        }
+        strategy = JsonCssExtractionStrategy(schema)
+        results = strategy.extract(None, NESTED_SIBLING_HTML)
+        assert len(results) == 2
+        # The nested selector "div.details" runs inside the sibling div.details,
+        # which IS div.details itself — so BS4 select won't find it as a descendant.
+        # But the element itself is div.details, so we can extract spans from it directly.
+        # Actually, nested type does _get_elements(element, "div.details") which searches descendants.
+        # The resolved element IS div.details, so searching for div.details inside it won't work.
+        # Let's adjust: for nested with source, the selector should target children of the sibling.
+        # This is actually fine — let's just use "source" with flat fields instead.
+
+    def test_source_on_flat_fields_from_sibling(self):
+        """source on individual fields targeting data in sibling div."""
+        schema = {
+            "name": "Items",
+            "baseSelector": "div.item",
+            "fields": [
+                {"name": "name", "selector": "span.name", "type": "text"},
+                {"name": "price", "selector": "span.price", "type": "text", "source": "+ div.details"},
+                {"name": "stock", "selector": "span.stock", "type": "text", "source": "+ div.details"},
+            ],
+        }
+        strategy = JsonCssExtractionStrategy(schema)
+        results = strategy.extract(None, NESTED_SIBLING_HTML)
+        assert len(results) == 2
+        assert results[0]["name"] == "Item A"
+        assert results[0]["price"] == "$10"
+        assert results[0]["stock"] == "In Stock"
+        assert results[1]["name"] == "Item B"
+        assert results[1]["price"] == "$20"
+        assert results[1]["stock"] == "Out of Stock"
+
+
+class TestXPathSourceNested:
+    """Test source with nested field types (XPath)."""
+
+    def test_source_on_flat_fields_from_sibling(self):
+        """source on individual fields targeting data in sibling div."""
+        schema = {
+            "name": "Items",
+            "baseSelector": "//div[@class='item']",
+            "fields": [
+                {"name": "name", "selector": ".//span[@class='name']", "type": "text"},
+                {"name": "price", "selector": ".//span[@class='price']", "type": "text", "source": "+ div.details"},
+                {"name": "stock", "selector": ".//span[@class='stock']", "type": "text", "source": "+ div.details"},
+            ],
+        }
+        strategy = JsonXPathExtractionStrategy(schema)
+        results = strategy.extract(None, NESTED_SIBLING_HTML)
+        assert len(results) == 2
+        assert results[0]["name"] == "Item A"
+        assert results[0]["price"] == "$10"
+        assert results[0]["stock"] == "In Stock"
+        assert results[1]["name"] == "Item B"
+        assert results[1]["price"] == "$20"
+        assert results[1]["stock"] == "Out of Stock"
+
+
+# ---------------------------------------------------------------------------
+# Test invalid source syntax (no "+") returns None gracefully
+# ---------------------------------------------------------------------------
+class TestInvalidSourceSyntax:
+    def test_css_invalid_source_returns_default(self):
+        schema = {
+            "name": "test",
+            "baseSelector": "tr.athing.submission",
+            "fields": [
+                {
+                    "name": "bad",
+                    "selector": "span.score",
+                    "type": "text",
+                    "source": "tr",  # Missing "+" prefix
+                    "default": "fallback",
+                },
+            ],
+        }
+        strategy = JsonCssExtractionStrategy(schema)
+        results = strategy.extract(None, HN_HTML)
+        assert results[0]["bad"] == "fallback"
+
+    def test_xpath_invalid_source_returns_default(self):
+        schema = {
+            "name": "test",
+            "baseSelector": "//tr[contains(@class, 'athing')]",
+            "fields": [
+                {
+                    "name": "bad",
+                    "selector": ".//span[@class='score']",
+                    "type": "text",
+                    "source": "tr",  # Missing "+" prefix
+                    "default": "fallback",
+                },
+            ],
+        }
+        strategy = JsonXPathExtractionStrategy(schema)
+        results = strategy.extract(None, HN_HTML)
+        assert results[0]["bad"] == "fallback"
diff --git a/tests/test_table_gfm_compliance.py b/tests/test_table_gfm_compliance.py
new file mode 100644
index 0000000..f420008
--- /dev/null
+++ b/tests/test_table_gfm_compliance.py
@@ -0,0 +1,247 @@
+"""
+Unit tests for GFM-compliant markdown table generation.
+
+Tests that html2text generates tables with proper leading and trailing
+pipe delimiters as per GitHub Flavored Markdown specification.
+
+Fixes: https://github.com/unclecode/crawl4ai/issues/1731
+"""
+
+import pytest
+import sys
+import os
+
+# Add parent directory to path to import html2text
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawl4ai.html2text import HTML2Text
+
+
+def _table_lines(result: str) -> list[str]:
+    """Extract table lines (containing |) from html2text output, stripped."""
+    return [line.strip() for line in result.split('\n') if '|' in line]
+
+
+class TestTableGFMCompliance:
+    """Test suite for GFM-compliant table generation (pad_tables=False)."""
+
+    def test_table_has_leading_pipes(self):
+        """All table rows start with |"""
+        html = '
      AB
      12
      '
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        lines = _table_lines(result)
+        assert len(lines) > 0, "No table rows found in output"
+
+        for i, line in enumerate(lines):
+            assert line.startswith('|'), f"Line {i+1} missing leading pipe: {repr(line)}"
+
+    def test_table_has_trailing_pipes(self):
+        """All table rows end with |"""
+        html = '
      AB
      12
      '
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        lines = _table_lines(result)
+        assert len(lines) > 0, "No table rows found in output"
+
+        for i, line in enumerate(lines):
+            assert line.endswith('|'), f"Line {i+1} missing trailing pipe: {repr(line)}"
+
+    def test_cells_have_space_padding(self):
+        """Cell content has spaces on both sides of pipe delimiters.
+
+        Correct:   | A | B |
+        Incorrect: | A| B |  (missing space after A)
+        """
+        html = '
      AB
      12
      '
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        lines = _table_lines(result)
+        # Check header and data rows (skip separator)
+        for line in lines:
+            if '---' in line:
+                continue
+            # Split by | and check that internal cells have spaces
+            cells = line.split('|')
+            # cells[0] is '' (before first |), cells[-1] is '' (after last |)
+            for cell in cells[1:-1]:
+                if cell.strip():  # Non-empty cell
+                    assert cell.startswith(' '), f"Cell missing leading space: {repr(line)}"
+                    assert cell.endswith(' '), f"Cell missing trailing space: {repr(line)}"
+
+    def test_separator_row_format(self):
+        """Separator row has format | --- | --- |"""
+        html = '
      AB
      12
      '
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        separators = [line.strip() for line in result.split('\n') if '---' in line]
+        assert len(separators) > 0, "No separator row found"
+
+        sep = separators[0]
+        assert sep.startswith('|'), f"Separator missing leading pipe: {repr(sep)}"
+        assert sep.endswith('|'), f"Separator missing trailing pipe: {repr(sep)}"
+        assert sep == '| --- | --- |', f"Unexpected separator format: {repr(sep)}"
+
+    def test_multirow_table(self):
+        """Multi-row tables maintain GFM compliance throughout."""
+        html = '''
+            
+            
+            
+            
+        
      ParameterGuidelineSources
      Arsenic0.010Naturally occurring
      Lead0.005Plumbing
      Mercury0.001Industrial
      '''
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        lines = _table_lines(result)
+        # 1 header + 1 separator + 3 data rows = 5 rows
+        assert len(lines) >= 5, f"Expected at least 5 table rows, got {len(lines)}"
+
+        for i, line in enumerate(lines):
+            assert line.startswith('|'), f"Line {i+1} missing leading pipe"
+            assert line.endswith('|'), f"Line {i+1} missing trailing pipe"
+
+    def test_single_column_table(self):
+        """Single-column tables are GFM compliant."""
+        html = '
      Header
      Data
      '
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        lines = _table_lines(result)
+        assert len(lines) > 0, "No table rows found"
+        for line in lines:
+            assert line.startswith('|') and line.endswith('|'), \
+                f"Single column row not GFM compliant: {repr(line)}"
+
+    def test_empty_cells(self):
+        """Tables with empty cells are GFM compliant."""
+        html = '
      AB
      Data
      '
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        lines = _table_lines(result)
+        assert len(lines) > 0, "No table rows found"
+        for line in lines:
+            assert line.startswith('|') and line.endswith('|'), \
+                f"Row with empty cell not GFM compliant: {repr(line)}"
+
+    def test_table_starts_on_own_line(self):
+        """Table starts on its own line, not inline with preceding content."""
+        html = '
      Text before.
      AB
      12
      '
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        # The first table line must start at the beginning of a line
+        # (no leading whitespace before the pipe)
+        for line in result.split('\n'):
+            stripped = line.strip()
+            if stripped.startswith('|'):
+                # Check no leading whitespace (table row at column 0)
+                assert line.startswith('|'), f"Table row not at line start: {repr(line)}"
+                break
+        else:
+            pytest.fail("No table row starting with | found")
+
+    def test_nested_table_starts_on_own_line(self):
+        """Table nested in list item starts on its own line."""
+        html = '
      • Item
        X
        1
      '
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        # Find the first | line — it should not be on the same line as "Item"
+        lines = result.split('\n')
+        for line in lines:
+            if 'Item' in line:
+                assert '|' not in line, \
+                    f"Table pipe on same line as 'Item': {repr(line)}"
+                break
+
+    def test_caption_does_not_merge_with_header(self):
+        """Table with  renders caption on its own line, not inline with header."""
+        html = '''
+            
+            
+            
+        
      Table 1. Parameters
      NameValue
      Lead0.005
      '''
+        h = HTML2Text()
+        h.body_width = 0
+        result = h.handle(html)
+
+        # Caption text should NOT be on the same line as the header pipe row
+        for line in result.split('\n'):
+            if 'Table 1' in line:
+                assert '|' not in line, \
+                    f"Caption on same line as table row: {repr(line)}"
+                break
+
+        # Header row should start with |
+        table_lines = _table_lines(result)
+        assert len(table_lines) >= 3, f"Expected at least 3 table rows, got {len(table_lines)}"
+        assert table_lines[0].startswith('|'), f"Header not starting with pipe: {repr(table_lines[0])}"
+
+
+class TestPadTablesUnchanged:
+    """Verify pad_tables=True behavior is unchanged from upstream."""
+
+    def test_pad_tables_produces_aligned_output(self):
+        """pad_tables=True produces properly aligned GFM output."""
+        html = '
      ParameterValue
      Lead0.005 mg/L
      '
+        h = HTML2Text()
+        h.body_width = 0
+        h.pad_tables = True
+        result = h.handle(html)
+
+        lines = _table_lines(result)
+        assert len(lines) >= 3, f"Expected at least 3 rows, got {len(lines)}"
+
+        # All rows should have leading and trailing pipes
+        for line in lines:
+            assert line.startswith('|'), f"Padded row missing leading pipe: {repr(line)}"
+            assert line.endswith('|'), f"Padded row missing trailing pipe: {repr(line)}"
+
+        # All rows should have same width (padded alignment)
+        widths = [len(line) for line in lines]
+        assert len(set(widths)) == 1, f"Rows have uneven widths: {widths}"
+
+    def test_pad_tables_no_double_pipes(self):
+        """pad_tables=True does not produce double pipes |  | or | |."""
+        html = '
      AB
      12
      '
+        h = HTML2Text()
+        h.body_width = 0
+        h.pad_tables = True
+        result = h.handle(html)
+
+        lines = _table_lines(result)
+        for line in lines:
+            # Should not have pipe-space-pipe (double boundary)
+            assert '| |' not in line, f"Double pipes found: {repr(line)}"
+            # Line should not start with |  | (extra pipe from both systems)
+            assert not line.startswith('|  |'), f"Extra leading pipe: {repr(line)}"
+
+    def test_pad_tables_separator_has_dashes(self):
+        """pad_tables=True separator row uses dashes with proper alignment."""
+        html = '
      AB
      12
      '
+        h = HTML2Text()
+        h.body_width = 0
+        h.pad_tables = True
+        result = h.handle(html)
+
+        separators = [line.strip() for line in result.split('\n') if '---' in line]
+        assert len(separators) >= 1, "No separator row found in padded table"
+
+
+if __name__ == '__main__':
+    pytest.main([__file__, '-v'])
diff --git a/tests/test_type_annotations.py b/tests/test_type_annotations.py
new file mode 100644
index 0000000..4554e32
--- /dev/null
+++ b/tests/test_type_annotations.py
@@ -0,0 +1,194 @@
+"""
+Tests for type annotation correctness across the crawl4ai public API.
+
+Catches issues like #1898 where arun() was annotated as returning
+RunManyReturn (includes AsyncGenerator) but actually always returns
+CrawlResultContainer.
+
+These tests use static analysis (inspect + typing introspection) to verify
+return type annotations match actual behavior — without needing pyright/mypy
+installed in CI.
+"""
+import asyncio
+import collections.abc
+import inspect
+import typing
+from typing import Union, AsyncGenerator, get_type_hints
+import pytest
+
+
+# ── Return type annotation tests ─────────────────────────────────────────
+
+class TestReturnTypeAnnotations:
+    """Verify that public method return types are correctly annotated."""
+
+    def _get_return_annotation(self, cls, method_name):
+        """Get the return type annotation for a method."""
+        method = getattr(cls, method_name)
+        hints = typing.get_type_hints(method)
+        return hints.get("return")
+
+    def _annotation_contains(self, annotation, target_type):
+        """Check if a type annotation contains a specific type (including in unions).
+
+        Handles both typing.AsyncGenerator and collections.abc.AsyncGenerator,
+        since typing generics have __origin__ pointing to the abc class.
+        """
+        if annotation is target_type:
+            return True
+        origin = getattr(annotation, "__origin__", None)
+        if origin is target_type:
+            return True
+        if origin is Union:
+            return any(
+                self._annotation_contains(arg, target_type)
+                for arg in annotation.__args__
+            )
+        return False
+
+    def test_arun_does_not_include_async_generator(self):
+        """
+        #1898: arun() always returns CrawlResultContainer, never AsyncGenerator.
+        The return type should NOT include AsyncGenerator.
+        """
+        from crawl4ai import AsyncWebCrawler
+        ret = self._get_return_annotation(AsyncWebCrawler, "arun")
+        assert ret is not None, "arun() has no return type annotation"
+
+        # Check both typing.AsyncGenerator and collections.abc.AsyncGenerator
+        # (typing generics have __origin__ = collections.abc.AsyncGenerator)
+        has_async_gen = (
+            self._annotation_contains(ret, collections.abc.AsyncGenerator)
+            or self._annotation_contains(ret, AsyncGenerator)
+        )
+        assert not has_async_gen, (
+            f"arun() return type includes AsyncGenerator but arun() never yields. "
+            f"Current annotation: {ret}. "
+            f"Should be CrawlResultContainer or CrawlResult."
+        )
+
+    def test_arun_many_return_type_exists(self):
+        """arun_many() should have a return type annotation."""
+        from crawl4ai import AsyncWebCrawler
+        ret = self._get_return_annotation(AsyncWebCrawler, "arun_many")
+        assert ret is not None, "arun_many() has no return type annotation"
+
+    def test_aprocess_html_return_type_exists(self):
+        """aprocess_html() should have a return type annotation."""
+        from crawl4ai import AsyncWebCrawler
+        ret = self._get_return_annotation(AsyncWebCrawler, "aprocess_html")
+        assert ret is not None, "aprocess_html() has no return type annotation"
+
+
+# ── Parameter annotation tests ───────────────────────────────────────────
+
+class TestParameterAnnotations:
+    """Verify that public method parameters have type annotations."""
+
+    def _get_untyped_params(self, cls, method_name, ignore=("self", "kwargs")):
+        """Find parameters without type annotations."""
+        method = getattr(cls, method_name)
+        sig = inspect.signature(method)
+        untyped = []
+        for name, param in sig.parameters.items():
+            if name in ignore:
+                continue
+            if param.annotation is inspect.Parameter.empty:
+                untyped.append(name)
+        return untyped
+
+    def test_arun_params_annotated(self):
+        """arun() public params should have type annotations."""
+        from crawl4ai import AsyncWebCrawler
+        untyped = self._get_untyped_params(AsyncWebCrawler, "arun")
+        # Allow **kwargs to be untyped, but core params should be typed
+        assert "url" not in untyped, "arun(url=...) missing type annotation"
+        assert "config" not in untyped, "arun(config=...) missing type annotation"
+
+    def test_arun_many_params_annotated(self):
+        """arun_many() public params should have type annotations."""
+        from crawl4ai import AsyncWebCrawler
+        untyped = self._get_untyped_params(AsyncWebCrawler, "arun_many")
+        assert "urls" not in untyped, "arun_many(urls=...) missing type annotation"
+
+
+# ── Consistency tests ────────────────────────────────────────────────────
+
+class TestAnnotationConsistency:
+    """Verify that annotations match actual runtime behavior."""
+
+    def test_arun_actually_returns_what_annotation_says(self):
+        """
+        Whatever arun()'s return annotation says, verify that
+        CrawlResultContainer is compatible with it.
+        """
+        from crawl4ai import AsyncWebCrawler
+        from crawl4ai.models import CrawlResultContainer
+
+        hints = typing.get_type_hints(AsyncWebCrawler.arun)
+        ret = hints.get("return")
+        if ret is None:
+            pytest.skip("No return annotation to check")
+
+        # Get all types in the annotation
+        origin = getattr(ret, "__origin__", None)
+        if origin is Union:
+            allowed_types = [
+                getattr(arg, "__origin__", arg) for arg in ret.__args__
+            ]
+        else:
+            allowed_types = [getattr(ret, "__origin__", ret)]
+
+        # CrawlResultContainer should be in the allowed types
+        is_compatible = any(
+            t is CrawlResultContainer
+            or (isinstance(t, type) and issubclass(CrawlResultContainer, t))
+            for t in allowed_types
+        )
+        assert is_compatible, (
+            f"arun() returns CrawlResultContainer at runtime, but annotation {ret} "
+            f"doesn't include it. Allowed types: {allowed_types}"
+        )
+
+    def test_config_classes_init_params_match_attributes(self):
+        """
+        CrawlerRunConfig.__init__ params should become attributes.
+        Catches cases where a param is added to __init__ but not stored.
+        """
+        from crawl4ai import CrawlerRunConfig
+
+        sig = inspect.signature(CrawlerRunConfig.__init__)
+        config = CrawlerRunConfig()
+
+        missing = []
+        for name in sig.parameters:
+            if name == "self":
+                continue
+            if not hasattr(config, name):
+                missing.append(name)
+
+        assert not missing, (
+            f"CrawlerRunConfig.__init__ has params that aren't stored as attributes: {missing}"
+        )
+
+
+# ── Public API export tests ──────────────────────────────────────────────
+
+class TestPublicAPIExports:
+    """Verify that types referenced in public annotations are importable."""
+
+    def test_crawl_result_importable(self):
+        from crawl4ai import CrawlResult
+        assert CrawlResult is not None
+
+    def test_crawl_result_container_importable(self):
+        from crawl4ai.models import CrawlResultContainer
+        assert CrawlResultContainer is not None
+
+    def test_run_many_return_importable(self):
+        from crawl4ai.models import RunManyReturn
+        assert RunManyReturn is not None
+
+    def test_markdown_generation_result_importable(self):
+        from crawl4ai import MarkdownGenerationResult
+        assert MarkdownGenerationResult is not None
diff --git a/tests/test_virtual_scroll.py b/tests/test_virtual_scroll.py
new file mode 100644
index 0000000..1e7a789
--- /dev/null
+++ b/tests/test_virtual_scroll.py
@@ -0,0 +1,197 @@
+"""
+Test virtual scroll implementation according to the design:
+- Create a page with virtual scroll that replaces content
+- Verify all 1000 items are captured
+"""
+
+import asyncio
+import os
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, VirtualScrollConfig, CacheMode, BrowserConfig
+
+async def test_virtual_scroll():
+    """Test virtual scroll with content replacement (true virtual scroll)"""
+    
+    # Create test HTML with true virtual scroll that replaces content
+    test_html = '''
+    
+    
+        
+    
+    
+        
      Virtual Scroll Test - 1000 Items
      
+        
      
+        
+    
+    
+    '''
+    
+    # Save test HTML to a file
+    import tempfile
+    
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.html', delete=False) as f:
+        f.write(test_html)
+        test_file_path = f.name
+    
+    httpd = None
+    old_cwd = os.getcwd()
+    
+    try:
+        # Start a simple HTTP server
+        import http.server
+        import socketserver
+        import threading
+        import random
+        
+        # Find available port
+        for _ in range(10):
+            PORT = random.randint(8000, 9999)
+            try:
+                Handler = http.server.SimpleHTTPRequestHandler
+                os.chdir(os.path.dirname(test_file_path))
+                httpd = socketserver.TCPServer(("", PORT), Handler)
+                break
+            except OSError:
+                continue
+        
+        if httpd is None:
+            raise RuntimeError("Could not find available port")
+            
+        server_thread = threading.Thread(target=httpd.serve_forever)
+        server_thread.daemon = True
+        server_thread.start()
+        
+        # Give server time to start
+        await asyncio.sleep(0.5)
+        
+        # Configure virtual scroll
+        # With 10 items per page and 1000 total, we need 100 pages
+        # Let's do 120 scrolls to ensure we get everything
+        virtual_config = VirtualScrollConfig(
+            container_selector="#container",
+            scroll_count=120,
+            scroll_by="container_height",  # Scroll by container height
+            wait_after_scroll=0.1  # Quick wait for test
+        )
+        
+        config = CrawlerRunConfig(
+            virtual_scroll_config=virtual_config,
+            cache_mode=CacheMode.BYPASS,
+            verbose=True
+        )
+        
+        browserConfig = BrowserConfig(
+            headless= False
+        )
+        
+        async with AsyncWebCrawler(verbose=True, config=browserConfig) as crawler:
+            result = await crawler.arun(
+                url=f"http://localhost:{PORT}/{os.path.basename(test_file_path)}",
+                config=config
+            )
+            
+            # Count all items in the result
+            import re
+            items = re.findall(r'data-index="(\d+)"', result.html)
+            unique_indices = sorted(set(int(idx) for idx in items))
+            
+            print(f"\n{'='*60}")
+            print(f"TEST RESULTS:")
+            print(f"HTML Length: {len(result.html)}")
+            print(f"Total items found: {len(items)}")
+            print(f"Unique items: {len(unique_indices)}")
+            
+            if unique_indices:
+                print(f"Item indices: {min(unique_indices)} to {max(unique_indices)}")
+                print(f"Expected: 0 to 999")
+                
+                # Check for gaps
+                expected = set(range(1000))
+                actual = set(unique_indices)
+                missing = expected - actual
+                
+                if missing:
+                    print(f"\n❌ FAILED! Missing {len(missing)} items")
+                    print(f"Missing indices: {sorted(missing)[:10]}{'...' if len(missing) > 10 else ''}")
+                else:
+                    print(f"\n✅ SUCCESS! All 1000 items captured!")
+                    
+                # Show some sample items
+                print(f"\nSample items from result:")
+                sample_items = re.findall(r'
      ]*>([^<]+)
      ', result.html)[:5]
+                for item in sample_items:
+                    print(f"  - {item}")
+            
+            print(f"{'='*60}\n")
+                
+    finally:
+        # Clean up
+        if httpd:
+            httpd.shutdown()
+        os.chdir(old_cwd)
+        os.unlink(test_file_path)
+
+if __name__ == "__main__":
+    asyncio.run(test_virtual_scroll())
\ No newline at end of file
diff --git a/tests/test_web_crawler.py b/tests/test_web_crawler.py
new file mode 100644
index 0000000..162f85f
--- /dev/null
+++ b/tests/test_web_crawler.py
@@ -0,0 +1,149 @@
+import unittest, os
+from crawl4ai import LLMConfig
+from crawl4ai.web_crawler import WebCrawler
+from crawl4ai.chunking_strategy import (
+    RegexChunking,
+    FixedLengthWordChunking,
+    SlidingWindowChunking,
+)
+from crawl4ai import (
+    CosineStrategy,
+    LLMExtractionStrategy,
+    TopicExtractionStrategy,
+    NoExtractionStrategy,
+)
+
+
+class TestWebCrawler(unittest.TestCase):
+    def setUp(self):
+        self.crawler = WebCrawler()
+
+    def test_warmup(self):
+        self.crawler.warmup()
+        self.assertTrue(self.crawler.ready, "WebCrawler failed to warm up")
+
+    def test_run_default_strategies(self):
+        result = self.crawler.run(
+            url="https://www.nbcnews.com/business",
+            word_count_threshold=5,
+            chunking_strategy=RegexChunking(),
+            extraction_strategy=CosineStrategy(),
+            bypass_cache=True,
+        )
+        self.assertTrue(
+            result.success, "Failed to crawl and extract using default strategies"
+        )
+
+    def test_run_different_strategies(self):
+        url = "https://www.nbcnews.com/business"
+
+        # Test with FixedLengthWordChunking and LLMExtractionStrategy
+        result = self.crawler.run(
+            url=url,
+            word_count_threshold=5,
+            chunking_strategy=FixedLengthWordChunking(chunk_size=100),
+            extraction_strategy=LLMExtractionStrategy(
+                llm_config=LLMConfig(provider="openai/gpt-3.5-turbo", api_token=os.getenv("OPENAI_API_KEY"))
+            ),
+            bypass_cache=True,
+        )
+        self.assertTrue(
+            result.success,
+            "Failed to crawl and extract with FixedLengthWordChunking and LLMExtractionStrategy",
+        )
+
+        # Test with SlidingWindowChunking and TopicExtractionStrategy
+        result = self.crawler.run(
+            url=url,
+            word_count_threshold=5,
+            chunking_strategy=SlidingWindowChunking(window_size=100, step=50),
+            extraction_strategy=TopicExtractionStrategy(num_keywords=5),
+            bypass_cache=True,
+        )
+        self.assertTrue(
+            result.success,
+            "Failed to crawl and extract with SlidingWindowChunking and TopicExtractionStrategy",
+        )
+
+    def test_invalid_url(self):
+        with self.assertRaises(Exception) as context:
+            self.crawler.run(url="invalid_url", bypass_cache=True)
+        self.assertIn("Invalid URL", str(context.exception))
+
+    def test_unsupported_extraction_strategy(self):
+        with self.assertRaises(Exception) as context:
+            self.crawler.run(
+                url="https://www.nbcnews.com/business",
+                extraction_strategy="UnsupportedStrategy",
+                bypass_cache=True,
+            )
+        self.assertIn("Unsupported extraction strategy", str(context.exception))
+
+    def test_invalid_css_selector(self):
+        with self.assertRaises(ValueError) as context:
+            self.crawler.run(
+                url="https://www.nbcnews.com/business",
+                css_selector="invalid_selector",
+                bypass_cache=True,
+            )
+        self.assertIn("Invalid CSS selector", str(context.exception))
+
+    def test_crawl_with_cache_and_bypass_cache(self):
+        url = "https://www.nbcnews.com/business"
+
+        # First crawl with cache enabled
+        result = self.crawler.run(url=url, bypass_cache=False)
+        self.assertTrue(result.success, "Failed to crawl and cache the result")
+
+        # Second crawl with bypass_cache=True
+        result = self.crawler.run(url=url, bypass_cache=True)
+        self.assertTrue(result.success, "Failed to bypass cache and fetch fresh data")
+
+    def test_fetch_multiple_pages(self):
+        urls = ["https://www.nbcnews.com/business", "https://www.bbc.com/news"]
+        results = []
+        for url in urls:
+            result = self.crawler.run(
+                url=url,
+                word_count_threshold=5,
+                chunking_strategy=RegexChunking(),
+                extraction_strategy=CosineStrategy(),
+                bypass_cache=True,
+            )
+            results.append(result)
+
+        self.assertEqual(len(results), 2, "Failed to crawl and extract multiple pages")
+        for result in results:
+            self.assertTrue(
+                result.success, "Failed to crawl and extract a page in the list"
+            )
+
+    def test_run_fixed_length_word_chunking_and_no_extraction(self):
+        result = self.crawler.run(
+            url="https://www.nbcnews.com/business",
+            word_count_threshold=5,
+            chunking_strategy=FixedLengthWordChunking(chunk_size=100),
+            extraction_strategy=NoExtractionStrategy(),
+            bypass_cache=True,
+        )
+        self.assertTrue(
+            result.success,
+            "Failed to crawl and extract with FixedLengthWordChunking and NoExtractionStrategy",
+        )
+
+    def test_run_sliding_window_and_no_extraction(self):
+        result = self.crawler.run(
+            url="https://www.nbcnews.com/business",
+            word_count_threshold=5,
+            chunking_strategy=SlidingWindowChunking(window_size=100, step=50),
+            extraction_strategy=NoExtractionStrategy(),
+            bypass_cache=True,
+        )
+        self.assertTrue(
+            result.success,
+            "Failed to crawl and extract with SlidingWindowChunking and NoExtractionStrategy",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/unit/test_domain_mapper_unit.py b/tests/unit/test_domain_mapper_unit.py
new file mode 100644
index 0000000..74c92d4
--- /dev/null
+++ b/tests/unit/test_domain_mapper_unit.py
@@ -0,0 +1,406 @@
+"""Unit tests for DomainMapper: soft-404, robots.txt, feeds, normalization, nonsense filter."""
+import asyncio
+import hashlib
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+from crawl4ai.domain_mapper import (
+    DomainMapper,
+    Soft404Fingerprint,
+    _NONSENSE_SUFFIXES,
+    _ASSET_EXTENSIONS,
+)
+
+
+# ════════════════════════════════════════════════════════════════════════
+#  Soft-404 Detection
+# ════════════════════════════════════════════════════════════════════════
+
+class TestSoft404Detection:
+
+    def test_is_soft_404_title_match(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        fp = Soft404Fingerprint(
+            status_code=200, title="Page Not Found",
+            content_length=1234, body_hash="abc123",
+        )
+        body = b"Page Not FoundOops"
+        assert mapper._is_soft_404(200, body, fp) is True
+
+    def test_is_soft_404_hash_match(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        body = b"Different TitleError content"
+        body_hash = hashlib.md5(body[:2048]).hexdigest()
+        fp = Soft404Fingerprint(
+            status_code=200, title="Other Title",
+            content_length=len(body), body_hash=body_hash,
+        )
+        assert mapper._is_soft_404(200, body, fp) is True
+
+    def test_is_soft_404_real_404(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        fp = Soft404Fingerprint(
+            status_code=200, title="Not Found",
+            content_length=100, body_hash="abc",
+        )
+        body = b"Not Found"
+        # Real 404 status — NOT a soft-404
+        assert mapper._is_soft_404(404, body, fp) is False
+
+    def test_is_soft_404_no_fingerprint(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        body = b"Anything"
+        assert mapper._is_soft_404(200, body, None) is False
+
+    def test_is_soft_404_different_content(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        fp = Soft404Fingerprint(
+            status_code=200, title="Not Found",
+            content_length=100, body_hash="abc123",
+        )
+        body = b"Real PageActual content here"
+        assert mapper._is_soft_404(200, body, fp) is False
+
+    def test_is_soft_404_no_title_in_body(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        fp = Soft404Fingerprint(
+            status_code=200, title="Not Found",
+            content_length=100, body_hash="abc123",
+        )
+        body = b"No title tag"
+        assert mapper._is_soft_404(200, body, fp) is False
+
+
+# ════════════════════════════════════════════════════════════════════════
+#  robots.txt Parsing
+# ════════════════════════════════════════════════════════════════════════
+
+class TestRobotsTxtParsing:
+
+    @pytest.mark.asyncio
+    async def test_parse_sitemap_directives(self):
+        robots_text = (
+            "User-agent: *\n"
+            "Disallow: /private/\n"
+            "Sitemap: https://example.com/sitemap.xml\n"
+            "Sitemap: https://example.com/sitemap-posts.xml\n"
+        )
+        mapper = DomainMapper.__new__(DomainMapper)
+        mapper.logger = None
+        mapper.client = AsyncMock()
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.text = robots_text
+        mapper.client.get = AsyncMock(return_value=resp)
+
+        from crawl4ai.async_configs import DomainMapperConfig
+        config = DomainMapperConfig()
+
+        sitemap_urls, disallow_paths = await mapper._scan_robots_txt("example.com", config)
+        assert len(sitemap_urls) == 2
+        assert "https://example.com/sitemap.xml" in sitemap_urls
+        assert "/private/" in disallow_paths
+
+    @pytest.mark.asyncio
+    async def test_parse_disallow_ignores_wildcards(self):
+        robots_text = (
+            "User-agent: *\n"
+            "Disallow: /admin/\n"
+            "Disallow: /search?*\n"
+            "Disallow: /\n"
+            "Allow: /public/\n"
+        )
+        mapper = DomainMapper.__new__(DomainMapper)
+        mapper.logger = None
+        mapper.client = AsyncMock()
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.text = robots_text
+        mapper.client.get = AsyncMock(return_value=resp)
+
+        from crawl4ai.async_configs import DomainMapperConfig
+        config = DomainMapperConfig()
+
+        _, paths = await mapper._scan_robots_txt("example.com", config)
+        assert "/admin/" in paths
+        assert "/public/" in paths
+        # Wildcards should be skipped
+        assert "/search?*" not in paths
+        # Single "/" is too short (len <= 1)
+        assert "/" not in paths
+
+    @pytest.mark.asyncio
+    async def test_empty_robots(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        mapper.logger = None
+        mapper.client = AsyncMock()
+        resp = MagicMock()
+        resp.status_code = 404
+        mapper.client.get = AsyncMock(return_value=resp)
+
+        from crawl4ai.async_configs import DomainMapperConfig
+        config = DomainMapperConfig()
+
+        sitemap_urls, paths = await mapper._scan_robots_txt("example.com", config)
+        assert sitemap_urls == []
+        assert paths == []
+
+
+# ════════════════════════════════════════════════════════════════════════
+#  Feed Parsing
+# ════════════════════════════════════════════════════════════════════════
+
+class TestFeedParsing:
+
+    def test_parse_rss_feed(self):
+        rss = """
+        
+          
+            https://example.com/post-1
+            https://example.com/post-2
+          
+        """
+        mapper = DomainMapper.__new__(DomainMapper)
+        urls = mapper._parse_feed_xml(rss, "https://example.com/feed")
+        assert len(urls) == 2
+        assert "https://example.com/post-1" in urls
+        assert "https://example.com/post-2" in urls
+
+    def test_parse_atom_feed(self):
+        atom = """
+        
+          
+            
+          
+          
+            
+          
+        """
+        mapper = DomainMapper.__new__(DomainMapper)
+        urls = mapper._parse_feed_xml(atom, "https://example.com/atom.xml")
+        assert len(urls) == 2
+        assert "https://example.com/entry-1" in urls
+
+    def test_parse_rss_guid_fallback(self):
+        rss = """
+        
+          
+            
+              https://example.com/guid-post
+            
+          
+        """
+        mapper = DomainMapper.__new__(DomainMapper)
+        urls = mapper._parse_feed_xml(rss, "https://example.com/feed")
+        assert "https://example.com/guid-post" in urls
+
+    def test_malformed_feed(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        urls = mapper._parse_feed_xml("not xml at all <><>", "https://example.com/feed")
+        assert urls == []
+
+
+# ════════════════════════════════════════════════════════════════════════
+#  URL Normalization & Dedup
+# ════════════════════════════════════════════════════════════════════════
+
+class TestNormalizationDedup:
+
+    def test_trailing_slash_dedup(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        results = [
+            {"url": "https://example.com/about", "host": "example.com", "source": "sitemap", "status": "valid", "head_data": {}},
+            {"url": "https://example.com/about/", "host": "example.com", "source": "probe", "status": "valid", "head_data": {}},
+        ]
+        deduped = mapper._normalize_and_dedup(results, "example.com")
+        assert len(deduped) == 1
+        assert "probe" in deduped[0]["source"] or "sitemap" in deduped[0]["source"]
+
+    def test_source_merging(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        results = [
+            {"url": "https://example.com/page", "host": "example.com", "source": "sitemap", "status": "valid", "head_data": {}},
+            {"url": "https://example.com/page", "host": "example.com", "source": "homepage", "status": "valid", "head_data": {}},
+        ]
+        deduped = mapper._normalize_and_dedup(results, "example.com")
+        assert len(deduped) == 1
+        sources = set(deduped[0]["source"].split("+"))
+        assert "sitemap" in sources
+        assert "homepage" in sources
+
+
+# ════════════════════════════════════════════════════════════════════════
+#  Nonsense Filter
+# ════════════════════════════════════════════════════════════════════════
+
+class TestNonsenseFilter:
+
+    def test_filters_robots_txt(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/robots.txt") is True
+
+    def test_filters_sitemap_xml(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/sitemap.xml") is True
+
+    def test_filters_js_assets(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/app.bundle.js") is True
+
+    def test_filters_css_assets(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/style.css") is True
+
+    def test_filters_images(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/logo.png") is True
+        assert mapper._is_nonsense("https://example.com/photo.jpg") is True
+
+    def test_filters_next_js_chunks(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/_next/static/chunks/main.js") is True
+
+    def test_filters_wayback_garbage(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/%5Cn-") is True
+        assert mapper._is_nonsense("https://example.com/%5CnJoin") is True
+
+    def test_keeps_login(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/login") is False
+
+    def test_keeps_dashboard(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/dashboard") is False
+
+    def test_keeps_docs(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/docs") is False
+
+    def test_keeps_api_docs(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/api-docs") is False
+
+    def test_filters_dotfiles(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/.env") is True
+        assert mapper._is_nonsense("https://example.com/.git/config") is True
+
+    def test_filters_fonts(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        assert mapper._is_nonsense("https://example.com/fonts/arial.woff2") is True
+
+
+# ════════════════════════════════════════════════════════════════════════
+#  crt.sh Response Parsing
+# ════════════════════════════════════════════════════════════════════════
+
+class TestCrtShParsing:
+
+    @pytest.mark.asyncio
+    async def test_parse_crt_response(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        mapper.logger = None
+        mapper.client = AsyncMock()
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.json.return_value = [
+            {"common_name": "example.com", "name_value": "example.com"},
+            {"common_name": "docs.example.com", "name_value": "docs.example.com\napi.example.com"},
+            {"common_name": "*.example.com", "name_value": "*.example.com"},
+        ]
+        mapper.client.get = AsyncMock(return_value=resp)
+
+        from crawl4ai.async_configs import DomainMapperConfig
+        hosts = await mapper._discover_via_crt("example.com", DomainMapperConfig())
+        assert "example.com" in hosts
+        assert "docs.example.com" in hosts
+        assert "api.example.com" in hosts
+        # Wildcards should be resolved to base
+        assert "*.example.com" not in hosts
+
+    @pytest.mark.asyncio
+    async def test_crt_filters_unrelated_domains(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        mapper.logger = None
+        mapper.client = AsyncMock()
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.json.return_value = [
+            {"common_name": "example.com", "name_value": "example.com"},
+            {"common_name": "evil.com", "name_value": "evil.com"},
+        ]
+        mapper.client.get = AsyncMock(return_value=resp)
+
+        from crawl4ai.async_configs import DomainMapperConfig
+        hosts = await mapper._discover_via_crt("example.com", DomainMapperConfig())
+        assert "example.com" in hosts
+        assert "evil.com" not in hosts
+
+    @pytest.mark.asyncio
+    async def test_crt_handles_failure(self):
+        mapper = DomainMapper.__new__(DomainMapper)
+        mapper.logger = None
+        mapper.client = AsyncMock()
+        mapper.client.get = AsyncMock(side_effect=Exception("timeout"))
+
+        from crawl4ai.async_configs import DomainMapperConfig
+        hosts = await mapper._discover_via_crt("example.com", DomainMapperConfig())
+        assert hosts == set()
+
+
+# ════════════════════════════════════════════════════════════════════════
+#  Homepage Link Extraction
+# ════════════════════════════════════════════════════════════════════════
+
+class TestHomepageLinkExtraction:
+
+    @pytest.mark.asyncio
+    async def test_extract_internal_links(self):
+        html = """
+        Test
+        
+                  About
+            Blog
+            External
+        """
+
+        mapper = DomainMapper.__new__(DomainMapper)
+        mapper.logger = None
+        mapper.client = AsyncMock()
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.text = html
+        resp.url = "https://example.com/"
+        mapper.client.get = AsyncMock(return_value=resp)
+
+        from crawl4ai.async_configs import DomainMapperConfig
+        urls = await mapper._scan_homepage("example.com", "example.com", DomainMapperConfig())
+        # Should have internal links, not external
+        assert any("/about" in u for u in urls)
+        assert any("/blog" in u for u in urls)
+        assert not any("external.com" in u for u in urls)
+
+    @pytest.mark.asyncio
+    async def test_extract_link_tags(self):
+        html = """
+        
+            Test
+            
+            
+        
+        Link"""
+
+        mapper = DomainMapper.__new__(DomainMapper)
+        mapper.logger = None
+        mapper.client = AsyncMock()
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.text = html
+        resp.url = "https://example.com/"
+        mapper.client.get = AsyncMock(return_value=resp)
+
+        from crawl4ai.async_configs import DomainMapperConfig
+        urls = await mapper._scan_homepage("example.com", "example.com", DomainMapperConfig())
+        # Should include link tags
+        assert any("/es/" in u for u in urls)
+        assert any("/features" in u for u in urls)
diff --git a/tests/unit/test_resource_filtering_config.py b/tests/unit/test_resource_filtering_config.py
new file mode 100644
index 0000000..738cc46
--- /dev/null
+++ b/tests/unit/test_resource_filtering_config.py
@@ -0,0 +1,100 @@
+"""Unit tests for BrowserConfig avoid_ads / avoid_css flags.
+
+Tests the config plumbing: defaults, serialization, cloning, roundtrips.
+No browser or network required.
+"""
+
+import pytest
+from crawl4ai.async_configs import BrowserConfig
+
+
+@pytest.fixture(autouse=True)
+def _reset_defaults():
+    """Ensure clean slate for each test."""
+    BrowserConfig.reset_defaults()
+    yield
+    BrowserConfig.reset_defaults()
+
+
+class TestResourceFilteringDefaults:
+    """Both flags must default to False (opt-in only)."""
+
+    def test_default_values_are_false(self):
+        config = BrowserConfig()
+        assert config.avoid_ads is False
+        assert config.avoid_css is False
+
+    def test_custom_values(self):
+        config = BrowserConfig(avoid_ads=True, avoid_css=True)
+        assert config.avoid_ads is True
+        assert config.avoid_css is True
+
+    def test_mixed_values(self):
+        c1 = BrowserConfig(avoid_ads=True, avoid_css=False)
+        assert c1.avoid_ads is True
+        assert c1.avoid_css is False
+
+        c2 = BrowserConfig(avoid_ads=False, avoid_css=True)
+        assert c2.avoid_ads is False
+        assert c2.avoid_css is True
+
+
+class TestResourceFilteringSerialization:
+    """Flags must survive to_dict / from_kwargs / dump / load roundtrips."""
+
+    def test_to_dict_includes_flags(self):
+        config = BrowserConfig(avoid_ads=True, avoid_css=True)
+        d = config.to_dict()
+        assert "avoid_ads" in d
+        assert "avoid_css" in d
+        assert d["avoid_ads"] is True
+        assert d["avoid_css"] is True
+
+    def test_to_dict_includes_false_values(self):
+        config = BrowserConfig()
+        d = config.to_dict()
+        assert d["avoid_ads"] is False
+        assert d["avoid_css"] is False
+
+    def test_from_kwargs_roundtrip(self):
+        original = BrowserConfig(avoid_ads=True, avoid_css=False)
+        d = original.to_dict()
+        restored = BrowserConfig.from_kwargs(d)
+        assert restored.avoid_ads is True
+        assert restored.avoid_css is False
+
+    def test_from_kwargs_with_true_values(self):
+        restored = BrowserConfig.from_kwargs({"avoid_ads": True, "avoid_css": True})
+        assert restored.avoid_ads is True
+        assert restored.avoid_css is True
+
+    def test_dump_load_roundtrip(self):
+        original = BrowserConfig(avoid_ads=True, avoid_css=True)
+        dumped = original.dump()
+        restored = BrowserConfig.load(dumped)
+        assert restored.avoid_ads is True
+        assert restored.avoid_css is True
+
+
+class TestResourceFilteringClone:
+    """clone() must preserve flags and allow overrides."""
+
+    def test_clone_preserves_flags(self):
+        config = BrowserConfig(avoid_ads=True, avoid_css=True)
+        cloned = config.clone()
+        assert cloned.avoid_ads is True
+        assert cloned.avoid_css is True
+
+    def test_clone_allows_override(self):
+        config = BrowserConfig(avoid_ads=True, avoid_css=False)
+        cloned = config.clone(avoid_css=True)
+        assert cloned.avoid_ads is True
+        assert cloned.avoid_css is True
+        # original unchanged
+        assert config.avoid_css is False
+
+    def test_clone_can_disable_flag(self):
+        config = BrowserConfig(avoid_ads=True, avoid_css=True)
+        cloned = config.clone(avoid_ads=False)
+        assert cloned.avoid_ads is False
+        assert cloned.avoid_css is True
diff --git a/tests/unit/test_sitemap_namespace_parsing.py b/tests/unit/test_sitemap_namespace_parsing.py
new file mode 100644
index 0000000..3370ddb
--- /dev/null
+++ b/tests/unit/test_sitemap_namespace_parsing.py
@@ -0,0 +1,134 @@
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+# Provide a lightweight stub for rank_bm25 before importing the seeder to avoid
+# optional dependency issues (e.g., incompatible wheels in CI).
+class _FakeBM25:
+    def __init__(self, corpus):
+        self._scores = [1.0] * len(corpus)
+
+    def get_scores(self, tokens):
+        return self._scores
+
+
+sys.modules.setdefault("rank_bm25", SimpleNamespace(BM25Okapi=_FakeBM25))
+
+from crawl4ai.async_url_seeder import AsyncUrlSeeder
+
+
+class DummyResponse:
+    def __init__(self, request_url: str, text: str):
+        self.status_code = 200
+        self._content = text.encode("utf-8")
+        self.url = request_url
+
+    def raise_for_status(self):
+        return None
+
+    @property
+    def content(self):
+        return self._content
+
+    @property
+    def text(self):
+        return self._content.decode("utf-8")
+
+
+class DummyAsyncClient:
+    def __init__(self, response_map):
+        self._responses = response_map
+
+    async def get(self, url, **kwargs):
+        payload = self._responses[url]
+        if callable(payload):
+            payload = payload()
+        return DummyResponse(url, payload)
+
+
+@pytest.mark.asyncio
+async def test_iter_sitemap_handles_namespace_less_sitemaps():
+    xml = """
+    
+        https://example.com/a
+        https://example.com/b
+    
+    """
+    seeder = AsyncUrlSeeder(client=DummyAsyncClient({"https://example.com/sitemap.xml": xml}))
+
+    urls = []
+    async for u in seeder._iter_sitemap("https://example.com/sitemap.xml"):
+        urls.append(u)
+
+    assert urls == ["https://example.com/a", "https://example.com/b"]
+
+
+@pytest.mark.asyncio
+async def test_iter_sitemap_handles_custom_namespace():
+    xml = """
+    
+        https://example.com/ns
+    
+    """
+    seeder = AsyncUrlSeeder(client=DummyAsyncClient({"https://example.com/ns-sitemap.xml": xml}))
+
+    urls = []
+    async for u in seeder._iter_sitemap("https://example.com/ns-sitemap.xml"):
+        urls.append(u)
+
+    assert urls == ["https://example.com/ns"]
+
+
+@pytest.mark.asyncio
+async def test_iter_sitemap_handles_namespace_index_and_children():
+    index_xml = """
+    
+        
+            https://example.com/child-1.xml
+        
+        
+            https://example.com/child-2.xml
+        
+    
+    """
+    child_xml = """
+    
+        https://example.com/page-{n}
+    
+    """
+    responses = {
+        "https://example.com/index.xml": index_xml,
+        "https://example.com/child-1.xml": child_xml.format(n=1),
+        "https://example.com/child-2.xml": child_xml.format(n=2),
+    }
+    seeder = AsyncUrlSeeder(client=DummyAsyncClient(responses))
+
+    urls = []
+    async for u in seeder._iter_sitemap("https://example.com/index.xml"):
+        urls.append(u)
+
+    assert sorted(urls) == [
+        "https://example.com/page-1",
+        "https://example.com/page-2",
+    ]
+
+
+@pytest.mark.asyncio
+async def test_iter_sitemap_normalizes_relative_locations():
+    xml = """
+    
+        /relative-path
+        https://example.com/absolute
+    
+    """
+    seeder = AsyncUrlSeeder(client=DummyAsyncClient({"https://example.com/sitemap.xml": xml}))
+
+    urls = []
+    async for u in seeder._iter_sitemap("https://example.com/sitemap.xml"):
+        urls.append(u)
+
+    assert urls == [
+        "https://example.com/relative-path",
+        "https://example.com/absolute",
+    ]
diff --git a/uv.lock b/uv.lock
new file mode 100644
index 0000000..50ce228
--- /dev/null
+++ b/uv.lock
@@ -0,0 +1,3049 @@
+version = 1
+revision = 3
+requires-python = ">=3.10"
+resolution-markers = [
+    "python_full_version >= '3.12'",
+    "python_full_version < '3.12'",
+]
+
+[[package]]
+name = "aiofiles"
+version = "24.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0b/03/a88171e277e8caa88a4c77808c20ebb04ba74cc4681bf1e9416c862de237/aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c", size = 30247, upload-time = "2024-06-24T11:02:03.584Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a5/45/30bb92d442636f570cb5651bc661f52b610e2eec3f891a5dc3a4c3667db0/aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5", size = 15896, upload-time = "2024-06-24T11:02:01.529Z" },
+]
+
+[[package]]
+name = "aiohappyeyeballs"
+version = "2.6.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" },
+]
+
+[[package]]
+name = "aiohttp"
+version = "3.12.13"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "aiohappyeyeballs" },
+    { name = "aiosignal" },
+    { name = "async-timeout", marker = "python_full_version < '3.11'" },
+    { name = "attrs" },
+    { name = "frozenlist" },
+    { name = "multidict" },
+    { name = "propcache" },
+    { name = "yarl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/42/6e/ab88e7cb2a4058bed2f7870276454f85a7c56cd6da79349eb314fc7bbcaa/aiohttp-3.12.13.tar.gz", hash = "sha256:47e2da578528264a12e4e3dd8dd72a7289e5f812758fe086473fab037a10fcce", size = 7819160, upload-time = "2025-06-14T15:15:41.354Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8b/2d/27e4347660723738b01daa3f5769d56170f232bf4695dd4613340da135bb/aiohttp-3.12.13-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5421af8f22a98f640261ee48aae3a37f0c41371e99412d55eaf2f8a46d5dad29", size = 702090, upload-time = "2025-06-14T15:12:58.938Z" },
+    { url = "https://files.pythonhosted.org/packages/10/0b/4a8e0468ee8f2b9aff3c05f2c3a6be1dfc40b03f68a91b31041d798a9510/aiohttp-3.12.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fcda86f6cb318ba36ed8f1396a6a4a3fd8f856f84d426584392083d10da4de0", size = 478440, upload-time = "2025-06-14T15:13:02.981Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/c8/2086df2f9a842b13feb92d071edf756be89250f404f10966b7bc28317f17/aiohttp-3.12.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cd71c9fb92aceb5a23c4c39d8ecc80389c178eba9feab77f19274843eb9412d", size = 466215, upload-time = "2025-06-14T15:13:04.817Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/3d/d23e5bd978bc8012a65853959b13bd3b55c6e5afc172d89c26ad6624c52b/aiohttp-3.12.13-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34ebf1aca12845066c963016655dac897651e1544f22a34c9b461ac3b4b1d3aa", size = 1648271, upload-time = "2025-06-14T15:13:06.532Z" },
+    { url = "https://files.pythonhosted.org/packages/31/31/e00122447bb137591c202786062f26dd383574c9f5157144127077d5733e/aiohttp-3.12.13-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:893a4639694c5b7edd4bdd8141be296042b6806e27cc1d794e585c43010cc294", size = 1622329, upload-time = "2025-06-14T15:13:08.394Z" },
+    { url = "https://files.pythonhosted.org/packages/04/01/caef70be3ac38986969045f21f5fb802ce517b3f371f0615206bf8aa6423/aiohttp-3.12.13-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:663d8ee3ffb3494502ebcccb49078faddbb84c1d870f9c1dd5a29e85d1f747ce", size = 1694734, upload-time = "2025-06-14T15:13:09.979Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/15/328b71fedecf69a9fd2306549b11c8966e420648a3938d75d3ed5bcb47f6/aiohttp-3.12.13-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0f8f6a85a0006ae2709aa4ce05749ba2cdcb4b43d6c21a16c8517c16593aabe", size = 1737049, upload-time = "2025-06-14T15:13:11.672Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/7a/d85866a642158e1147c7da5f93ad66b07e5452a84ec4258e5f06b9071e92/aiohttp-3.12.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1582745eb63df267c92d8b61ca655a0ce62105ef62542c00a74590f306be8cb5", size = 1641715, upload-time = "2025-06-14T15:13:13.548Z" },
+    { url = "https://files.pythonhosted.org/packages/14/57/3588800d5d2f5f3e1cb6e7a72747d1abc1e67ba5048e8b845183259c2e9b/aiohttp-3.12.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d59227776ee2aa64226f7e086638baa645f4b044f2947dbf85c76ab11dcba073", size = 1581836, upload-time = "2025-06-14T15:13:15.086Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/55/c913332899a916d85781aa74572f60fd98127449b156ad9c19e23135b0e4/aiohttp-3.12.13-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:06b07c418bde1c8e737d8fa67741072bd3f5b0fb66cf8c0655172188c17e5fa6", size = 1625685, upload-time = "2025-06-14T15:13:17.163Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/34/26cded195f3bff128d6a6d58d7a0be2ae7d001ea029e0fe9008dcdc6a009/aiohttp-3.12.13-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:9445c1842680efac0f81d272fd8db7163acfcc2b1436e3f420f4c9a9c5a50795", size = 1636471, upload-time = "2025-06-14T15:13:19.086Z" },
+    { url = "https://files.pythonhosted.org/packages/19/21/70629ca006820fccbcec07f3cd5966cbd966e2d853d6da55339af85555b9/aiohttp-3.12.13-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:09c4767af0b0b98c724f5d47f2bf33395c8986995b0a9dab0575ca81a554a8c0", size = 1611923, upload-time = "2025-06-14T15:13:20.997Z" },
+    { url = "https://files.pythonhosted.org/packages/31/80/7fa3f3bebf533aa6ae6508b51ac0de9965e88f9654fa679cc1a29d335a79/aiohttp-3.12.13-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f3854fbde7a465318ad8d3fc5bef8f059e6d0a87e71a0d3360bb56c0bf87b18a", size = 1691511, upload-time = "2025-06-14T15:13:22.54Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/7a/359974653a3cdd3e9cee8ca10072a662c3c0eb46a359c6a1f667b0296e2f/aiohttp-3.12.13-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2332b4c361c05ecd381edb99e2a33733f3db906739a83a483974b3df70a51b40", size = 1714751, upload-time = "2025-06-14T15:13:24.366Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/24/0aa03d522171ce19064347afeefadb008be31ace0bbb7d44ceb055700a14/aiohttp-3.12.13-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1561db63fa1b658cd94325d303933553ea7d89ae09ff21cc3bcd41b8521fbbb6", size = 1643090, upload-time = "2025-06-14T15:13:26.231Z" },
+    { url = "https://files.pythonhosted.org/packages/86/2e/7d4b0026a41e4b467e143221c51b279083b7044a4b104054f5c6464082ff/aiohttp-3.12.13-cp310-cp310-win32.whl", hash = "sha256:a0be857f0b35177ba09d7c472825d1b711d11c6d0e8a2052804e3b93166de1ad", size = 427526, upload-time = "2025-06-14T15:13:27.988Z" },
+    { url = "https://files.pythonhosted.org/packages/17/de/34d998da1e7f0de86382160d039131e9b0af1962eebfe53dda2b61d250e7/aiohttp-3.12.13-cp310-cp310-win_amd64.whl", hash = "sha256:fcc30ad4fb5cb41a33953292d45f54ef4066746d625992aeac33b8c681173178", size = 450734, upload-time = "2025-06-14T15:13:29.394Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/65/5566b49553bf20ffed6041c665a5504fb047cefdef1b701407b8ce1a47c4/aiohttp-3.12.13-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7c229b1437aa2576b99384e4be668af1db84b31a45305d02f61f5497cfa6f60c", size = 709401, upload-time = "2025-06-14T15:13:30.774Z" },
+    { url = "https://files.pythonhosted.org/packages/14/b5/48e4cc61b54850bdfafa8fe0b641ab35ad53d8e5a65ab22b310e0902fa42/aiohttp-3.12.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04076d8c63471e51e3689c93940775dc3d12d855c0c80d18ac5a1c68f0904358", size = 481669, upload-time = "2025-06-14T15:13:32.316Z" },
+    { url = "https://files.pythonhosted.org/packages/04/4f/e3f95c8b2a20a0437d51d41d5ccc4a02970d8ad59352efb43ea2841bd08e/aiohttp-3.12.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55683615813ce3601640cfaa1041174dc956d28ba0511c8cbd75273eb0587014", size = 469933, upload-time = "2025-06-14T15:13:34.104Z" },
+    { url = "https://files.pythonhosted.org/packages/41/c9/c5269f3b6453b1cfbd2cfbb6a777d718c5f086a3727f576c51a468b03ae2/aiohttp-3.12.13-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:921bc91e602d7506d37643e77819cb0b840d4ebb5f8d6408423af3d3bf79a7b7", size = 1740128, upload-time = "2025-06-14T15:13:35.604Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/49/a3f76caa62773d33d0cfaa842bdf5789a78749dbfe697df38ab1badff369/aiohttp-3.12.13-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e72d17fe0974ddeae8ed86db297e23dba39c7ac36d84acdbb53df2e18505a013", size = 1688796, upload-time = "2025-06-14T15:13:37.125Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/e4/556fccc4576dc22bf18554b64cc873b1a3e5429a5bdb7bbef7f5d0bc7664/aiohttp-3.12.13-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0653d15587909a52e024a261943cf1c5bdc69acb71f411b0dd5966d065a51a47", size = 1787589, upload-time = "2025-06-14T15:13:38.745Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/3d/d81b13ed48e1a46734f848e26d55a7391708421a80336e341d2aef3b6db2/aiohttp-3.12.13-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a77b48997c66722c65e157c06c74332cdf9c7ad00494b85ec43f324e5c5a9b9a", size = 1826635, upload-time = "2025-06-14T15:13:40.733Z" },
+    { url = "https://files.pythonhosted.org/packages/75/a5/472e25f347da88459188cdaadd1f108f6292f8a25e62d226e63f860486d1/aiohttp-3.12.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6946bae55fd36cfb8e4092c921075cde029c71c7cb571d72f1079d1e4e013bc", size = 1729095, upload-time = "2025-06-14T15:13:42.312Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/fe/322a78b9ac1725bfc59dfc301a5342e73d817592828e4445bd8f4ff83489/aiohttp-3.12.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f95db8c8b219bcf294a53742c7bda49b80ceb9d577c8e7aa075612b7f39ffb7", size = 1666170, upload-time = "2025-06-14T15:13:44.884Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/77/ec80912270e231d5e3839dbd6c065472b9920a159ec8a1895cf868c2708e/aiohttp-3.12.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:03d5eb3cfb4949ab4c74822fb3326cd9655c2b9fe22e4257e2100d44215b2e2b", size = 1714444, upload-time = "2025-06-14T15:13:46.401Z" },
+    { url = "https://files.pythonhosted.org/packages/21/b2/fb5aedbcb2b58d4180e58500e7c23ff8593258c27c089abfbcc7db65bd40/aiohttp-3.12.13-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:6383dd0ffa15515283c26cbf41ac8e6705aab54b4cbb77bdb8935a713a89bee9", size = 1709604, upload-time = "2025-06-14T15:13:48.377Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/15/a94c05f7c4dc8904f80b6001ad6e07e035c58a8ebfcc15e6b5d58500c858/aiohttp-3.12.13-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6548a411bc8219b45ba2577716493aa63b12803d1e5dc70508c539d0db8dbf5a", size = 1689786, upload-time = "2025-06-14T15:13:50.401Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/fd/0d2e618388f7a7a4441eed578b626bda9ec6b5361cd2954cfc5ab39aa170/aiohttp-3.12.13-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:81b0fcbfe59a4ca41dc8f635c2a4a71e63f75168cc91026c61be665945739e2d", size = 1783389, upload-time = "2025-06-14T15:13:51.945Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/6b/6986d0c75996ef7e64ff7619b9b7449b1d1cbbe05c6755e65d92f1784fe9/aiohttp-3.12.13-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6a83797a0174e7995e5edce9dcecc517c642eb43bc3cba296d4512edf346eee2", size = 1803853, upload-time = "2025-06-14T15:13:53.533Z" },
+    { url = "https://files.pythonhosted.org/packages/21/65/cd37b38f6655d95dd07d496b6d2f3924f579c43fd64b0e32b547b9c24df5/aiohttp-3.12.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a5734d8469a5633a4e9ffdf9983ff7cdb512524645c7a3d4bc8a3de45b935ac3", size = 1716909, upload-time = "2025-06-14T15:13:55.148Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/20/2de7012427dc116714c38ca564467f6143aec3d5eca3768848d62aa43e62/aiohttp-3.12.13-cp311-cp311-win32.whl", hash = "sha256:fef8d50dfa482925bb6b4c208b40d8e9fa54cecba923dc65b825a72eed9a5dbd", size = 427036, upload-time = "2025-06-14T15:13:57.076Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/b6/98518bcc615ef998a64bef371178b9afc98ee25895b4f476c428fade2220/aiohttp-3.12.13-cp311-cp311-win_amd64.whl", hash = "sha256:9a27da9c3b5ed9d04c36ad2df65b38a96a37e9cfba6f1381b842d05d98e6afe9", size = 451427, upload-time = "2025-06-14T15:13:58.505Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/6a/ce40e329788013cd190b1d62bbabb2b6a9673ecb6d836298635b939562ef/aiohttp-3.12.13-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0aa580cf80558557285b49452151b9c69f2fa3ad94c5c9e76e684719a8791b73", size = 700491, upload-time = "2025-06-14T15:14:00.048Z" },
+    { url = "https://files.pythonhosted.org/packages/28/d9/7150d5cf9163e05081f1c5c64a0cdf3c32d2f56e2ac95db2a28fe90eca69/aiohttp-3.12.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b103a7e414b57e6939cc4dece8e282cfb22043efd0c7298044f6594cf83ab347", size = 475104, upload-time = "2025-06-14T15:14:01.691Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/91/d42ba4aed039ce6e449b3e2db694328756c152a79804e64e3da5bc19dffc/aiohttp-3.12.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78f64e748e9e741d2eccff9597d09fb3cd962210e5b5716047cbb646dc8fe06f", size = 467948, upload-time = "2025-06-14T15:14:03.561Z" },
+    { url = "https://files.pythonhosted.org/packages/99/3b/06f0a632775946981d7c4e5a865cddb6e8dfdbaed2f56f9ade7bb4a1039b/aiohttp-3.12.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c955989bf4c696d2ededc6b0ccb85a73623ae6e112439398935362bacfaaf6", size = 1714742, upload-time = "2025-06-14T15:14:05.558Z" },
+    { url = "https://files.pythonhosted.org/packages/92/a6/2552eebad9ec5e3581a89256276009e6a974dc0793632796af144df8b740/aiohttp-3.12.13-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d640191016763fab76072c87d8854a19e8e65d7a6fcfcbf017926bdbbb30a7e5", size = 1697393, upload-time = "2025-06-14T15:14:07.194Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/9f/bd08fdde114b3fec7a021381b537b21920cdd2aa29ad48c5dffd8ee314f1/aiohttp-3.12.13-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dc507481266b410dede95dd9f26c8d6f5a14315372cc48a6e43eac652237d9b", size = 1752486, upload-time = "2025-06-14T15:14:08.808Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/e1/affdea8723aec5bd0959171b5490dccd9a91fcc505c8c26c9f1dca73474d/aiohttp-3.12.13-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8a94daa873465d518db073bd95d75f14302e0208a08e8c942b2f3f1c07288a75", size = 1798643, upload-time = "2025-06-14T15:14:10.767Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/9d/666d856cc3af3a62ae86393baa3074cc1d591a47d89dc3bf16f6eb2c8d32/aiohttp-3.12.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f52420cde4ce0bb9425a375d95577fe082cb5721ecb61da3049b55189e4e6", size = 1718082, upload-time = "2025-06-14T15:14:12.38Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/ce/3c185293843d17be063dada45efd2712bb6bf6370b37104b4eda908ffdbd/aiohttp-3.12.13-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f7df1f620ec40f1a7fbcb99ea17d7326ea6996715e78f71a1c9a021e31b96b8", size = 1633884, upload-time = "2025-06-14T15:14:14.415Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/5b/f3413f4b238113be35dfd6794e65029250d4b93caa0974ca572217745bdb/aiohttp-3.12.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3062d4ad53b36e17796dce1c0d6da0ad27a015c321e663657ba1cc7659cfc710", size = 1694943, upload-time = "2025-06-14T15:14:16.48Z" },
+    { url = "https://files.pythonhosted.org/packages/82/c8/0e56e8bf12081faca85d14a6929ad5c1263c146149cd66caa7bc12255b6d/aiohttp-3.12.13-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:8605e22d2a86b8e51ffb5253d9045ea73683d92d47c0b1438e11a359bdb94462", size = 1716398, upload-time = "2025-06-14T15:14:18.589Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/f3/33192b4761f7f9b2f7f4281365d925d663629cfaea093a64b658b94fc8e1/aiohttp-3.12.13-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:54fbbe6beafc2820de71ece2198458a711e224e116efefa01b7969f3e2b3ddae", size = 1657051, upload-time = "2025-06-14T15:14:20.223Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/0b/26ddd91ca8f84c48452431cb4c5dd9523b13bc0c9766bda468e072ac9e29/aiohttp-3.12.13-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:050bd277dfc3768b606fd4eae79dd58ceda67d8b0b3c565656a89ae34525d15e", size = 1736611, upload-time = "2025-06-14T15:14:21.988Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/8d/e04569aae853302648e2c138a680a6a2f02e374c5b6711732b29f1e129cc/aiohttp-3.12.13-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2637a60910b58f50f22379b6797466c3aa6ae28a6ab6404e09175ce4955b4e6a", size = 1764586, upload-time = "2025-06-14T15:14:23.979Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/98/c193c1d1198571d988454e4ed75adc21c55af247a9fda08236602921c8c8/aiohttp-3.12.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e986067357550d1aaa21cfe9897fa19e680110551518a5a7cf44e6c5638cb8b5", size = 1724197, upload-time = "2025-06-14T15:14:25.692Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/9e/07bb8aa11eec762c6b1ff61575eeeb2657df11ab3d3abfa528d95f3e9337/aiohttp-3.12.13-cp312-cp312-win32.whl", hash = "sha256:ac941a80aeea2aaae2875c9500861a3ba356f9ff17b9cb2dbfb5cbf91baaf5bf", size = 421771, upload-time = "2025-06-14T15:14:27.364Z" },
+    { url = "https://files.pythonhosted.org/packages/52/66/3ce877e56ec0813069cdc9607cd979575859c597b6fb9b4182c6d5f31886/aiohttp-3.12.13-cp312-cp312-win_amd64.whl", hash = "sha256:671f41e6146a749b6c81cb7fd07f5a8356d46febdaaaf07b0e774ff04830461e", size = 447869, upload-time = "2025-06-14T15:14:29.05Z" },
+    { url = "https://files.pythonhosted.org/packages/11/0f/db19abdf2d86aa1deec3c1e0e5ea46a587b97c07a16516b6438428b3a3f8/aiohttp-3.12.13-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d4a18e61f271127465bdb0e8ff36e8f02ac4a32a80d8927aa52371e93cd87938", size = 694910, upload-time = "2025-06-14T15:14:30.604Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/81/0ab551e1b5d7f1339e2d6eb482456ccbe9025605b28eed2b1c0203aaaade/aiohttp-3.12.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:532542cb48691179455fab429cdb0d558b5e5290b033b87478f2aa6af5d20ace", size = 472566, upload-time = "2025-06-14T15:14:32.275Z" },
+    { url = "https://files.pythonhosted.org/packages/34/3f/6b7d336663337672d29b1f82d1f252ec1a040fe2d548f709d3f90fa2218a/aiohttp-3.12.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d7eea18b52f23c050ae9db5d01f3d264ab08f09e7356d6f68e3f3ac2de9dfabb", size = 464856, upload-time = "2025-06-14T15:14:34.132Z" },
+    { url = "https://files.pythonhosted.org/packages/26/7f/32ca0f170496aa2ab9b812630fac0c2372c531b797e1deb3deb4cea904bd/aiohttp-3.12.13-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad7c8e5c25f2a26842a7c239de3f7b6bfb92304593ef997c04ac49fb703ff4d7", size = 1703683, upload-time = "2025-06-14T15:14:36.034Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/53/d5513624b33a811c0abea8461e30a732294112318276ce3dbf047dbd9d8b/aiohttp-3.12.13-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6af355b483e3fe9d7336d84539fef460120c2f6e50e06c658fe2907c69262d6b", size = 1684946, upload-time = "2025-06-14T15:14:38Z" },
+    { url = "https://files.pythonhosted.org/packages/37/72/4c237dd127827b0247dc138d3ebd49c2ded6114c6991bbe969058575f25f/aiohttp-3.12.13-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a95cf9f097498f35c88e3609f55bb47b28a5ef67f6888f4390b3d73e2bac6177", size = 1737017, upload-time = "2025-06-14T15:14:39.951Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/67/8a7eb3afa01e9d0acc26e1ef847c1a9111f8b42b82955fcd9faeb84edeb4/aiohttp-3.12.13-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8ed8c38a1c584fe99a475a8f60eefc0b682ea413a84c6ce769bb19a7ff1c5ef", size = 1786390, upload-time = "2025-06-14T15:14:42.151Z" },
+    { url = "https://files.pythonhosted.org/packages/48/19/0377df97dd0176ad23cd8cad4fd4232cfeadcec6c1b7f036315305c98e3f/aiohttp-3.12.13-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0b9170d5d800126b5bc89d3053a2363406d6e327afb6afaeda2d19ee8bb103", size = 1708719, upload-time = "2025-06-14T15:14:44.039Z" },
+    { url = "https://files.pythonhosted.org/packages/61/97/ade1982a5c642b45f3622255173e40c3eed289c169f89d00eeac29a89906/aiohttp-3.12.13-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:372feeace612ef8eb41f05ae014a92121a512bd5067db8f25101dd88a8db11da", size = 1622424, upload-time = "2025-06-14T15:14:45.945Z" },
+    { url = "https://files.pythonhosted.org/packages/99/ab/00ad3eea004e1d07ccc406e44cfe2b8da5acb72f8c66aeeb11a096798868/aiohttp-3.12.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a946d3702f7965d81f7af7ea8fb03bb33fe53d311df48a46eeca17e9e0beed2d", size = 1675447, upload-time = "2025-06-14T15:14:47.911Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/fe/74e5ce8b2ccaba445fe0087abc201bfd7259431d92ae608f684fcac5d143/aiohttp-3.12.13-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a0c4725fae86555bbb1d4082129e21de7264f4ab14baf735278c974785cd2041", size = 1707110, upload-time = "2025-06-14T15:14:50.334Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/c4/39af17807f694f7a267bd8ab1fbacf16ad66740862192a6c8abac2bff813/aiohttp-3.12.13-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:9b28ea2f708234f0a5c44eb6c7d9eb63a148ce3252ba0140d050b091b6e842d1", size = 1649706, upload-time = "2025-06-14T15:14:52.378Z" },
+    { url = "https://files.pythonhosted.org/packages/38/e8/f5a0a5f44f19f171d8477059aa5f28a158d7d57fe1a46c553e231f698435/aiohttp-3.12.13-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d4f5becd2a5791829f79608c6f3dc745388162376f310eb9c142c985f9441cc1", size = 1725839, upload-time = "2025-06-14T15:14:54.617Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/ac/81acc594c7f529ef4419d3866913f628cd4fa9cab17f7bf410a5c3c04c53/aiohttp-3.12.13-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:60f2ce6b944e97649051d5f5cc0f439360690b73909230e107fd45a359d3e911", size = 1759311, upload-time = "2025-06-14T15:14:56.597Z" },
+    { url = "https://files.pythonhosted.org/packages/38/0d/aabe636bd25c6ab7b18825e5a97d40024da75152bec39aa6ac8b7a677630/aiohttp-3.12.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69fc1909857401b67bf599c793f2183fbc4804717388b0b888f27f9929aa41f3", size = 1708202, upload-time = "2025-06-14T15:14:58.598Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/ab/561ef2d8a223261683fb95a6283ad0d36cb66c87503f3a7dde7afe208bb2/aiohttp-3.12.13-cp313-cp313-win32.whl", hash = "sha256:7d7e68787a2046b0e44ba5587aa723ce05d711e3a3665b6b7545328ac8e3c0dd", size = 420794, upload-time = "2025-06-14T15:15:00.939Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/47/b11d0089875a23bff0abd3edb5516bcd454db3fefab8604f5e4b07bd6210/aiohttp-3.12.13-cp313-cp313-win_amd64.whl", hash = "sha256:5a178390ca90419bfd41419a809688c368e63c86bd725e1186dd97f6b89c2706", size = 446735, upload-time = "2025-06-14T15:15:02.858Z" },
+]
+
+[[package]]
+name = "aiosignal"
+version = "1.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "frozenlist" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" },
+]
+
+[[package]]
+name = "aiosqlite"
+version = "0.21.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/13/7d/8bca2bf9a247c2c5dfeec1d7a5f40db6518f88d314b8bca9da29670d2671/aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3", size = 13454, upload-time = "2025-02-03T07:30:16.235Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" },
+]
+
+[[package]]
+name = "alphashape"
+version = "1.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "click-log" },
+    { name = "networkx" },
+    { name = "numpy" },
+    { name = "rtree" },
+    { name = "scipy" },
+    { name = "shapely" },
+    { name = "trimesh" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2e/83/67ff905694df5b34a777123b59fdfd05998d5a31766f188aafbf5b340055/alphashape-1.3.1.tar.gz", hash = "sha256:7a27340afc5f8ed301577acec46bb0cf2bada5410045f7289142e735ef6977ec", size = 26316, upload-time = "2021-04-16T17:47:19.486Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e4/ad/77fad9d6f974ec58d837cb49fb9b483d6227a420c4f908c3578633de1d47/alphashape-1.3.1-py2.py3-none-any.whl", hash = "sha256:96a5ddd5f09534a35f03a8916aeeaac00fe4d6bec2f9ad78f87f57be3007f795", size = 13122, upload-time = "2021-04-16T17:47:17.773Z" },
+]
+
+[[package]]
+name = "annotated-types"
+version = "0.7.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" },
+]
+
+[[package]]
+name = "anyio"
+version = "4.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
+    { name = "idna" },
+    { name = "sniffio" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" },
+]
+
+[[package]]
+name = "async-timeout"
+version = "5.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" },
+]
+
+[[package]]
+name = "attrs"
+version = "25.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" },
+]
+
+[[package]]
+name = "beautifulsoup4"
+version = "4.13.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "soupsieve" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d8/e4/0c4c39e18fd76d6a628d4dd8da40543d136ce2d1752bd6eeeab0791f4d6b/beautifulsoup4-4.13.4.tar.gz", hash = "sha256:dbb3c4e1ceae6aefebdaf2423247260cd062430a410e38c66f2baa50a8437195", size = 621067, upload-time = "2025-04-15T17:05:13.836Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/50/cd/30110dc0ffcf3b131156077b90e9f60ed75711223f306da4db08eff8403b/beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b", size = 187285, upload-time = "2025-04-15T17:05:12.221Z" },
+]
+
+[[package]]
+name = "brotli"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270, upload-time = "2023-09-07T14:05:41.643Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6d/3a/dbf4fb970c1019a57b5e492e1e0eae745d32e59ba4d6161ab5422b08eefe/Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752", size = 873045, upload-time = "2023-09-07T14:03:16.894Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/11/afc14026ea7f44bd6eb9316d800d439d092c8d508752055ce8d03086079a/Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9", size = 446218, upload-time = "2023-09-07T14:03:18.917Z" },
+    { url = "https://files.pythonhosted.org/packages/36/83/7545a6e7729db43cb36c4287ae388d6885c85a86dd251768a47015dfde32/Brotli-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3", size = 2903872, upload-time = "2023-09-07T14:03:20.398Z" },
+    { url = "https://files.pythonhosted.org/packages/32/23/35331c4d9391fcc0f29fd9bec2c76e4b4eeab769afbc4b11dd2e1098fb13/Brotli-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d", size = 2941254, upload-time = "2023-09-07T14:03:21.914Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/24/1671acb450c902edb64bd765d73603797c6c7280a9ada85a195f6b78c6e5/Brotli-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e", size = 2857293, upload-time = "2023-09-07T14:03:24Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/00/40f760cc27007912b327fe15bf6bfd8eaecbe451687f72a8abc587d503b3/Brotli-1.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da", size = 3002385, upload-time = "2023-09-07T14:03:26.248Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/cb/8aaa83f7a4caa131757668c0fb0c4b6384b09ffa77f2fba9570d87ab587d/Brotli-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80", size = 2911104, upload-time = "2023-09-07T14:03:27.849Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/c4/65456561d89d3c49f46b7fbeb8fe6e449f13bdc8ea7791832c5d476b2faf/Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d", size = 2809981, upload-time = "2023-09-07T14:03:29.92Z" },
+    { url = "https://files.pythonhosted.org/packages/05/1b/cf49528437bae28abce5f6e059f0d0be6fecdcc1d3e33e7c54b3ca498425/Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0", size = 2935297, upload-time = "2023-09-07T14:03:32.035Z" },
+    { url = "https://files.pythonhosted.org/packages/81/ff/190d4af610680bf0c5a09eb5d1eac6e99c7c8e216440f9c7cfd42b7adab5/Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e", size = 2930735, upload-time = "2023-09-07T14:03:33.801Z" },
+    { url = "https://files.pythonhosted.org/packages/80/7d/f1abbc0c98f6e09abd3cad63ec34af17abc4c44f308a7a539010f79aae7a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c", size = 2933107, upload-time = "2024-10-18T12:32:09.016Z" },
+    { url = "https://files.pythonhosted.org/packages/34/ce/5a5020ba48f2b5a4ad1c0522d095ad5847a0be508e7d7569c8630ce25062/Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1", size = 2845400, upload-time = "2024-10-18T12:32:11.134Z" },
+    { url = "https://files.pythonhosted.org/packages/44/89/fa2c4355ab1eecf3994e5a0a7f5492c6ff81dfcb5f9ba7859bd534bb5c1a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2", size = 3031985, upload-time = "2024-10-18T12:32:12.813Z" },
+    { url = "https://files.pythonhosted.org/packages/af/a4/79196b4a1674143d19dca400866b1a4d1a089040df7b93b88ebae81f3447/Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec", size = 2927099, upload-time = "2024-10-18T12:32:14.733Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/54/1c0278556a097f9651e657b873ab08f01b9a9ae4cac128ceb66427d7cd20/Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2", size = 333172, upload-time = "2023-09-07T14:03:35.212Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/65/b785722e941193fd8b571afd9edbec2a9b838ddec4375d8af33a50b8dab9/Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128", size = 357255, upload-time = "2023-09-07T14:03:36.447Z" },
+    { url = "https://files.pythonhosted.org/packages/96/12/ad41e7fadd5db55459c4c401842b47f7fee51068f86dd2894dd0dcfc2d2a/Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc", size = 873068, upload-time = "2023-09-07T14:03:37.779Z" },
+    { url = "https://files.pythonhosted.org/packages/95/4e/5afab7b2b4b61a84e9c75b17814198ce515343a44e2ed4488fac314cd0a9/Brotli-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6", size = 446244, upload-time = "2023-09-07T14:03:39.223Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/e6/f305eb61fb9a8580c525478a4a34c5ae1a9bcb12c3aee619114940bc513d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd", size = 2906500, upload-time = "2023-09-07T14:03:40.858Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/4f/af6846cfbc1550a3024e5d3775ede1e00474c40882c7bf5b37a43ca35e91/Brotli-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf", size = 2943950, upload-time = "2023-09-07T14:03:42.896Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/e7/ca2993c7682d8629b62630ebf0d1f3bb3d579e667ce8e7ca03a0a0576a2d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61", size = 2918527, upload-time = "2023-09-07T14:03:44.552Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/96/da98e7bedc4c51104d29cc61e5f449a502dd3dbc211944546a4cc65500d3/Brotli-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327", size = 2845489, upload-time = "2023-09-07T14:03:46.594Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/ef/ccbc16947d6ce943a7f57e1a40596c75859eeb6d279c6994eddd69615265/Brotli-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd", size = 2914080, upload-time = "2023-09-07T14:03:48.204Z" },
+    { url = "https://files.pythonhosted.org/packages/80/d6/0bd38d758d1afa62a5524172f0b18626bb2392d717ff94806f741fcd5ee9/Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9", size = 2813051, upload-time = "2023-09-07T14:03:50.348Z" },
+    { url = "https://files.pythonhosted.org/packages/14/56/48859dd5d129d7519e001f06dcfbb6e2cf6db92b2702c0c2ce7d97e086c1/Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265", size = 2938172, upload-time = "2023-09-07T14:03:52.395Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/77/a236d5f8cd9e9f4348da5acc75ab032ab1ab2c03cc8f430d24eea2672888/Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8", size = 2933023, upload-time = "2023-09-07T14:03:53.96Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/87/3b283efc0f5cb35f7f84c0c240b1e1a1003a5e47141a4881bf87c86d0ce2/Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f", size = 2935871, upload-time = "2024-10-18T12:32:16.688Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/eb/2be4cc3e2141dc1a43ad4ca1875a72088229de38c68e842746b342667b2a/Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757", size = 2847784, upload-time = "2024-10-18T12:32:18.459Z" },
+    { url = "https://files.pythonhosted.org/packages/66/13/b58ddebfd35edde572ccefe6890cf7c493f0c319aad2a5badee134b4d8ec/Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0", size = 3034905, upload-time = "2024-10-18T12:32:20.192Z" },
+    { url = "https://files.pythonhosted.org/packages/84/9c/bc96b6c7db824998a49ed3b38e441a2cae9234da6fa11f6ed17e8cf4f147/Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b", size = 2929467, upload-time = "2024-10-18T12:32:21.774Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/71/8f161dee223c7ff7fea9d44893fba953ce97cf2c3c33f78ba260a91bcff5/Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50", size = 333169, upload-time = "2023-09-07T14:03:55.404Z" },
+    { url = "https://files.pythonhosted.org/packages/02/8a/fece0ee1057643cb2a5bbf59682de13f1725f8482b2c057d4e799d7ade75/Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1", size = 357253, upload-time = "2023-09-07T14:03:56.643Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693, upload-time = "2024-10-18T12:32:23.824Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489, upload-time = "2024-10-18T12:32:25.641Z" },
+    { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081, upload-time = "2023-09-07T14:03:57.967Z" },
+    { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244, upload-time = "2023-09-07T14:03:59.319Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505, upload-time = "2023-09-07T14:04:01.327Z" },
+    { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152, upload-time = "2023-09-07T14:04:03.033Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252, upload-time = "2023-09-07T14:04:04.675Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955, upload-time = "2023-09-07T14:04:06.585Z" },
+    { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304, upload-time = "2023-09-07T14:04:08.668Z" },
+    { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452, upload-time = "2023-09-07T14:04:10.736Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751, upload-time = "2023-09-07T14:04:12.875Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757, upload-time = "2023-09-07T14:04:14.551Z" },
+    { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146, upload-time = "2024-10-18T12:32:27.257Z" },
+    { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055, upload-time = "2024-10-18T12:32:29.376Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102, upload-time = "2024-10-18T12:32:31.371Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029, upload-time = "2024-10-18T12:32:33.293Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276, upload-time = "2023-09-07T14:04:16.49Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255, upload-time = "2023-09-07T14:04:17.83Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/9f/fb37bb8ffc52a8da37b1c03c459a8cd55df7a57bdccd8831d500e994a0ca/Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5", size = 815681, upload-time = "2024-10-18T12:32:34.942Z" },
+    { url = "https://files.pythonhosted.org/packages/06/b3/dbd332a988586fefb0aa49c779f59f47cae76855c2d00f450364bb574cac/Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8", size = 422475, upload-time = "2024-10-18T12:32:36.485Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/80/6aaddc2f63dbcf2d93c2d204e49c11a9ec93a8c7c63261e2b4bd35198283/Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f", size = 2906173, upload-time = "2024-10-18T12:32:37.978Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/1d/e6ca79c96ff5b641df6097d299347507d39a9604bde8915e76bf026d6c77/Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648", size = 2943803, upload-time = "2024-10-18T12:32:39.606Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/a3/d98d2472e0130b7dd3acdbb7f390d478123dbf62b7d32bda5c830a96116d/Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0", size = 2918946, upload-time = "2024-10-18T12:32:41.679Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/a5/c69e6d272aee3e1423ed005d8915a7eaa0384c7de503da987f2d224d0721/Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089", size = 2845707, upload-time = "2024-10-18T12:32:43.478Z" },
+    { url = "https://files.pythonhosted.org/packages/58/9f/4149d38b52725afa39067350696c09526de0125ebfbaab5acc5af28b42ea/Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368", size = 2936231, upload-time = "2024-10-18T12:32:45.224Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/5a/145de884285611838a16bebfdb060c231c52b8f84dfbe52b852a15780386/Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c", size = 2848157, upload-time = "2024-10-18T12:32:46.894Z" },
+    { url = "https://files.pythonhosted.org/packages/50/ae/408b6bfb8525dadebd3b3dd5b19d631da4f7d46420321db44cd99dcf2f2c/Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284", size = 3035122, upload-time = "2024-10-18T12:32:48.844Z" },
+    { url = "https://files.pythonhosted.org/packages/af/85/a94e5cfaa0ca449d8f91c3d6f78313ebf919a0dbd55a100c711c6e9655bc/Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7", size = 2930206, upload-time = "2024-10-18T12:32:51.198Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/f0/a61d9262cd01351df22e57ad7c34f66794709acab13f34be2675f45bf89d/Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0", size = 333804, upload-time = "2024-10-18T12:32:52.661Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517, upload-time = "2024-10-18T12:32:54.066Z" },
+]
+
+[[package]]
+name = "certifi"
+version = "2025.7.9"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/de/8a/c729b6b60c66a38f590c4e774decc4b2ec7b0576be8f1aa984a53ffa812a/certifi-2025.7.9.tar.gz", hash = "sha256:c1d2ec05395148ee10cf672ffc28cd37ea0ab0d99f9cc74c43e588cbd111b079", size = 160386, upload-time = "2025-07-09T02:13:58.874Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/66/f3/80a3f974c8b535d394ff960a11ac20368e06b736da395b551a49ce950cce/certifi-2025.7.9-py3-none-any.whl", hash = "sha256:d842783a14f8fdd646895ac26f719a061408834473cfc10203f6a575beb15d39", size = 159230, upload-time = "2025-07-09T02:13:57.007Z" },
+]
+
+[[package]]
+name = "cffi"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pycparser", marker = "implementation_name != 'PyPy'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" },
+    { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" },
+    { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" },
+    { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" },
+    { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" },
+    { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" },
+    { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" },
+    { url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" },
+    { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" },
+    { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" },
+    { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" },
+    { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" },
+    { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" },
+    { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" },
+    { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" },
+    { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" },
+    { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" },
+    { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" },
+    { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" },
+    { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" },
+    { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" },
+    { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" },
+    { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" },
+    { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" },
+    { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" },
+]
+
+[[package]]
+name = "chardet"
+version = "5.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618, upload-time = "2023-08-01T19:23:02.662Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" },
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.4.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/95/28/9901804da60055b406e1a1c5ba7aac1276fb77f1dde635aabfc7fd84b8ab/charset_normalizer-3.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941", size = 201818, upload-time = "2025-05-02T08:31:46.725Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/9b/892a8c8af9110935e5adcbb06d9c6fe741b6bb02608c6513983048ba1a18/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd", size = 144649, upload-time = "2025-05-02T08:31:48.889Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/a5/4179abd063ff6414223575e008593861d62abfc22455b5d1a44995b7c101/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6", size = 155045, upload-time = "2025-05-02T08:31:50.757Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/95/bc08c7dfeddd26b4be8c8287b9bb055716f31077c8b0ea1cd09553794665/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d", size = 147356, upload-time = "2025-05-02T08:31:52.634Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/2d/7a5b635aa65284bf3eab7653e8b4151ab420ecbae918d3e359d1947b4d61/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86", size = 149471, upload-time = "2025-05-02T08:31:56.207Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/38/51fc6ac74251fd331a8cfdb7ec57beba8c23fd5493f1050f71c87ef77ed0/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c", size = 151317, upload-time = "2025-05-02T08:31:57.613Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/17/edee1e32215ee6e9e46c3e482645b46575a44a2d72c7dfd49e49f60ce6bf/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0", size = 146368, upload-time = "2025-05-02T08:31:59.468Z" },
+    { url = "https://files.pythonhosted.org/packages/26/2c/ea3e66f2b5f21fd00b2825c94cafb8c326ea6240cd80a91eb09e4a285830/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef", size = 154491, upload-time = "2025-05-02T08:32:01.219Z" },
+    { url = "https://files.pythonhosted.org/packages/52/47/7be7fa972422ad062e909fd62460d45c3ef4c141805b7078dbab15904ff7/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6", size = 157695, upload-time = "2025-05-02T08:32:03.045Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/42/9f02c194da282b2b340f28e5fb60762de1151387a36842a92b533685c61e/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366", size = 154849, upload-time = "2025-05-02T08:32:04.651Z" },
+    { url = "https://files.pythonhosted.org/packages/67/44/89cacd6628f31fb0b63201a618049be4be2a7435a31b55b5eb1c3674547a/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db", size = 150091, upload-time = "2025-05-02T08:32:06.719Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/79/4b8da9f712bc079c0f16b6d67b099b0b8d808c2292c937f267d816ec5ecc/charset_normalizer-3.4.2-cp310-cp310-win32.whl", hash = "sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a", size = 98445, upload-time = "2025-05-02T08:32:08.66Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/d7/96970afb4fb66497a40761cdf7bd4f6fca0fc7bafde3a84f836c1f57a926/charset_normalizer-3.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509", size = 105782, upload-time = "2025-05-02T08:32:10.46Z" },
+    { url = "https://files.pythonhosted.org/packages/05/85/4c40d00dcc6284a1c1ad5de5e0996b06f39d8232f1031cd23c2f5c07ee86/charset_normalizer-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", size = 198794, upload-time = "2025-05-02T08:32:11.945Z" },
+    { url = "https://files.pythonhosted.org/packages/41/d9/7a6c0b9db952598e97e93cbdfcb91bacd89b9b88c7c983250a77c008703c/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", size = 142846, upload-time = "2025-05-02T08:32:13.946Z" },
+    { url = "https://files.pythonhosted.org/packages/66/82/a37989cda2ace7e37f36c1a8ed16c58cf48965a79c2142713244bf945c89/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", size = 153350, upload-time = "2025-05-02T08:32:15.873Z" },
+    { url = "https://files.pythonhosted.org/packages/df/68/a576b31b694d07b53807269d05ec3f6f1093e9545e8607121995ba7a8313/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", size = 145657, upload-time = "2025-05-02T08:32:17.283Z" },
+    { url = "https://files.pythonhosted.org/packages/92/9b/ad67f03d74554bed3aefd56fe836e1623a50780f7c998d00ca128924a499/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f", size = 147260, upload-time = "2025-05-02T08:32:18.807Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/e6/8aebae25e328160b20e31a7e9929b1578bbdc7f42e66f46595a432f8539e/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", size = 149164, upload-time = "2025-05-02T08:32:20.333Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/f2/b3c2f07dbcc248805f10e67a0262c93308cfa149a4cd3d1fe01f593e5fd2/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", size = 144571, upload-time = "2025-05-02T08:32:21.86Z" },
+    { url = "https://files.pythonhosted.org/packages/60/5b/c3f3a94bc345bc211622ea59b4bed9ae63c00920e2e8f11824aa5708e8b7/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", size = 151952, upload-time = "2025-05-02T08:32:23.434Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/4d/ff460c8b474122334c2fa394a3f99a04cf11c646da895f81402ae54f5c42/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", size = 155959, upload-time = "2025-05-02T08:32:24.993Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/2b/b964c6a2fda88611a1fe3d4c400d39c66a42d6c169c924818c848f922415/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", size = 153030, upload-time = "2025-05-02T08:32:26.435Z" },
+    { url = "https://files.pythonhosted.org/packages/59/2e/d3b9811db26a5ebf444bc0fa4f4be5aa6d76fc6e1c0fd537b16c14e849b6/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", size = 148015, upload-time = "2025-05-02T08:32:28.376Z" },
+    { url = "https://files.pythonhosted.org/packages/90/07/c5fd7c11eafd561bb51220d600a788f1c8d77c5eef37ee49454cc5c35575/charset_normalizer-3.4.2-cp311-cp311-win32.whl", hash = "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", size = 98106, upload-time = "2025-05-02T08:32:30.281Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/05/5e33dbef7e2f773d672b6d79f10ec633d4a71cd96db6673625838a4fd532/charset_normalizer-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", size = 105402, upload-time = "2025-05-02T08:32:32.191Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/a4/37f4d6035c89cac7930395a35cc0f1b872e652eaafb76a6075943754f095/charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", size = 199936, upload-time = "2025-05-02T08:32:33.712Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/8a/1a5e33b73e0d9287274f899d967907cd0bf9c343e651755d9307e0dbf2b3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", size = 143790, upload-time = "2025-05-02T08:32:35.768Z" },
+    { url = "https://files.pythonhosted.org/packages/66/52/59521f1d8e6ab1482164fa21409c5ef44da3e9f653c13ba71becdd98dec3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", size = 153924, upload-time = "2025-05-02T08:32:37.284Z" },
+    { url = "https://files.pythonhosted.org/packages/86/2d/fb55fdf41964ec782febbf33cb64be480a6b8f16ded2dbe8db27a405c09f/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", size = 146626, upload-time = "2025-05-02T08:32:38.803Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/73/6ede2ec59bce19b3edf4209d70004253ec5f4e319f9a2e3f2f15601ed5f7/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", size = 148567, upload-time = "2025-05-02T08:32:40.251Z" },
+    { url = "https://files.pythonhosted.org/packages/09/14/957d03c6dc343c04904530b6bef4e5efae5ec7d7990a7cbb868e4595ee30/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", size = 150957, upload-time = "2025-05-02T08:32:41.705Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/c8/8174d0e5c10ccebdcb1b53cc959591c4c722a3ad92461a273e86b9f5a302/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", size = 145408, upload-time = "2025-05-02T08:32:43.709Z" },
+    { url = "https://files.pythonhosted.org/packages/58/aa/8904b84bc8084ac19dc52feb4f5952c6df03ffb460a887b42615ee1382e8/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", size = 153399, upload-time = "2025-05-02T08:32:46.197Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/26/89ee1f0e264d201cb65cf054aca6038c03b1a0c6b4ae998070392a3ce605/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", size = 156815, upload-time = "2025-05-02T08:32:48.105Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/07/68e95b4b345bad3dbbd3a8681737b4338ff2c9df29856a6d6d23ac4c73cb/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", size = 154537, upload-time = "2025-05-02T08:32:49.719Z" },
+    { url = "https://files.pythonhosted.org/packages/77/1a/5eefc0ce04affb98af07bc05f3bac9094513c0e23b0562d64af46a06aae4/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", size = 149565, upload-time = "2025-05-02T08:32:51.404Z" },
+    { url = "https://files.pythonhosted.org/packages/37/a0/2410e5e6032a174c95e0806b1a6585eb21e12f445ebe239fac441995226a/charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", size = 98357, upload-time = "2025-05-02T08:32:53.079Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/4f/c02d5c493967af3eda9c771ad4d2bbc8df6f99ddbeb37ceea6e8716a32bc/charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", size = 105776, upload-time = "2025-05-02T08:32:54.573Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/12/a93df3366ed32db1d907d7593a94f1fe6293903e3e92967bebd6950ed12c/charset_normalizer-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", size = 199622, upload-time = "2025-05-02T08:32:56.363Z" },
+    { url = "https://files.pythonhosted.org/packages/04/93/bf204e6f344c39d9937d3c13c8cd5bbfc266472e51fc8c07cb7f64fcd2de/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", size = 143435, upload-time = "2025-05-02T08:32:58.551Z" },
+    { url = "https://files.pythonhosted.org/packages/22/2a/ea8a2095b0bafa6c5b5a55ffdc2f924455233ee7b91c69b7edfcc9e02284/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", size = 153653, upload-time = "2025-05-02T08:33:00.342Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/57/1b090ff183d13cef485dfbe272e2fe57622a76694061353c59da52c9a659/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1", size = 146231, upload-time = "2025-05-02T08:33:02.081Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/28/ffc026b26f441fc67bd21ab7f03b313ab3fe46714a14b516f931abe1a2d8/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c", size = 148243, upload-time = "2025-05-02T08:33:04.063Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/0f/9abe9bd191629c33e69e47c6ef45ef99773320e9ad8e9cb08b8ab4a8d4cb/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691", size = 150442, upload-time = "2025-05-02T08:33:06.418Z" },
+    { url = "https://files.pythonhosted.org/packages/67/7c/a123bbcedca91d5916c056407f89a7f5e8fdfce12ba825d7d6b9954a1a3c/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0", size = 145147, upload-time = "2025-05-02T08:33:08.183Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/fe/1ac556fa4899d967b83e9893788e86b6af4d83e4726511eaaad035e36595/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b", size = 153057, upload-time = "2025-05-02T08:33:09.986Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/ff/acfc0b0a70b19e3e54febdd5301a98b72fa07635e56f24f60502e954c461/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff", size = 156454, upload-time = "2025-05-02T08:33:11.814Z" },
+    { url = "https://files.pythonhosted.org/packages/92/08/95b458ce9c740d0645feb0e96cea1f5ec946ea9c580a94adfe0b617f3573/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b", size = 154174, upload-time = "2025-05-02T08:33:13.707Z" },
+    { url = "https://files.pythonhosted.org/packages/78/be/8392efc43487ac051eee6c36d5fbd63032d78f7728cb37aebcc98191f1ff/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148", size = 149166, upload-time = "2025-05-02T08:33:15.458Z" },
+    { url = "https://files.pythonhosted.org/packages/44/96/392abd49b094d30b91d9fbda6a69519e95802250b777841cf3bda8fe136c/charset_normalizer-3.4.2-cp313-cp313-win32.whl", hash = "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7", size = 98064, upload-time = "2025-05-02T08:33:17.06Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/b0/0200da600134e001d91851ddc797809e2fe0ea72de90e09bec5a2fbdaccb/charset_normalizer-3.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980", size = 105641, upload-time = "2025-05-02T08:33:18.753Z" },
+    { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" },
+]
+
+[[package]]
+name = "click"
+version = "8.1.8"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" },
+]
+
+[[package]]
+name = "click-log"
+version = "0.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/32/32/228be4f971e4bd556c33d52a22682bfe318ffe57a1ddb7a546f347a90260/click-log-0.4.0.tar.gz", hash = "sha256:3970f8570ac54491237bcdb3d8ab5e3eef6c057df29f8c3d1151a51a9c23b975", size = 9985, upload-time = "2022-03-13T11:10:15.262Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ae/5a/4f025bc751087833686892e17e7564828e409c43b632878afeae554870cd/click_log-0.4.0-py2.py3-none-any.whl", hash = "sha256:a43e394b528d52112af599f2fc9e4b7cf3c15f94e53581f74fa6867e68c91756", size = 4273, upload-time = "2022-03-13T11:10:17.594Z" },
+]
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
+[[package]]
+name = "crawl4ai"
+source = { editable = "." }
+dependencies = [
+    { name = "aiofiles" },
+    { name = "aiohttp" },
+    { name = "aiosqlite" },
+    { name = "alphashape" },
+    { name = "anyio" },
+    { name = "beautifulsoup4" },
+    { name = "brotli" },
+    { name = "chardet" },
+    { name = "click" },
+    { name = "cssselect" },
+    { name = "fake-useragent" },
+    { name = "httpx", extra = ["http2"] },
+    { name = "humanize" },
+    { name = "lark" },
+    { name = "litellm" },
+    { name = "lxml" },
+    { name = "nltk" },
+    { name = "numpy" },
+    { name = "patchright" },
+    { name = "pillow" },
+    { name = "playwright" },
+    { name = "psutil" },
+    { name = "pydantic" },
+    { name = "pyopenssl" },
+    { name = "python-dotenv" },
+    { name = "pyyaml" },
+    { name = "rank-bm25" },
+    { name = "requests" },
+    { name = "rich" },
+    { name = "shapely" },
+    { name = "snowballstemmer" },
+    { name = "tf-playwright-stealth" },
+    { name = "xxhash" },
+]
+
+[package.optional-dependencies]
+all = [
+    { name = "nltk" },
+    { name = "pypdf" },
+    { name = "scikit-learn" },
+    { name = "selenium" },
+    { name = "sentence-transformers" },
+    { name = "tokenizers" },
+    { name = "torch" },
+    { name = "transformers" },
+]
+cosine = [
+    { name = "nltk" },
+    { name = "sentence-transformers" },
+    { name = "torch" },
+    { name = "transformers" },
+]
+pdf = [
+    { name = "pypdf" },
+]
+sync = [
+    { name = "selenium" },
+]
+torch = [
+    { name = "nltk" },
+    { name = "scikit-learn" },
+    { name = "torch" },
+]
+transformer = [
+    { name = "sentence-transformers" },
+    { name = "tokenizers" },
+    { name = "transformers" },
+]
+
+[package.dev-dependencies]
+dev = [
+    { name = "crawl4ai" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "aiofiles", specifier = ">=24.1.0" },
+    { name = "aiohttp", specifier = ">=3.11.11" },
+    { name = "aiosqlite", specifier = "~=0.20" },
+    { name = "alphashape", specifier = ">=1.3.1" },
+    { name = "anyio", specifier = ">=4.0.0" },
+    { name = "beautifulsoup4", specifier = "~=4.12" },
+    { name = "brotli", specifier = ">=1.1.0" },
+    { name = "chardet", specifier = ">=5.2.0" },
+    { name = "click", specifier = ">=8.1.7" },
+    { name = "cssselect", specifier = ">=1.2.0" },
+    { name = "fake-useragent", specifier = ">=2.0.3" },
+    { name = "httpx", specifier = ">=0.27.2" },
+    { name = "httpx", extras = ["http2"], specifier = ">=0.27.2" },
+    { name = "humanize", specifier = ">=4.10.0" },
+    { name = "lark", specifier = ">=1.2.2" },
+    { name = "litellm", specifier = ">=1.53.1" },
+    { name = "lxml", specifier = "~=5.3" },
+    { name = "nltk", specifier = ">=3.9.1" },
+    { name = "nltk", marker = "extra == 'all'" },
+    { name = "nltk", marker = "extra == 'cosine'" },
+    { name = "nltk", marker = "extra == 'torch'" },
+    { name = "numpy", specifier = ">=1.26.0,<3" },
+    { name = "patchright", specifier = ">=1.49.0" },
+    { name = "pillow", specifier = ">=10.4" },
+    { name = "playwright", specifier = ">=1.49.0" },
+    { name = "psutil", specifier = ">=6.1.1" },
+    { name = "pydantic", specifier = ">=2.10" },
+    { name = "pyopenssl", specifier = ">=25.3.0" },
+    { name = "pypdf", marker = "extra == 'all'" },
+    { name = "pypdf", marker = "extra == 'pdf'" },
+    { name = "python-dotenv", specifier = "~=1.0" },
+    { name = "pyyaml", specifier = ">=6.0" },
+    { name = "rank-bm25", specifier = "~=0.2" },
+    { name = "requests", specifier = "~=2.26" },
+    { name = "rich", specifier = ">=13.9.4" },
+    { name = "scikit-learn", marker = "extra == 'all'" },
+    { name = "scikit-learn", marker = "extra == 'torch'" },
+    { name = "selenium", marker = "extra == 'all'" },
+    { name = "selenium", marker = "extra == 'sync'" },
+    { name = "sentence-transformers", marker = "extra == 'all'" },
+    { name = "sentence-transformers", marker = "extra == 'cosine'" },
+    { name = "sentence-transformers", marker = "extra == 'transformer'" },
+    { name = "shapely", specifier = ">=2.0.0" },
+    { name = "snowballstemmer", specifier = "~=2.2" },
+    { name = "tf-playwright-stealth", specifier = ">=1.1.0" },
+    { name = "tokenizers", marker = "extra == 'all'" },
+    { name = "tokenizers", marker = "extra == 'transformer'" },
+    { name = "torch", marker = "extra == 'all'" },
+    { name = "torch", marker = "extra == 'cosine'" },
+    { name = "torch", marker = "extra == 'torch'" },
+    { name = "transformers", marker = "extra == 'all'" },
+    { name = "transformers", marker = "extra == 'cosine'" },
+    { name = "transformers", marker = "extra == 'transformer'" },
+    { name = "xxhash", specifier = "~=3.4" },
+]
+provides-extras = ["pdf", "torch", "transformer", "cosine", "sync", "all"]
+
+[package.metadata.requires-dev]
+dev = [{ name = "crawl4ai", editable = "." }]
+
+[[package]]
+name = "cryptography"
+version = "46.0.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cffi", marker = "platform_python_implementation != 'PyPy'" },
+    { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9f/33/c00162f49c0e2fe8064a62cb92b93e50c74a72bc370ab92f86112b33ff62/cryptography-46.0.3.tar.gz", hash = "sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1", size = 749258, upload-time = "2025-10-15T23:18:31.74Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1d/42/9c391dd801d6cf0d561b5890549d4b27bafcc53b39c31a817e69d87c625b/cryptography-46.0.3-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:109d4ddfadf17e8e7779c39f9b18111a09efb969a301a31e987416a0191ed93a", size = 7225004, upload-time = "2025-10-15T23:16:52.239Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/67/38769ca6b65f07461eb200e85fc1639b438bdc667be02cf7f2cd6a64601c/cryptography-46.0.3-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:09859af8466b69bc3c27bdf4f5d84a665e0f7ab5088412e9e2ec49758eca5cbc", size = 4296667, upload-time = "2025-10-15T23:16:54.369Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/49/498c86566a1d80e978b42f0d702795f69887005548c041636df6ae1ca64c/cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d", size = 4450807, upload-time = "2025-10-15T23:16:56.414Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/0a/863a3604112174c8624a2ac3c038662d9e59970c7f926acdcfaed8d61142/cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb", size = 4299615, upload-time = "2025-10-15T23:16:58.442Z" },
+    { url = "https://files.pythonhosted.org/packages/64/02/b73a533f6b64a69f3cd3872acb6ebc12aef924d8d103133bb3ea750dc703/cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849", size = 4016800, upload-time = "2025-10-15T23:17:00.378Z" },
+    { url = "https://files.pythonhosted.org/packages/25/d5/16e41afbfa450cde85a3b7ec599bebefaef16b5c6ba4ec49a3532336ed72/cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8", size = 4984707, upload-time = "2025-10-15T23:17:01.98Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/56/e7e69b427c3878352c2fb9b450bd0e19ed552753491d39d7d0a2f5226d41/cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec", size = 4482541, upload-time = "2025-10-15T23:17:04.078Z" },
+    { url = "https://files.pythonhosted.org/packages/78/f6/50736d40d97e8483172f1bb6e698895b92a223dba513b0ca6f06b2365339/cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91", size = 4299464, upload-time = "2025-10-15T23:17:05.483Z" },
+    { url = "https://files.pythonhosted.org/packages/00/de/d8e26b1a855f19d9994a19c702fa2e93b0456beccbcfe437eda00e0701f2/cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e", size = 4950838, upload-time = "2025-10-15T23:17:07.425Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/29/798fc4ec461a1c9e9f735f2fc58741b0daae30688f41b2497dcbc9ed1355/cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926", size = 4481596, upload-time = "2025-10-15T23:17:09.343Z" },
+    { url = "https://files.pythonhosted.org/packages/15/8d/03cd48b20a573adfff7652b76271078e3045b9f49387920e7f1f631d125e/cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71", size = 4426782, upload-time = "2025-10-15T23:17:11.22Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/b1/ebacbfe53317d55cf33165bda24c86523497a6881f339f9aae5c2e13e57b/cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac", size = 4698381, upload-time = "2025-10-15T23:17:12.829Z" },
+    { url = "https://files.pythonhosted.org/packages/96/92/8a6a9525893325fc057a01f654d7efc2c64b9de90413adcf605a85744ff4/cryptography-46.0.3-cp311-abi3-win32.whl", hash = "sha256:f260d0d41e9b4da1ed1e0f1ce571f97fe370b152ab18778e9e8f67d6af432018", size = 3055988, upload-time = "2025-10-15T23:17:14.65Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/bf/80fbf45253ea585a1e492a6a17efcb93467701fa79e71550a430c5e60df0/cryptography-46.0.3-cp311-abi3-win_amd64.whl", hash = "sha256:a9a3008438615669153eb86b26b61e09993921ebdd75385ddd748702c5adfddb", size = 3514451, upload-time = "2025-10-15T23:17:16.142Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/af/9b302da4c87b0beb9db4e756386a7c6c5b8003cd0e742277888d352ae91d/cryptography-46.0.3-cp311-abi3-win_arm64.whl", hash = "sha256:5d7f93296ee28f68447397bf5198428c9aeeab45705a55d53a6343455dcb2c3c", size = 2928007, upload-time = "2025-10-15T23:17:18.04Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/e2/a510aa736755bffa9d2f75029c229111a1d02f8ecd5de03078f4c18d91a3/cryptography-46.0.3-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:00a5e7e87938e5ff9ff5447ab086a5706a957137e6e433841e9d24f38a065217", size = 7158012, upload-time = "2025-10-15T23:17:19.982Z" },
+    { url = "https://files.pythonhosted.org/packages/73/dc/9aa866fbdbb95b02e7f9d086f1fccfeebf8953509b87e3f28fff927ff8a0/cryptography-46.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c8daeb2d2174beb4575b77482320303f3d39b8e81153da4f0fb08eb5fe86a6c5", size = 4288728, upload-time = "2025-10-15T23:17:21.527Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/fd/bc1daf8230eaa075184cbbf5f8cd00ba9db4fd32d63fb83da4671b72ed8a/cryptography-46.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715", size = 4435078, upload-time = "2025-10-15T23:17:23.042Z" },
+    { url = "https://files.pythonhosted.org/packages/82/98/d3bd5407ce4c60017f8ff9e63ffee4200ab3e23fe05b765cab805a7db008/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54", size = 4293460, upload-time = "2025-10-15T23:17:24.885Z" },
+    { url = "https://files.pythonhosted.org/packages/26/e9/e23e7900983c2b8af7a08098db406cf989d7f09caea7897e347598d4cd5b/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459", size = 3995237, upload-time = "2025-10-15T23:17:26.449Z" },
+    { url = "https://files.pythonhosted.org/packages/91/15/af68c509d4a138cfe299d0d7ddb14afba15233223ebd933b4bbdbc7155d3/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422", size = 4967344, upload-time = "2025-10-15T23:17:28.06Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/e3/8643d077c53868b681af077edf6b3cb58288b5423610f21c62aadcbe99f4/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7", size = 4466564, upload-time = "2025-10-15T23:17:29.665Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/43/c1e8726fa59c236ff477ff2b5dc071e54b21e5a1e51aa2cee1676f1c986f/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044", size = 4292415, upload-time = "2025-10-15T23:17:31.686Z" },
+    { url = "https://files.pythonhosted.org/packages/42/f9/2f8fefdb1aee8a8e3256a0568cffc4e6d517b256a2fe97a029b3f1b9fe7e/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665", size = 4931457, upload-time = "2025-10-15T23:17:33.478Z" },
+    { url = "https://files.pythonhosted.org/packages/79/30/9b54127a9a778ccd6d27c3da7563e9f2d341826075ceab89ae3b41bf5be2/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3", size = 4466074, upload-time = "2025-10-15T23:17:35.158Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/68/b4f4a10928e26c941b1b6a179143af9f4d27d88fe84a6a3c53592d2e76bf/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20", size = 4420569, upload-time = "2025-10-15T23:17:37.188Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/49/3746dab4c0d1979888f125226357d3262a6dd40e114ac29e3d2abdf1ec55/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de", size = 4681941, upload-time = "2025-10-15T23:17:39.236Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/30/27654c1dbaf7e4a3531fa1fc77986d04aefa4d6d78259a62c9dc13d7ad36/cryptography-46.0.3-cp314-cp314t-win32.whl", hash = "sha256:8a6e050cb6164d3f830453754094c086ff2d0b2f3a897a1d9820f6139a1f0914", size = 3022339, upload-time = "2025-10-15T23:17:40.888Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/30/640f34ccd4d2a1bc88367b54b926b781b5a018d65f404d409aba76a84b1c/cryptography-46.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:760f83faa07f8b64e9c33fc963d790a2edb24efb479e3520c14a45741cd9b2db", size = 3494315, upload-time = "2025-10-15T23:17:42.769Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/8b/88cc7e3bd0a8e7b861f26981f7b820e1f46aa9d26cc482d0feba0ecb4919/cryptography-46.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:516ea134e703e9fe26bcd1277a4b59ad30586ea90c365a87781d7887a646fe21", size = 2919331, upload-time = "2025-10-15T23:17:44.468Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/23/45fe7f376a7df8daf6da3556603b36f53475a99ce4faacb6ba2cf3d82021/cryptography-46.0.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:cb3d760a6117f621261d662bccc8ef5bc32ca673e037c83fbe565324f5c46936", size = 7218248, upload-time = "2025-10-15T23:17:46.294Z" },
+    { url = "https://files.pythonhosted.org/packages/27/32/b68d27471372737054cbd34c84981f9edbc24fe67ca225d389799614e27f/cryptography-46.0.3-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4b7387121ac7d15e550f5cb4a43aef2559ed759c35df7336c402bb8275ac9683", size = 4294089, upload-time = "2025-10-15T23:17:48.269Z" },
+    { url = "https://files.pythonhosted.org/packages/26/42/fa8389d4478368743e24e61eea78846a0006caffaf72ea24a15159215a14/cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d", size = 4440029, upload-time = "2025-10-15T23:17:49.837Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/eb/f483db0ec5ac040824f269e93dd2bd8a21ecd1027e77ad7bdf6914f2fd80/cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0", size = 4297222, upload-time = "2025-10-15T23:17:51.357Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/cf/da9502c4e1912cb1da3807ea3618a6829bee8207456fbbeebc361ec38ba3/cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc", size = 4012280, upload-time = "2025-10-15T23:17:52.964Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/8f/9adb86b93330e0df8b3dcf03eae67c33ba89958fc2e03862ef1ac2b42465/cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3", size = 4978958, upload-time = "2025-10-15T23:17:54.965Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/a0/5fa77988289c34bdb9f913f5606ecc9ada1adb5ae870bd0d1054a7021cc4/cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971", size = 4473714, upload-time = "2025-10-15T23:17:56.754Z" },
+    { url = "https://files.pythonhosted.org/packages/14/e5/fc82d72a58d41c393697aa18c9abe5ae1214ff6f2a5c18ac470f92777895/cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac", size = 4296970, upload-time = "2025-10-15T23:17:58.588Z" },
+    { url = "https://files.pythonhosted.org/packages/78/06/5663ed35438d0b09056973994f1aec467492b33bd31da36e468b01ec1097/cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04", size = 4940236, upload-time = "2025-10-15T23:18:00.897Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/59/873633f3f2dcd8a053b8dd1d38f783043b5fce589c0f6988bf55ef57e43e/cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506", size = 4472642, upload-time = "2025-10-15T23:18:02.749Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/39/8e71f3930e40f6877737d6f69248cf74d4e34b886a3967d32f919cc50d3b/cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963", size = 4423126, upload-time = "2025-10-15T23:18:04.85Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/c7/f65027c2810e14c3e7268353b1681932b87e5a48e65505d8cc17c99e36ae/cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4", size = 4686573, upload-time = "2025-10-15T23:18:06.908Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/6e/1c8331ddf91ca4730ab3086a0f1be19c65510a33b5a441cb334e7a2d2560/cryptography-46.0.3-cp38-abi3-win32.whl", hash = "sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df", size = 3036695, upload-time = "2025-10-15T23:18:08.672Z" },
+    { url = "https://files.pythonhosted.org/packages/90/45/b0d691df20633eff80955a0fc7695ff9051ffce8b69741444bd9ed7bd0db/cryptography-46.0.3-cp38-abi3-win_amd64.whl", hash = "sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f", size = 3501720, upload-time = "2025-10-15T23:18:10.632Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/cd/1a8633802d766a0fa46f382a77e096d7e209e0817892929655fe0586ae32/cryptography-46.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a23582810fedb8c0bc47524558fb6c56aac3fc252cb306072fd2815da2a47c32", size = 3689163, upload-time = "2025-10-15T23:18:13.821Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/59/6b26512964ace6480c3e54681a9859c974172fb141c38df11eadd8416947/cryptography-46.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7aec276d68421f9574040c26e2a7c3771060bc0cff408bae1dcb19d3ab1e63c", size = 3429474, upload-time = "2025-10-15T23:18:15.477Z" },
+    { url = "https://files.pythonhosted.org/packages/06/8a/e60e46adab4362a682cf142c7dcb5bf79b782ab2199b0dcb81f55970807f/cryptography-46.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7ce938a99998ed3c8aa7e7272dca1a610401ede816d36d0693907d863b10d9ea", size = 3698132, upload-time = "2025-10-15T23:18:17.056Z" },
+    { url = "https://files.pythonhosted.org/packages/da/38/f59940ec4ee91e93d3311f7532671a5cef5570eb04a144bf203b58552d11/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:191bb60a7be5e6f54e30ba16fdfae78ad3a342a0599eb4193ba88e3f3d6e185b", size = 4243992, upload-time = "2025-10-15T23:18:18.695Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/0c/35b3d92ddebfdfda76bb485738306545817253d0a3ded0bfe80ef8e67aa5/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c70cc23f12726be8f8bc72e41d5065d77e4515efae3690326764ea1b07845cfb", size = 4409944, upload-time = "2025-10-15T23:18:20.597Z" },
+    { url = "https://files.pythonhosted.org/packages/99/55/181022996c4063fc0e7666a47049a1ca705abb9c8a13830f074edb347495/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:9394673a9f4de09e28b5356e7fff97d778f8abad85c9d5ac4a4b7e25a0de7717", size = 4242957, upload-time = "2025-10-15T23:18:22.18Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/af/72cd6ef29f9c5f731251acadaeb821559fe25f10852f44a63374c9ca08c1/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:94cd0549accc38d1494e1f8de71eca837d0509d0d44bf11d158524b0e12cebf9", size = 4409447, upload-time = "2025-10-15T23:18:24.209Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/c3/e90f4a4feae6410f914f8ebac129b9ae7a8c92eb60a638012dde42030a9d/cryptography-46.0.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6b5063083824e5509fdba180721d55909ffacccc8adbec85268b48439423d78c", size = 3438528, upload-time = "2025-10-15T23:18:26.227Z" },
+]
+
+[[package]]
+name = "cssselect"
+version = "1.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/0a/c3ea9573b1dc2e151abfe88c7fe0c26d1892fe6ed02d0cdb30f0d57029d5/cssselect-1.3.0.tar.gz", hash = "sha256:57f8a99424cfab289a1b6a816a43075a4b00948c86b4dcf3ef4ee7e15f7ab0c7", size = 42870, upload-time = "2025-03-10T09:30:29.638Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ee/58/257350f7db99b4ae12b614a36256d9cc870d71d9e451e79c2dc3b23d7c3c/cssselect-1.3.0-py3-none-any.whl", hash = "sha256:56d1bf3e198080cc1667e137bc51de9cadfca259f03c2d4e09037b3e01e30f0d", size = 18786, upload-time = "2025-03-10T09:30:28.048Z" },
+]
+
+[[package]]
+name = "distro"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" },
+]
+
+[[package]]
+name = "exceptiongroup"
+version = "1.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" },
+]
+
+[[package]]
+name = "fake-http-header"
+version = "0.3.5"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e3/0b/2849c87d9f13766e29c0a2f4d31681aa72e035016b251ab19d99bde7b592/fake_http_header-0.3.5-py3-none-any.whl", hash = "sha256:cd05f4bebf1b7e38b5f5c03d7fb820c0c17e87d9614fbee0afa39c32c7a2ad3c", size = 14938, upload-time = "2024-10-15T07:27:10.671Z" },
+]
+
+[[package]]
+name = "fake-useragent"
+version = "2.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/41/43/948d10bf42735709edb5ae51e23297d034086f17fc7279fef385a7acb473/fake_useragent-2.2.0.tar.gz", hash = "sha256:4e6ab6571e40cc086d788523cf9e018f618d07f9050f822ff409a4dfe17c16b2", size = 158898, upload-time = "2025-04-14T15:32:19.238Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/51/37/b3ea9cd5558ff4cb51957caca2193981c6b0ff30bd0d2630ac62505d99d0/fake_useragent-2.2.0-py3-none-any.whl", hash = "sha256:67f35ca4d847b0d298187443aaf020413746e56acd985a611908c73dba2daa24", size = 161695, upload-time = "2025-04-14T15:32:17.732Z" },
+]
+
+[[package]]
+name = "filelock"
+version = "3.18.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2", size = 18075, upload-time = "2025-03-14T07:11:40.47Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de", size = 16215, upload-time = "2025-03-14T07:11:39.145Z" },
+]
+
+[[package]]
+name = "frozenlist"
+version = "1.7.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078, upload-time = "2025-06-09T23:02:35.538Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/af/36/0da0a49409f6b47cc2d060dc8c9040b897b5902a8a4e37d9bc1deb11f680/frozenlist-1.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc4df77d638aa2ed703b878dd093725b72a824c3c546c076e8fdf276f78ee84a", size = 81304, upload-time = "2025-06-09T22:59:46.226Z" },
+    { url = "https://files.pythonhosted.org/packages/77/f0/77c11d13d39513b298e267b22eb6cb559c103d56f155aa9a49097221f0b6/frozenlist-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:716a9973a2cc963160394f701964fe25012600f3d311f60c790400b00e568b61", size = 47735, upload-time = "2025-06-09T22:59:48.133Z" },
+    { url = "https://files.pythonhosted.org/packages/37/12/9d07fa18971a44150593de56b2f2947c46604819976784bcf6ea0d5db43b/frozenlist-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0fd1bad056a3600047fb9462cff4c5322cebc59ebf5d0a3725e0ee78955001d", size = 46775, upload-time = "2025-06-09T22:59:49.564Z" },
+    { url = "https://files.pythonhosted.org/packages/70/34/f73539227e06288fcd1f8a76853e755b2b48bca6747e99e283111c18bcd4/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3789ebc19cb811163e70fe2bd354cea097254ce6e707ae42e56f45e31e96cb8e", size = 224644, upload-time = "2025-06-09T22:59:51.35Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/68/c1d9c2f4a6e438e14613bad0f2973567586610cc22dcb1e1241da71de9d3/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af369aa35ee34f132fcfad5be45fbfcde0e3a5f6a1ec0712857f286b7d20cca9", size = 222125, upload-time = "2025-06-09T22:59:52.884Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/d0/98e8f9a515228d708344d7c6986752be3e3192d1795f748c24bcf154ad99/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac64b6478722eeb7a3313d494f8342ef3478dff539d17002f849101b212ef97c", size = 233455, upload-time = "2025-06-09T22:59:54.74Z" },
+    { url = "https://files.pythonhosted.org/packages/79/df/8a11bcec5600557f40338407d3e5bea80376ed1c01a6c0910fcfdc4b8993/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f89f65d85774f1797239693cef07ad4c97fdd0639544bad9ac4b869782eb1981", size = 227339, upload-time = "2025-06-09T22:59:56.187Z" },
+    { url = "https://files.pythonhosted.org/packages/50/82/41cb97d9c9a5ff94438c63cc343eb7980dac4187eb625a51bdfdb7707314/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1073557c941395fdfcfac13eb2456cb8aad89f9de27bae29fabca8e563b12615", size = 212969, upload-time = "2025-06-09T22:59:57.604Z" },
+    { url = "https://files.pythonhosted.org/packages/13/47/f9179ee5ee4f55629e4f28c660b3fdf2775c8bfde8f9c53f2de2d93f52a9/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed8d2fa095aae4bdc7fdd80351009a48d286635edffee66bf865e37a9125c50", size = 222862, upload-time = "2025-06-09T22:59:59.498Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/52/df81e41ec6b953902c8b7e3a83bee48b195cb0e5ec2eabae5d8330c78038/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:24c34bea555fe42d9f928ba0a740c553088500377448febecaa82cc3e88aa1fa", size = 222492, upload-time = "2025-06-09T23:00:01.026Z" },
+    { url = "https://files.pythonhosted.org/packages/84/17/30d6ea87fa95a9408245a948604b82c1a4b8b3e153cea596421a2aef2754/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:69cac419ac6a6baad202c85aaf467b65ac860ac2e7f2ac1686dc40dbb52f6577", size = 238250, upload-time = "2025-06-09T23:00:03.401Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/00/ecbeb51669e3c3df76cf2ddd66ae3e48345ec213a55e3887d216eb4fbab3/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:960d67d0611f4c87da7e2ae2eacf7ea81a5be967861e0c63cf205215afbfac59", size = 218720, upload-time = "2025-06-09T23:00:05.282Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/c0/c224ce0e0eb31cc57f67742071bb470ba8246623c1823a7530be0e76164c/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:41be2964bd4b15bf575e5daee5a5ce7ed3115320fb3c2b71fca05582ffa4dc9e", size = 232585, upload-time = "2025-06-09T23:00:07.962Z" },
+    { url = "https://files.pythonhosted.org/packages/55/3c/34cb694abf532f31f365106deebdeac9e45c19304d83cf7d51ebbb4ca4d1/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:46d84d49e00c9429238a7ce02dc0be8f6d7cd0cd405abd1bebdc991bf27c15bd", size = 234248, upload-time = "2025-06-09T23:00:09.428Z" },
+    { url = "https://files.pythonhosted.org/packages/98/c0/2052d8b6cecda2e70bd81299e3512fa332abb6dcd2969b9c80dfcdddbf75/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:15900082e886edb37480335d9d518cec978afc69ccbc30bd18610b7c1b22a718", size = 221621, upload-time = "2025-06-09T23:00:11.32Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/bf/7dcebae315436903b1d98ffb791a09d674c88480c158aa171958a3ac07f0/frozenlist-1.7.0-cp310-cp310-win32.whl", hash = "sha256:400ddd24ab4e55014bba442d917203c73b2846391dd42ca5e38ff52bb18c3c5e", size = 39578, upload-time = "2025-06-09T23:00:13.526Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/5f/f69818f017fa9a3d24d1ae39763e29b7f60a59e46d5f91b9c6b21622f4cd/frozenlist-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:6eb93efb8101ef39d32d50bce242c84bcbddb4f7e9febfa7b524532a239b4464", size = 43830, upload-time = "2025-06-09T23:00:14.98Z" },
+    { url = "https://files.pythonhosted.org/packages/34/7e/803dde33760128acd393a27eb002f2020ddb8d99d30a44bfbaab31c5f08a/frozenlist-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa51e147a66b2d74de1e6e2cf5921890de6b0f4820b257465101d7f37b49fb5a", size = 82251, upload-time = "2025-06-09T23:00:16.279Z" },
+    { url = "https://files.pythonhosted.org/packages/75/a9/9c2c5760b6ba45eae11334db454c189d43d34a4c0b489feb2175e5e64277/frozenlist-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9b35db7ce1cd71d36ba24f80f0c9e7cff73a28d7a74e91fe83e23d27c7828750", size = 48183, upload-time = "2025-06-09T23:00:17.698Z" },
+    { url = "https://files.pythonhosted.org/packages/47/be/4038e2d869f8a2da165f35a6befb9158c259819be22eeaf9c9a8f6a87771/frozenlist-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34a69a85e34ff37791e94542065c8416c1afbf820b68f720452f636d5fb990cd", size = 47107, upload-time = "2025-06-09T23:00:18.952Z" },
+    { url = "https://files.pythonhosted.org/packages/79/26/85314b8a83187c76a37183ceed886381a5f992975786f883472fcb6dc5f2/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a646531fa8d82c87fe4bb2e596f23173caec9185bfbca5d583b4ccfb95183e2", size = 237333, upload-time = "2025-06-09T23:00:20.275Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/fd/e5b64f7d2c92a41639ffb2ad44a6a82f347787abc0c7df5f49057cf11770/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:79b2ffbba483f4ed36a0f236ccb85fbb16e670c9238313709638167670ba235f", size = 231724, upload-time = "2025-06-09T23:00:21.705Z" },
+    { url = "https://files.pythonhosted.org/packages/20/fb/03395c0a43a5976af4bf7534759d214405fbbb4c114683f434dfdd3128ef/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a26f205c9ca5829cbf82bb2a84b5c36f7184c4316617d7ef1b271a56720d6b30", size = 245842, upload-time = "2025-06-09T23:00:23.148Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/15/c01c8e1dffdac5d9803507d824f27aed2ba76b6ed0026fab4d9866e82f1f/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bcacfad3185a623fa11ea0e0634aac7b691aa925d50a440f39b458e41c561d98", size = 239767, upload-time = "2025-06-09T23:00:25.103Z" },
+    { url = "https://files.pythonhosted.org/packages/14/99/3f4c6fe882c1f5514b6848aa0a69b20cb5e5d8e8f51a339d48c0e9305ed0/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72c1b0fe8fe451b34f12dce46445ddf14bd2a5bcad7e324987194dc8e3a74c86", size = 224130, upload-time = "2025-06-09T23:00:27.061Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/83/220a374bd7b2aeba9d0725130665afe11de347d95c3620b9b82cc2fcab97/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61d1a5baeaac6c0798ff6edfaeaa00e0e412d49946c53fae8d4b8e8b3566c4ae", size = 235301, upload-time = "2025-06-09T23:00:29.02Z" },
+    { url = "https://files.pythonhosted.org/packages/03/3c/3e3390d75334a063181625343e8daab61b77e1b8214802cc4e8a1bb678fc/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7edf5c043c062462f09b6820de9854bf28cc6cc5b6714b383149745e287181a8", size = 234606, upload-time = "2025-06-09T23:00:30.514Z" },
+    { url = "https://files.pythonhosted.org/packages/23/1e/58232c19608b7a549d72d9903005e2d82488f12554a32de2d5fb59b9b1ba/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d50ac7627b3a1bd2dcef6f9da89a772694ec04d9a61b66cf87f7d9446b4a0c31", size = 248372, upload-time = "2025-06-09T23:00:31.966Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/a4/e4a567e01702a88a74ce8a324691e62a629bf47d4f8607f24bf1c7216e7f/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ce48b2fece5aeb45265bb7a58259f45027db0abff478e3077e12b05b17fb9da7", size = 229860, upload-time = "2025-06-09T23:00:33.375Z" },
+    { url = "https://files.pythonhosted.org/packages/73/a6/63b3374f7d22268b41a9db73d68a8233afa30ed164c46107b33c4d18ecdd/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fe2365ae915a1fafd982c146754e1de6ab3478def8a59c86e1f7242d794f97d5", size = 245893, upload-time = "2025-06-09T23:00:35.002Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/eb/d18b3f6e64799a79673c4ba0b45e4cfbe49c240edfd03a68be20002eaeaa/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:45a6f2fdbd10e074e8814eb98b05292f27bad7d1883afbe009d96abdcf3bc898", size = 246323, upload-time = "2025-06-09T23:00:36.468Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/f5/720f3812e3d06cd89a1d5db9ff6450088b8f5c449dae8ffb2971a44da506/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:21884e23cffabb157a9dd7e353779077bf5b8f9a58e9b262c6caad2ef5f80a56", size = 233149, upload-time = "2025-06-09T23:00:37.963Z" },
+    { url = "https://files.pythonhosted.org/packages/69/68/03efbf545e217d5db8446acfd4c447c15b7c8cf4dbd4a58403111df9322d/frozenlist-1.7.0-cp311-cp311-win32.whl", hash = "sha256:284d233a8953d7b24f9159b8a3496fc1ddc00f4db99c324bd5fb5f22d8698ea7", size = 39565, upload-time = "2025-06-09T23:00:39.753Z" },
+    { url = "https://files.pythonhosted.org/packages/58/17/fe61124c5c333ae87f09bb67186d65038834a47d974fc10a5fadb4cc5ae1/frozenlist-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:387cbfdcde2f2353f19c2f66bbb52406d06ed77519ac7ee21be0232147c2592d", size = 44019, upload-time = "2025-06-09T23:00:40.988Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424, upload-time = "2025-06-09T23:00:42.24Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952, upload-time = "2025-06-09T23:00:43.481Z" },
+    { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688, upload-time = "2025-06-09T23:00:44.793Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084, upload-time = "2025-06-09T23:00:46.125Z" },
+    { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524, upload-time = "2025-06-09T23:00:47.73Z" },
+    { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493, upload-time = "2025-06-09T23:00:49.742Z" },
+    { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116, upload-time = "2025-06-09T23:00:51.352Z" },
+    { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557, upload-time = "2025-06-09T23:00:52.855Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820, upload-time = "2025-06-09T23:00:54.43Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542, upload-time = "2025-06-09T23:00:56.409Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350, upload-time = "2025-06-09T23:00:58.468Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093, upload-time = "2025-06-09T23:01:00.015Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482, upload-time = "2025-06-09T23:01:01.474Z" },
+    { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590, upload-time = "2025-06-09T23:01:02.961Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785, upload-time = "2025-06-09T23:01:05.095Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487, upload-time = "2025-06-09T23:01:06.54Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874, upload-time = "2025-06-09T23:01:07.752Z" },
+    { url = "https://files.pythonhosted.org/packages/24/90/6b2cebdabdbd50367273c20ff6b57a3dfa89bd0762de02c3a1eb42cb6462/frozenlist-1.7.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee80eeda5e2a4e660651370ebffd1286542b67e268aa1ac8d6dbe973120ef7ee", size = 79791, upload-time = "2025-06-09T23:01:09.368Z" },
+    { url = "https://files.pythonhosted.org/packages/83/2e/5b70b6a3325363293fe5fc3ae74cdcbc3e996c2a11dde2fd9f1fb0776d19/frozenlist-1.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d1a81c85417b914139e3a9b995d4a1c84559afc839a93cf2cb7f15e6e5f6ed2d", size = 47165, upload-time = "2025-06-09T23:01:10.653Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/25/a0895c99270ca6966110f4ad98e87e5662eab416a17e7fd53c364bf8b954/frozenlist-1.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbb65198a9132ebc334f237d7b0df163e4de83fb4f2bdfe46c1e654bdb0c5d43", size = 45881, upload-time = "2025-06-09T23:01:12.296Z" },
+    { url = "https://files.pythonhosted.org/packages/19/7c/71bb0bbe0832793c601fff68cd0cf6143753d0c667f9aec93d3c323f4b55/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dab46c723eeb2c255a64f9dc05b8dd601fde66d6b19cdb82b2e09cc6ff8d8b5d", size = 232409, upload-time = "2025-06-09T23:01:13.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/45/ed2798718910fe6eb3ba574082aaceff4528e6323f9a8570be0f7028d8e9/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6aeac207a759d0dedd2e40745575ae32ab30926ff4fa49b1635def65806fddee", size = 225132, upload-time = "2025-06-09T23:01:15.264Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/e2/8417ae0f8eacb1d071d4950f32f229aa6bf68ab69aab797b72a07ea68d4f/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bd8c4e58ad14b4fa7802b8be49d47993182fdd4023393899632c88fd8cd994eb", size = 237638, upload-time = "2025-06-09T23:01:16.752Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/b7/2ace5450ce85f2af05a871b8c8719b341294775a0a6c5585d5e6170f2ce7/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04fb24d104f425da3540ed83cbfc31388a586a7696142004c577fa61c6298c3f", size = 233539, upload-time = "2025-06-09T23:01:18.202Z" },
+    { url = "https://files.pythonhosted.org/packages/46/b9/6989292c5539553dba63f3c83dc4598186ab2888f67c0dc1d917e6887db6/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a5c505156368e4ea6b53b5ac23c92d7edc864537ff911d2fb24c140bb175e60", size = 215646, upload-time = "2025-06-09T23:01:19.649Z" },
+    { url = "https://files.pythonhosted.org/packages/72/31/bc8c5c99c7818293458fe745dab4fd5730ff49697ccc82b554eb69f16a24/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bd7eb96a675f18aa5c553eb7ddc24a43c8c18f22e1f9925528128c052cdbe00", size = 232233, upload-time = "2025-06-09T23:01:21.175Z" },
+    { url = "https://files.pythonhosted.org/packages/59/52/460db4d7ba0811b9ccb85af996019f5d70831f2f5f255f7cc61f86199795/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:05579bf020096fe05a764f1f84cd104a12f78eaab68842d036772dc6d4870b4b", size = 227996, upload-time = "2025-06-09T23:01:23.098Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/c9/f4b39e904c03927b7ecf891804fd3b4df3db29b9e487c6418e37988d6e9d/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:376b6222d114e97eeec13d46c486facd41d4f43bab626b7c3f6a8b4e81a5192c", size = 242280, upload-time = "2025-06-09T23:01:24.808Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/33/3f8d6ced42f162d743e3517781566b8481322be321b486d9d262adf70bfb/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0aa7e176ebe115379b5b1c95b4096fb1c17cce0847402e227e712c27bdb5a949", size = 217717, upload-time = "2025-06-09T23:01:26.28Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/e8/ad683e75da6ccef50d0ab0c2b2324b32f84fc88ceee778ed79b8e2d2fe2e/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3fbba20e662b9c2130dc771e332a99eff5da078b2b2648153a40669a6d0e36ca", size = 236644, upload-time = "2025-06-09T23:01:27.887Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/14/8d19ccdd3799310722195a72ac94ddc677541fb4bef4091d8e7775752360/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f3f4410a0a601d349dd406b5713fec59b4cee7e71678d5b17edda7f4655a940b", size = 238879, upload-time = "2025-06-09T23:01:29.524Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/13/c12bf657494c2fd1079a48b2db49fa4196325909249a52d8f09bc9123fd7/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cdfaaec6a2f9327bf43c933c0319a7c429058e8537c508964a133dffee412e", size = 232502, upload-time = "2025-06-09T23:01:31.287Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/8b/e7f9dfde869825489382bc0d512c15e96d3964180c9499efcec72e85db7e/frozenlist-1.7.0-cp313-cp313-win32.whl", hash = "sha256:5fc4df05a6591c7768459caba1b342d9ec23fa16195e744939ba5914596ae3e1", size = 39169, upload-time = "2025-06-09T23:01:35.503Z" },
+    { url = "https://files.pythonhosted.org/packages/35/89/a487a98d94205d85745080a37860ff5744b9820a2c9acbcdd9440bfddf98/frozenlist-1.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:52109052b9791a3e6b5d1b65f4b909703984b770694d3eb64fad124c835d7cba", size = 43219, upload-time = "2025-06-09T23:01:36.784Z" },
+    { url = "https://files.pythonhosted.org/packages/56/d5/5c4cf2319a49eddd9dd7145e66c4866bdc6f3dbc67ca3d59685149c11e0d/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a6f86e4193bb0e235ef6ce3dde5cbabed887e0b11f516ce8a0f4d3b33078ec2d", size = 84345, upload-time = "2025-06-09T23:01:38.295Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/7d/ec2c1e1dc16b85bc9d526009961953df9cec8481b6886debb36ec9107799/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:82d664628865abeb32d90ae497fb93df398a69bb3434463d172b80fc25b0dd7d", size = 48880, upload-time = "2025-06-09T23:01:39.887Z" },
+    { url = "https://files.pythonhosted.org/packages/69/86/f9596807b03de126e11e7d42ac91e3d0b19a6599c714a1989a4e85eeefc4/frozenlist-1.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:912a7e8375a1c9a68325a902f3953191b7b292aa3c3fb0d71a216221deca460b", size = 48498, upload-time = "2025-06-09T23:01:41.318Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/cb/df6de220f5036001005f2d726b789b2c0b65f2363b104bbc16f5be8084f8/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9537c2777167488d539bc5de2ad262efc44388230e5118868e172dd4a552b146", size = 292296, upload-time = "2025-06-09T23:01:42.685Z" },
+    { url = "https://files.pythonhosted.org/packages/83/1f/de84c642f17c8f851a2905cee2dae401e5e0daca9b5ef121e120e19aa825/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f34560fb1b4c3e30ba35fa9a13894ba39e5acfc5f60f57d8accde65f46cc5e74", size = 273103, upload-time = "2025-06-09T23:01:44.166Z" },
+    { url = "https://files.pythonhosted.org/packages/88/3c/c840bfa474ba3fa13c772b93070893c6e9d5c0350885760376cbe3b6c1b3/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:acd03d224b0175f5a850edc104ac19040d35419eddad04e7cf2d5986d98427f1", size = 292869, upload-time = "2025-06-09T23:01:45.681Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/1c/3efa6e7d5a39a1d5ef0abeb51c48fb657765794a46cf124e5aca2c7a592c/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2038310bc582f3d6a09b3816ab01737d60bf7b1ec70f5356b09e84fb7408ab1", size = 291467, upload-time = "2025-06-09T23:01:47.234Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/00/d5c5e09d4922c395e2f2f6b79b9a20dab4b67daaf78ab92e7729341f61f6/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c05e4c8e5f36e5e088caa1bf78a687528f83c043706640a92cb76cd6999384", size = 266028, upload-time = "2025-06-09T23:01:48.819Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/27/72765be905619dfde25a7f33813ac0341eb6b076abede17a2e3fbfade0cb/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:765bb588c86e47d0b68f23c1bee323d4b703218037765dcf3f25c838c6fecceb", size = 284294, upload-time = "2025-06-09T23:01:50.394Z" },
+    { url = "https://files.pythonhosted.org/packages/88/67/c94103a23001b17808eb7dd1200c156bb69fb68e63fcf0693dde4cd6228c/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:32dc2e08c67d86d0969714dd484fd60ff08ff81d1a1e40a77dd34a387e6ebc0c", size = 281898, upload-time = "2025-06-09T23:01:52.234Z" },
+    { url = "https://files.pythonhosted.org/packages/42/34/a3e2c00c00f9e2a9db5653bca3fec306349e71aff14ae45ecc6d0951dd24/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:c0303e597eb5a5321b4de9c68e9845ac8f290d2ab3f3e2c864437d3c5a30cd65", size = 290465, upload-time = "2025-06-09T23:01:53.788Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/73/f89b7fbce8b0b0c095d82b008afd0590f71ccb3dee6eee41791cf8cd25fd/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a47f2abb4e29b3a8d0b530f7c3598badc6b134562b1a5caee867f7c62fee51e3", size = 266385, upload-time = "2025-06-09T23:01:55.769Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/45/e365fdb554159462ca12df54bc59bfa7a9a273ecc21e99e72e597564d1ae/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:3d688126c242a6fabbd92e02633414d40f50bb6002fa4cf995a1d18051525657", size = 288771, upload-time = "2025-06-09T23:01:57.4Z" },
+    { url = "https://files.pythonhosted.org/packages/00/11/47b6117002a0e904f004d70ec5194fe9144f117c33c851e3d51c765962d0/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4e7e9652b3d367c7bd449a727dc79d5043f48b88d0cbfd4f9f1060cf2b414104", size = 288206, upload-time = "2025-06-09T23:01:58.936Z" },
+    { url = "https://files.pythonhosted.org/packages/40/37/5f9f3c3fd7f7746082ec67bcdc204db72dad081f4f83a503d33220a92973/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1a85e345b4c43db8b842cab1feb41be5cc0b10a1830e6295b69d7310f99becaf", size = 282620, upload-time = "2025-06-09T23:02:00.493Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/31/8fbc5af2d183bff20f21aa743b4088eac4445d2bb1cdece449ae80e4e2d1/frozenlist-1.7.0-cp313-cp313t-win32.whl", hash = "sha256:3a14027124ddb70dfcee5148979998066897e79f89f64b13328595c4bdf77c81", size = 43059, upload-time = "2025-06-09T23:02:02.072Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/ed/41956f52105b8dbc26e457c5705340c67c8cc2b79f394b79bffc09d0e938/frozenlist-1.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3bf8010d71d4507775f658e9823210b7427be36625b387221642725b515dcf3e", size = 47516, upload-time = "2025-06-09T23:02:03.779Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" },
+]
+
+[[package]]
+name = "fsspec"
+version = "2025.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/00/f7/27f15d41f0ed38e8fcc488584b57e902b331da7f7c6dcda53721b15838fc/fsspec-2025.5.1.tar.gz", hash = "sha256:2e55e47a540b91843b755e83ded97c6e897fa0942b11490113f09e9c443c2475", size = 303033, upload-time = "2025-05-24T12:03:23.792Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bb/61/78c7b3851add1481b048b5fdc29067397a1784e2910592bc81bb3f608635/fsspec-2025.5.1-py3-none-any.whl", hash = "sha256:24d3a2e663d5fc735ab256263c4075f374a174c3410c0b25e5bd1970bceaa462", size = 199052, upload-time = "2025-05-24T12:03:21.66Z" },
+]
+
+[[package]]
+name = "greenlet"
+version = "3.2.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c9/92/bb85bd6e80148a4d2e0c59f7c0c2891029f8fd510183afc7d8d2feeed9b6/greenlet-3.2.3.tar.gz", hash = "sha256:8b0dd8ae4c0d6f5e54ee55ba935eeb3d735a9b58a8a1e5b5cbab64e01a39f365", size = 185752, upload-time = "2025-06-05T16:16:09.955Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/92/db/b4c12cff13ebac2786f4f217f06588bccd8b53d260453404ef22b121fc3a/greenlet-3.2.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:1afd685acd5597349ee6d7a88a8bec83ce13c106ac78c196ee9dde7c04fe87be", size = 268977, upload-time = "2025-06-05T16:10:24.001Z" },
+    { url = "https://files.pythonhosted.org/packages/52/61/75b4abd8147f13f70986df2801bf93735c1bd87ea780d70e3b3ecda8c165/greenlet-3.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:761917cac215c61e9dc7324b2606107b3b292a8349bdebb31503ab4de3f559ac", size = 627351, upload-time = "2025-06-05T16:38:50.685Z" },
+    { url = "https://files.pythonhosted.org/packages/35/aa/6894ae299d059d26254779a5088632874b80ee8cf89a88bca00b0709d22f/greenlet-3.2.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a433dbc54e4a37e4fff90ef34f25a8c00aed99b06856f0119dcf09fbafa16392", size = 638599, upload-time = "2025-06-05T16:41:34.057Z" },
+    { url = "https://files.pythonhosted.org/packages/30/64/e01a8261d13c47f3c082519a5e9dbf9e143cc0498ed20c911d04e54d526c/greenlet-3.2.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:72e77ed69312bab0434d7292316d5afd6896192ac4327d44f3d613ecb85b037c", size = 634482, upload-time = "2025-06-05T16:48:16.26Z" },
+    { url = "https://files.pythonhosted.org/packages/47/48/ff9ca8ba9772d083a4f5221f7b4f0ebe8978131a9ae0909cf202f94cd879/greenlet-3.2.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:68671180e3849b963649254a882cd544a3c75bfcd2c527346ad8bb53494444db", size = 633284, upload-time = "2025-06-05T16:13:01.599Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/45/626e974948713bc15775b696adb3eb0bd708bec267d6d2d5c47bb47a6119/greenlet-3.2.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49c8cfb18fb419b3d08e011228ef8a25882397f3a859b9fe1436946140b6756b", size = 582206, upload-time = "2025-06-05T16:12:48.51Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/8e/8b6f42c67d5df7db35b8c55c9a850ea045219741bb14416255616808c690/greenlet-3.2.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:efc6dc8a792243c31f2f5674b670b3a95d46fa1c6a912b8e310d6f542e7b0712", size = 1111412, upload-time = "2025-06-05T16:36:45.479Z" },
+    { url = "https://files.pythonhosted.org/packages/05/46/ab58828217349500a7ebb81159d52ca357da747ff1797c29c6023d79d798/greenlet-3.2.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:731e154aba8e757aedd0781d4b240f1225b075b4409f1bb83b05ff410582cf00", size = 1135054, upload-time = "2025-06-05T16:12:36.478Z" },
+    { url = "https://files.pythonhosted.org/packages/68/7f/d1b537be5080721c0f0089a8447d4ef72839039cdb743bdd8ffd23046e9a/greenlet-3.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:96c20252c2f792defe9a115d3287e14811036d51e78b3aaddbee23b69b216302", size = 296573, upload-time = "2025-06-05T16:34:26.521Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/2e/d4fcb2978f826358b673f779f78fa8a32ee37df11920dc2bb5589cbeecef/greenlet-3.2.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:784ae58bba89fa1fa5733d170d42486580cab9decda3484779f4759345b29822", size = 270219, upload-time = "2025-06-05T16:10:10.414Z" },
+    { url = "https://files.pythonhosted.org/packages/16/24/929f853e0202130e4fe163bc1d05a671ce8dcd604f790e14896adac43a52/greenlet-3.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0921ac4ea42a5315d3446120ad48f90c3a6b9bb93dd9b3cf4e4d84a66e42de83", size = 630383, upload-time = "2025-06-05T16:38:51.785Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/b2/0320715eb61ae70c25ceca2f1d5ae620477d246692d9cc284c13242ec31c/greenlet-3.2.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d2971d93bb99e05f8c2c0c2f4aa9484a18d98c4c3bd3c62b65b7e6ae33dfcfaf", size = 642422, upload-time = "2025-06-05T16:41:35.259Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/49/445fd1a210f4747fedf77615d941444349c6a3a4a1135bba9701337cd966/greenlet-3.2.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c667c0bf9d406b77a15c924ef3285e1e05250948001220368e039b6aa5b5034b", size = 638375, upload-time = "2025-06-05T16:48:18.235Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/c8/ca19760cf6eae75fa8dc32b487e963d863b3ee04a7637da77b616703bc37/greenlet-3.2.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:592c12fb1165be74592f5de0d70f82bc5ba552ac44800d632214b76089945147", size = 637627, upload-time = "2025-06-05T16:13:02.858Z" },
+    { url = "https://files.pythonhosted.org/packages/65/89/77acf9e3da38e9bcfca881e43b02ed467c1dedc387021fc4d9bd9928afb8/greenlet-3.2.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29e184536ba333003540790ba29829ac14bb645514fbd7e32af331e8202a62a5", size = 585502, upload-time = "2025-06-05T16:12:49.642Z" },
+    { url = "https://files.pythonhosted.org/packages/97/c6/ae244d7c95b23b7130136e07a9cc5aadd60d59b5951180dc7dc7e8edaba7/greenlet-3.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:93c0bb79844a367782ec4f429d07589417052e621aa39a5ac1fb99c5aa308edc", size = 1114498, upload-time = "2025-06-05T16:36:46.598Z" },
+    { url = "https://files.pythonhosted.org/packages/89/5f/b16dec0cbfd3070658e0d744487919740c6d45eb90946f6787689a7efbce/greenlet-3.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:751261fc5ad7b6705f5f76726567375bb2104a059454e0226e1eef6c756748ba", size = 1139977, upload-time = "2025-06-05T16:12:38.262Z" },
+    { url = "https://files.pythonhosted.org/packages/66/77/d48fb441b5a71125bcac042fc5b1494c806ccb9a1432ecaa421e72157f77/greenlet-3.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:83a8761c75312361aa2b5b903b79da97f13f556164a7dd2d5448655425bd4c34", size = 297017, upload-time = "2025-06-05T16:25:05.225Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/94/ad0d435f7c48debe960c53b8f60fb41c2026b1d0fa4a99a1cb17c3461e09/greenlet-3.2.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:25ad29caed5783d4bd7a85c9251c651696164622494c00802a139c00d639242d", size = 271992, upload-time = "2025-06-05T16:11:23.467Z" },
+    { url = "https://files.pythonhosted.org/packages/93/5d/7c27cf4d003d6e77749d299c7c8f5fd50b4f251647b5c2e97e1f20da0ab5/greenlet-3.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88cd97bf37fe24a6710ec6a3a7799f3f81d9cd33317dcf565ff9950c83f55e0b", size = 638820, upload-time = "2025-06-05T16:38:52.882Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/7e/807e1e9be07a125bb4c169144937910bf59b9d2f6d931578e57f0bce0ae2/greenlet-3.2.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:baeedccca94880d2f5666b4fa16fc20ef50ba1ee353ee2d7092b383a243b0b0d", size = 653046, upload-time = "2025-06-05T16:41:36.343Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/ab/158c1a4ea1068bdbc78dba5a3de57e4c7aeb4e7fa034320ea94c688bfb61/greenlet-3.2.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:be52af4b6292baecfa0f397f3edb3c6092ce071b499dd6fe292c9ac9f2c8f264", size = 647701, upload-time = "2025-06-05T16:48:19.604Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/0d/93729068259b550d6a0288da4ff72b86ed05626eaf1eb7c0d3466a2571de/greenlet-3.2.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0cc73378150b8b78b0c9fe2ce56e166695e67478550769536a6742dca3651688", size = 649747, upload-time = "2025-06-05T16:13:04.628Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/f6/c82ac1851c60851302d8581680573245c8fc300253fc1ff741ae74a6c24d/greenlet-3.2.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:706d016a03e78df129f68c4c9b4c4f963f7d73534e48a24f5f5a7101ed13dbbb", size = 605461, upload-time = "2025-06-05T16:12:50.792Z" },
+    { url = "https://files.pythonhosted.org/packages/98/82/d022cf25ca39cf1200650fc58c52af32c90f80479c25d1cbf57980ec3065/greenlet-3.2.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:419e60f80709510c343c57b4bb5a339d8767bf9aef9b8ce43f4f143240f88b7c", size = 1121190, upload-time = "2025-06-05T16:36:48.59Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/e1/25297f70717abe8104c20ecf7af0a5b82d2f5a980eb1ac79f65654799f9f/greenlet-3.2.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:93d48533fade144203816783373f27a97e4193177ebaaf0fc396db19e5d61163", size = 1149055, upload-time = "2025-06-05T16:12:40.457Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/8f/8f9e56c5e82eb2c26e8cde787962e66494312dc8cb261c460e1f3a9c88bc/greenlet-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:7454d37c740bb27bdeddfc3f358f26956a07d5220818ceb467a483197d84f849", size = 297817, upload-time = "2025-06-05T16:29:49.244Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/cf/f5c0b23309070ae93de75c90d29300751a5aacefc0a3ed1b1d8edb28f08b/greenlet-3.2.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:500b8689aa9dd1ab26872a34084503aeddefcb438e2e7317b89b11eaea1901ad", size = 270732, upload-time = "2025-06-05T16:10:08.26Z" },
+    { url = "https://files.pythonhosted.org/packages/48/ae/91a957ba60482d3fecf9be49bc3948f341d706b52ddb9d83a70d42abd498/greenlet-3.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a07d3472c2a93117af3b0136f246b2833fdc0b542d4a9799ae5f41c28323faef", size = 639033, upload-time = "2025-06-05T16:38:53.983Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/df/20ffa66dd5a7a7beffa6451bdb7400d66251374ab40b99981478c69a67a8/greenlet-3.2.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:8704b3768d2f51150626962f4b9a9e4a17d2e37c8a8d9867bbd9fa4eb938d3b3", size = 652999, upload-time = "2025-06-05T16:41:37.89Z" },
+    { url = "https://files.pythonhosted.org/packages/51/b4/ebb2c8cb41e521f1d72bf0465f2f9a2fd803f674a88db228887e6847077e/greenlet-3.2.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5035d77a27b7c62db6cf41cf786cfe2242644a7a337a0e155c80960598baab95", size = 647368, upload-time = "2025-06-05T16:48:21.467Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/6a/1e1b5aa10dced4ae876a322155705257748108b7fd2e4fae3f2a091fe81a/greenlet-3.2.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2d8aa5423cd4a396792f6d4580f88bdc6efcb9205891c9d40d20f6e670992efb", size = 650037, upload-time = "2025-06-05T16:13:06.402Z" },
+    { url = "https://files.pythonhosted.org/packages/26/f2/ad51331a157c7015c675702e2d5230c243695c788f8f75feba1af32b3617/greenlet-3.2.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c724620a101f8170065d7dded3f962a2aea7a7dae133a009cada42847e04a7b", size = 608402, upload-time = "2025-06-05T16:12:51.91Z" },
+    { url = "https://files.pythonhosted.org/packages/26/bc/862bd2083e6b3aff23300900a956f4ea9a4059de337f5c8734346b9b34fc/greenlet-3.2.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:873abe55f134c48e1f2a6f53f7d1419192a3d1a4e873bace00499a4e45ea6af0", size = 1119577, upload-time = "2025-06-05T16:36:49.787Z" },
+    { url = "https://files.pythonhosted.org/packages/86/94/1fc0cc068cfde885170e01de40a619b00eaa8f2916bf3541744730ffb4c3/greenlet-3.2.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:024571bbce5f2c1cfff08bf3fbaa43bbc7444f580ae13b0099e95d0e6e67ed36", size = 1147121, upload-time = "2025-06-05T16:12:42.527Z" },
+    { url = "https://files.pythonhosted.org/packages/27/1a/199f9587e8cb08a0658f9c30f3799244307614148ffe8b1e3aa22f324dea/greenlet-3.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:5195fb1e75e592dd04ce79881c8a22becdfa3e6f500e7feb059b1e6fdd54d3e3", size = 297603, upload-time = "2025-06-05T16:20:12.651Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/ca/accd7aa5280eb92b70ed9e8f7fd79dc50a2c21d8c73b9a0856f5b564e222/greenlet-3.2.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:3d04332dddb10b4a211b68111dabaee2e1a073663d117dc10247b5b1642bac86", size = 271479, upload-time = "2025-06-05T16:10:47.525Z" },
+    { url = "https://files.pythonhosted.org/packages/55/71/01ed9895d9eb49223280ecc98a557585edfa56b3d0e965b9fa9f7f06b6d9/greenlet-3.2.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8186162dffde068a465deab08fc72c767196895c39db26ab1c17c0b77a6d8b97", size = 683952, upload-time = "2025-06-05T16:38:55.125Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/61/638c4bdf460c3c678a0a1ef4c200f347dff80719597e53b5edb2fb27ab54/greenlet-3.2.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f4bfbaa6096b1b7a200024784217defedf46a07c2eee1a498e94a1b5f8ec5728", size = 696917, upload-time = "2025-06-05T16:41:38.959Z" },
+    { url = "https://files.pythonhosted.org/packages/22/cc/0bd1a7eb759d1f3e3cc2d1bc0f0b487ad3cc9f34d74da4b80f226fde4ec3/greenlet-3.2.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:ed6cfa9200484d234d8394c70f5492f144b20d4533f69262d530a1a082f6ee9a", size = 692443, upload-time = "2025-06-05T16:48:23.113Z" },
+    { url = "https://files.pythonhosted.org/packages/67/10/b2a4b63d3f08362662e89c103f7fe28894a51ae0bc890fabf37d1d780e52/greenlet-3.2.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:02b0df6f63cd15012bed5401b47829cfd2e97052dc89da3cfaf2c779124eb892", size = 692995, upload-time = "2025-06-05T16:13:07.972Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/c6/ad82f148a4e3ce9564056453a71529732baf5448ad53fc323e37efe34f66/greenlet-3.2.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86c2d68e87107c1792e2e8d5399acec2487a4e993ab76c792408e59394d52141", size = 655320, upload-time = "2025-06-05T16:12:53.453Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/4f/aab73ecaa6b3086a4c89863d94cf26fa84cbff63f52ce9bc4342b3087a06/greenlet-3.2.3-cp314-cp314-win_amd64.whl", hash = "sha256:8c47aae8fbbfcf82cc13327ae802ba13c9c36753b67e760023fd116bc124a62a", size = 301236, upload-time = "2025-06-05T16:15:20.111Z" },
+]
+
+[[package]]
+name = "h11"
+version = "0.16.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
+]
+
+[[package]]
+name = "h2"
+version = "4.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "hpack" },
+    { name = "hyperframe" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1b/38/d7f80fd13e6582fb8e0df8c9a653dcc02b03ca34f4d72f34869298c5baf8/h2-4.2.0.tar.gz", hash = "sha256:c8a52129695e88b1a0578d8d2cc6842bbd79128ac685463b887ee278126ad01f", size = 2150682, upload-time = "2025-02-02T07:43:51.815Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d0/9e/984486f2d0a0bd2b024bf4bc1c62688fcafa9e61991f041fb0e2def4a982/h2-4.2.0-py3-none-any.whl", hash = "sha256:479a53ad425bb29af087f3458a61d30780bc818e4ebcf01f0b536ba916462ed0", size = 60957, upload-time = "2025-02-01T11:02:26.481Z" },
+]
+
+[[package]]
+name = "hf-xet"
+version = "1.1.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ed/d4/7685999e85945ed0d7f0762b686ae7015035390de1161dcea9d5276c134c/hf_xet-1.1.5.tar.gz", hash = "sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694", size = 495969, upload-time = "2025-06-20T21:48:38.007Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/00/89/a1119eebe2836cb25758e7661d6410d3eae982e2b5e974bcc4d250be9012/hf_xet-1.1.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23", size = 2687929, upload-time = "2025-06-20T21:48:32.284Z" },
+    { url = "https://files.pythonhosted.org/packages/de/5f/2c78e28f309396e71ec8e4e9304a6483dcbc36172b5cea8f291994163425/hf_xet-1.1.5-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8", size = 2556338, upload-time = "2025-06-20T21:48:30.079Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/2f/6cad7b5fe86b7652579346cb7f85156c11761df26435651cbba89376cd2c/hf_xet-1.1.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1", size = 3102894, upload-time = "2025-06-20T21:48:28.114Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/54/0fcf2b619720a26fbb6cc941e89f2472a522cd963a776c089b189559447f/hf_xet-1.1.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18", size = 3002134, upload-time = "2025-06-20T21:48:25.906Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/92/1d351ac6cef7c4ba8c85744d37ffbfac2d53d0a6c04d2cabeba614640a78/hf_xet-1.1.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14", size = 3171009, upload-time = "2025-06-20T21:48:33.987Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/65/4b2ddb0e3e983f2508528eb4501288ae2f84963586fbdfae596836d5e57a/hf_xet-1.1.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a", size = 3279245, upload-time = "2025-06-20T21:48:36.051Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/55/ef77a85ee443ae05a9e9cba1c9f0dd9241eb42da2aeba1dc50f51154c81a/hf_xet-1.1.5-cp37-abi3-win_amd64.whl", hash = "sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245", size = 2738931, upload-time = "2025-06-20T21:48:39.482Z" },
+]
+
+[[package]]
+name = "hpack"
+version = "4.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" },
+]
+
+[[package]]
+name = "httpcore"
+version = "1.0.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },
+]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "certifi" },
+    { name = "httpcore" },
+    { name = "idna" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
+]
+
+[package.optional-dependencies]
+http2 = [
+    { name = "h2" },
+]
+
+[[package]]
+name = "huggingface-hub"
+version = "0.33.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "filelock" },
+    { name = "fsspec" },
+    { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
+    { name = "packaging" },
+    { name = "pyyaml" },
+    { name = "requests" },
+    { name = "tqdm" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fa/42/8a95c5632080ae312c0498744b2b852195e10b05a20b1be11c5141092f4c/huggingface_hub-0.33.2.tar.gz", hash = "sha256:84221defaec8fa09c090390cd68c78b88e3c4c2b7befba68d3dc5aacbc3c2c5f", size = 426637, upload-time = "2025-07-02T06:26:05.156Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/44/f4/5f3f22e762ad1965f01122b42dae5bf0e009286e2dba601ce1d0dba72424/huggingface_hub-0.33.2-py3-none-any.whl", hash = "sha256:3749498bfa91e8cde2ddc2c1db92c79981f40e66434c20133b39e5928ac9bcc5", size = 515373, upload-time = "2025-07-02T06:26:03.072Z" },
+]
+
+[[package]]
+name = "humanize"
+version = "4.12.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/22/d1/bbc4d251187a43f69844f7fd8941426549bbe4723e8ff0a7441796b0789f/humanize-4.12.3.tar.gz", hash = "sha256:8430be3a615106fdfceb0b2c1b41c4c98c6b0fc5cc59663a5539b111dd325fb0", size = 80514, upload-time = "2025-04-30T11:51:07.98Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/1e/62a2ec3104394a2975a2629eec89276ede9dbe717092f6966fcf963e1bf0/humanize-4.12.3-py3-none-any.whl", hash = "sha256:2cbf6370af06568fa6d2da77c86edb7886f3160ecd19ee1ffef07979efc597f6", size = 128487, upload-time = "2025-04-30T11:51:06.468Z" },
+]
+
+[[package]]
+name = "hyperframe"
+version = "6.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" },
+]
+
+[[package]]
+name = "idna"
+version = "3.10"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" },
+]
+
+[[package]]
+name = "importlib-metadata"
+version = "8.7.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "zipp" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" },
+]
+
+[[package]]
+name = "jinja2"
+version = "3.1.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markupsafe" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" },
+]
+
+[[package]]
+name = "jiter"
+version = "0.10.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ee/9d/ae7ddb4b8ab3fb1b51faf4deb36cb48a4fbbd7cb36bad6a5fca4741306f7/jiter-0.10.0.tar.gz", hash = "sha256:07a7142c38aacc85194391108dc91b5b57093c978a9932bd86a36862759d9500", size = 162759, upload-time = "2025-05-18T19:04:59.73Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/be/7e/4011b5c77bec97cb2b572f566220364e3e21b51c48c5bd9c4a9c26b41b67/jiter-0.10.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:cd2fb72b02478f06a900a5782de2ef47e0396b3e1f7d5aba30daeb1fce66f303", size = 317215, upload-time = "2025-05-18T19:03:04.303Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/4f/144c1b57c39692efc7ea7d8e247acf28e47d0912800b34d0ad815f6b2824/jiter-0.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32bb468e3af278f095d3fa5b90314728a6916d89ba3d0ffb726dd9bf7367285e", size = 322814, upload-time = "2025-05-18T19:03:06.433Z" },
+    { url = "https://files.pythonhosted.org/packages/63/1f/db977336d332a9406c0b1f0b82be6f71f72526a806cbb2281baf201d38e3/jiter-0.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa8b3e0068c26ddedc7abc6fac37da2d0af16b921e288a5a613f4b86f050354f", size = 345237, upload-time = "2025-05-18T19:03:07.833Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/1c/aa30a4a775e8a672ad7f21532bdbfb269f0706b39c6ff14e1f86bdd9e5ff/jiter-0.10.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:286299b74cc49e25cd42eea19b72aa82c515d2f2ee12d11392c56d8701f52224", size = 370999, upload-time = "2025-05-18T19:03:09.338Z" },
+    { url = "https://files.pythonhosted.org/packages/35/df/f8257abc4207830cb18880781b5f5b716bad5b2a22fb4330cfd357407c5b/jiter-0.10.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ed5649ceeaeffc28d87fb012d25a4cd356dcd53eff5acff1f0466b831dda2a7", size = 491109, upload-time = "2025-05-18T19:03:11.13Z" },
+    { url = "https://files.pythonhosted.org/packages/06/76/9e1516fd7b4278aa13a2cc7f159e56befbea9aa65c71586305e7afa8b0b3/jiter-0.10.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2ab0051160cb758a70716448908ef14ad476c3774bd03ddce075f3c1f90a3d6", size = 388608, upload-time = "2025-05-18T19:03:12.911Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/64/67750672b4354ca20ca18d3d1ccf2c62a072e8a2d452ac3cf8ced73571ef/jiter-0.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03997d2f37f6b67d2f5c475da4412be584e1cec273c1cfc03d642c46db43f8cf", size = 352454, upload-time = "2025-05-18T19:03:14.741Z" },
+    { url = "https://files.pythonhosted.org/packages/96/4d/5c4e36d48f169a54b53a305114be3efa2bbffd33b648cd1478a688f639c1/jiter-0.10.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c404a99352d839fed80d6afd6c1d66071f3bacaaa5c4268983fc10f769112e90", size = 391833, upload-time = "2025-05-18T19:03:16.426Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/de/ce4a6166a78810bd83763d2fa13f85f73cbd3743a325469a4a9289af6dae/jiter-0.10.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66e989410b6666d3ddb27a74c7e50d0829704ede652fd4c858e91f8d64b403d0", size = 523646, upload-time = "2025-05-18T19:03:17.704Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/a6/3bc9acce53466972964cf4ad85efecb94f9244539ab6da1107f7aed82934/jiter-0.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b532d3af9ef4f6374609a3bcb5e05a1951d3bf6190dc6b176fdb277c9bbf15ee", size = 514735, upload-time = "2025-05-18T19:03:19.44Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/d8/243c2ab8426a2a4dea85ba2a2ba43df379ccece2145320dfd4799b9633c5/jiter-0.10.0-cp310-cp310-win32.whl", hash = "sha256:da9be20b333970e28b72edc4dff63d4fec3398e05770fb3205f7fb460eb48dd4", size = 210747, upload-time = "2025-05-18T19:03:21.184Z" },
+    { url = "https://files.pythonhosted.org/packages/37/7a/8021bd615ef7788b98fc76ff533eaac846322c170e93cbffa01979197a45/jiter-0.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:f59e533afed0c5b0ac3eba20d2548c4a550336d8282ee69eb07b37ea526ee4e5", size = 207484, upload-time = "2025-05-18T19:03:23.046Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/dd/6cefc6bd68b1c3c979cecfa7029ab582b57690a31cd2f346c4d0ce7951b6/jiter-0.10.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3bebe0c558e19902c96e99217e0b8e8b17d570906e72ed8a87170bc290b1e978", size = 317473, upload-time = "2025-05-18T19:03:25.942Z" },
+    { url = "https://files.pythonhosted.org/packages/be/cf/fc33f5159ce132be1d8dd57251a1ec7a631c7df4bd11e1cd198308c6ae32/jiter-0.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:558cc7e44fd8e507a236bee6a02fa17199ba752874400a0ca6cd6e2196cdb7dc", size = 321971, upload-time = "2025-05-18T19:03:27.255Z" },
+    { url = "https://files.pythonhosted.org/packages/68/a4/da3f150cf1d51f6c472616fb7650429c7ce053e0c962b41b68557fdf6379/jiter-0.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d613e4b379a07d7c8453c5712ce7014e86c6ac93d990a0b8e7377e18505e98d", size = 345574, upload-time = "2025-05-18T19:03:28.63Z" },
+    { url = "https://files.pythonhosted.org/packages/84/34/6e8d412e60ff06b186040e77da5f83bc158e9735759fcae65b37d681f28b/jiter-0.10.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f62cf8ba0618eda841b9bf61797f21c5ebd15a7a1e19daab76e4e4b498d515b2", size = 371028, upload-time = "2025-05-18T19:03:30.292Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/d9/9ee86173aae4576c35a2f50ae930d2ccb4c4c236f6cb9353267aa1d626b7/jiter-0.10.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:919d139cdfa8ae8945112398511cb7fca58a77382617d279556b344867a37e61", size = 491083, upload-time = "2025-05-18T19:03:31.654Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/2c/f955de55e74771493ac9e188b0f731524c6a995dffdcb8c255b89c6fb74b/jiter-0.10.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13ddbc6ae311175a3b03bd8994881bc4635c923754932918e18da841632349db", size = 388821, upload-time = "2025-05-18T19:03:33.184Z" },
+    { url = "https://files.pythonhosted.org/packages/81/5a/0e73541b6edd3f4aada586c24e50626c7815c561a7ba337d6a7eb0a915b4/jiter-0.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c440ea003ad10927a30521a9062ce10b5479592e8a70da27f21eeb457b4a9c5", size = 352174, upload-time = "2025-05-18T19:03:34.965Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/c0/61eeec33b8c75b31cae42be14d44f9e6fe3ac15a4e58010256ac3abf3638/jiter-0.10.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc347c87944983481e138dea467c0551080c86b9d21de6ea9306efb12ca8f606", size = 391869, upload-time = "2025-05-18T19:03:36.436Z" },
+    { url = "https://files.pythonhosted.org/packages/41/22/5beb5ee4ad4ef7d86f5ea5b4509f680a20706c4a7659e74344777efb7739/jiter-0.10.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:13252b58c1f4d8c5b63ab103c03d909e8e1e7842d302473f482915d95fefd605", size = 523741, upload-time = "2025-05-18T19:03:38.168Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/10/768e8818538e5817c637b0df52e54366ec4cebc3346108a4457ea7a98f32/jiter-0.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7d1bbf3c465de4a24ab12fb7766a0003f6f9bce48b8b6a886158c4d569452dc5", size = 514527, upload-time = "2025-05-18T19:03:39.577Z" },
+    { url = "https://files.pythonhosted.org/packages/73/6d/29b7c2dc76ce93cbedabfd842fc9096d01a0550c52692dfc33d3cc889815/jiter-0.10.0-cp311-cp311-win32.whl", hash = "sha256:db16e4848b7e826edca4ccdd5b145939758dadf0dc06e7007ad0e9cfb5928ae7", size = 210765, upload-time = "2025-05-18T19:03:41.271Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/c9/d394706deb4c660137caf13e33d05a031d734eb99c051142e039d8ceb794/jiter-0.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c9c1d5f10e18909e993f9641f12fe1c77b3e9b533ee94ffa970acc14ded3812", size = 209234, upload-time = "2025-05-18T19:03:42.918Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/b5/348b3313c58f5fbfb2194eb4d07e46a35748ba6e5b3b3046143f3040bafa/jiter-0.10.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1e274728e4a5345a6dde2d343c8da018b9d4bd4350f5a472fa91f66fda44911b", size = 312262, upload-time = "2025-05-18T19:03:44.637Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/4a/6a2397096162b21645162825f058d1709a02965606e537e3304b02742e9b/jiter-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7202ae396446c988cb2a5feb33a543ab2165b786ac97f53b59aafb803fef0744", size = 320124, upload-time = "2025-05-18T19:03:46.341Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/85/1ce02cade7516b726dd88f59a4ee46914bf79d1676d1228ef2002ed2f1c9/jiter-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23ba7722d6748b6920ed02a8f1726fb4b33e0fd2f3f621816a8b486c66410ab2", size = 345330, upload-time = "2025-05-18T19:03:47.596Z" },
+    { url = "https://files.pythonhosted.org/packages/75/d0/bb6b4f209a77190ce10ea8d7e50bf3725fc16d3372d0a9f11985a2b23eff/jiter-0.10.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:371eab43c0a288537d30e1f0b193bc4eca90439fc08a022dd83e5e07500ed026", size = 369670, upload-time = "2025-05-18T19:03:49.334Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/f5/a61787da9b8847a601e6827fbc42ecb12be2c925ced3252c8ffcb56afcaf/jiter-0.10.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c675736059020365cebc845a820214765162728b51ab1e03a1b7b3abb70f74c", size = 489057, upload-time = "2025-05-18T19:03:50.66Z" },
+    { url = "https://files.pythonhosted.org/packages/12/e4/6f906272810a7b21406c760a53aadbe52e99ee070fc5c0cb191e316de30b/jiter-0.10.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c5867d40ab716e4684858e4887489685968a47e3ba222e44cde6e4a2154f959", size = 389372, upload-time = "2025-05-18T19:03:51.98Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/ba/77013b0b8ba904bf3762f11e0129b8928bff7f978a81838dfcc958ad5728/jiter-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395bb9a26111b60141757d874d27fdea01b17e8fac958b91c20128ba8f4acc8a", size = 352038, upload-time = "2025-05-18T19:03:53.703Z" },
+    { url = "https://files.pythonhosted.org/packages/67/27/c62568e3ccb03368dbcc44a1ef3a423cb86778a4389e995125d3d1aaa0a4/jiter-0.10.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6842184aed5cdb07e0c7e20e5bdcfafe33515ee1741a6835353bb45fe5d1bd95", size = 391538, upload-time = "2025-05-18T19:03:55.046Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/72/0d6b7e31fc17a8fdce76164884edef0698ba556b8eb0af9546ae1a06b91d/jiter-0.10.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:62755d1bcea9876770d4df713d82606c8c1a3dca88ff39046b85a048566d56ea", size = 523557, upload-time = "2025-05-18T19:03:56.386Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/09/bc1661fbbcbeb6244bd2904ff3a06f340aa77a2b94e5a7373fd165960ea3/jiter-0.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:533efbce2cacec78d5ba73a41756beff8431dfa1694b6346ce7af3a12c42202b", size = 514202, upload-time = "2025-05-18T19:03:57.675Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/84/5a5d5400e9d4d54b8004c9673bbe4403928a00d28529ff35b19e9d176b19/jiter-0.10.0-cp312-cp312-win32.whl", hash = "sha256:8be921f0cadd245e981b964dfbcd6fd4bc4e254cdc069490416dd7a2632ecc01", size = 211781, upload-time = "2025-05-18T19:03:59.025Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/52/7ec47455e26f2d6e5f2ea4951a0652c06e5b995c291f723973ae9e724a65/jiter-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:a7c7d785ae9dda68c2678532a5a1581347e9c15362ae9f6e68f3fdbfb64f2e49", size = 206176, upload-time = "2025-05-18T19:04:00.305Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/b0/279597e7a270e8d22623fea6c5d4eeac328e7d95c236ed51a2b884c54f70/jiter-0.10.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e0588107ec8e11b6f5ef0e0d656fb2803ac6cf94a96b2b9fc675c0e3ab5e8644", size = 311617, upload-time = "2025-05-18T19:04:02.078Z" },
+    { url = "https://files.pythonhosted.org/packages/91/e3/0916334936f356d605f54cc164af4060e3e7094364add445a3bc79335d46/jiter-0.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cafc4628b616dc32530c20ee53d71589816cf385dd9449633e910d596b1f5c8a", size = 318947, upload-time = "2025-05-18T19:04:03.347Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/8e/fd94e8c02d0e94539b7d669a7ebbd2776e51f329bb2c84d4385e8063a2ad/jiter-0.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:520ef6d981172693786a49ff5b09eda72a42e539f14788124a07530f785c3ad6", size = 344618, upload-time = "2025-05-18T19:04:04.709Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/b0/f9f0a2ec42c6e9c2e61c327824687f1e2415b767e1089c1d9135f43816bd/jiter-0.10.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:554dedfd05937f8fc45d17ebdf298fe7e0c77458232bcb73d9fbbf4c6455f5b3", size = 368829, upload-time = "2025-05-18T19:04:06.912Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/57/5bbcd5331910595ad53b9fd0c610392ac68692176f05ae48d6ce5c852967/jiter-0.10.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5bc299da7789deacf95f64052d97f75c16d4fc8c4c214a22bf8d859a4288a1c2", size = 491034, upload-time = "2025-05-18T19:04:08.222Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/be/c393df00e6e6e9e623a73551774449f2f23b6ec6a502a3297aeeece2c65a/jiter-0.10.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5161e201172de298a8a1baad95eb85db4fb90e902353b1f6a41d64ea64644e25", size = 388529, upload-time = "2025-05-18T19:04:09.566Z" },
+    { url = "https://files.pythonhosted.org/packages/42/3e/df2235c54d365434c7f150b986a6e35f41ebdc2f95acea3036d99613025d/jiter-0.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e2227db6ba93cb3e2bf67c87e594adde0609f146344e8207e8730364db27041", size = 350671, upload-time = "2025-05-18T19:04:10.98Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/77/71b0b24cbcc28f55ab4dbfe029f9a5b73aeadaba677843fc6dc9ed2b1d0a/jiter-0.10.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:15acb267ea5e2c64515574b06a8bf393fbfee6a50eb1673614aa45f4613c0cca", size = 390864, upload-time = "2025-05-18T19:04:12.722Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/d3/ef774b6969b9b6178e1d1e7a89a3bd37d241f3d3ec5f8deb37bbd203714a/jiter-0.10.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:901b92f2e2947dc6dfcb52fd624453862e16665ea909a08398dde19c0731b7f4", size = 522989, upload-time = "2025-05-18T19:04:14.261Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/41/9becdb1d8dd5d854142f45a9d71949ed7e87a8e312b0bede2de849388cb9/jiter-0.10.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d0cb9a125d5a3ec971a094a845eadde2db0de85b33c9f13eb94a0c63d463879e", size = 513495, upload-time = "2025-05-18T19:04:15.603Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/36/3468e5a18238bdedae7c4d19461265b5e9b8e288d3f86cd89d00cbb48686/jiter-0.10.0-cp313-cp313-win32.whl", hash = "sha256:48a403277ad1ee208fb930bdf91745e4d2d6e47253eedc96e2559d1e6527006d", size = 211289, upload-time = "2025-05-18T19:04:17.541Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/07/1c96b623128bcb913706e294adb5f768fb7baf8db5e1338ce7b4ee8c78ef/jiter-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:75f9eb72ecb640619c29bf714e78c9c46c9c4eaafd644bf78577ede459f330d4", size = 205074, upload-time = "2025-05-18T19:04:19.21Z" },
+    { url = "https://files.pythonhosted.org/packages/54/46/caa2c1342655f57d8f0f2519774c6d67132205909c65e9aa8255e1d7b4f4/jiter-0.10.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:28ed2a4c05a1f32ef0e1d24c2611330219fed727dae01789f4a335617634b1ca", size = 318225, upload-time = "2025-05-18T19:04:20.583Z" },
+    { url = "https://files.pythonhosted.org/packages/43/84/c7d44c75767e18946219ba2d703a5a32ab37b0bc21886a97bc6062e4da42/jiter-0.10.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a4c418b1ec86a195f1ca69da8b23e8926c752b685af665ce30777233dfe070", size = 350235, upload-time = "2025-05-18T19:04:22.363Z" },
+    { url = "https://files.pythonhosted.org/packages/01/16/f5a0135ccd968b480daad0e6ab34b0c7c5ba3bc447e5088152696140dcb3/jiter-0.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:d7bfed2fe1fe0e4dda6ef682cee888ba444b21e7a6553e03252e4feb6cf0adca", size = 207278, upload-time = "2025-05-18T19:04:23.627Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/9b/1d646da42c3de6c2188fdaa15bce8ecb22b635904fc68be025e21249ba44/jiter-0.10.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:5e9251a5e83fab8d87799d3e1a46cb4b7f2919b895c6f4483629ed2446f66522", size = 310866, upload-time = "2025-05-18T19:04:24.891Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/0e/26538b158e8a7c7987e94e7aeb2999e2e82b1f9d2e1f6e9874ddf71ebda0/jiter-0.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:023aa0204126fe5b87ccbcd75c8a0d0261b9abdbbf46d55e7ae9f8e22424eeb8", size = 318772, upload-time = "2025-05-18T19:04:26.161Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/fb/d302893151caa1c2636d6574d213e4b34e31fd077af6050a9c5cbb42f6fb/jiter-0.10.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c189c4f1779c05f75fc17c0c1267594ed918996a231593a21a5ca5438445216", size = 344534, upload-time = "2025-05-18T19:04:27.495Z" },
+    { url = "https://files.pythonhosted.org/packages/01/d8/5780b64a149d74e347c5128d82176eb1e3241b1391ac07935693466d6219/jiter-0.10.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:15720084d90d1098ca0229352607cd68256c76991f6b374af96f36920eae13c4", size = 369087, upload-time = "2025-05-18T19:04:28.896Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/5b/f235a1437445160e777544f3ade57544daf96ba7e96c1a5b24a6f7ac7004/jiter-0.10.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4f2fb68e5f1cfee30e2b2a09549a00683e0fde4c6a2ab88c94072fc33cb7426", size = 490694, upload-time = "2025-05-18T19:04:30.183Z" },
+    { url = "https://files.pythonhosted.org/packages/85/a9/9c3d4617caa2ff89cf61b41e83820c27ebb3f7b5fae8a72901e8cd6ff9be/jiter-0.10.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce541693355fc6da424c08b7edf39a2895f58d6ea17d92cc2b168d20907dee12", size = 388992, upload-time = "2025-05-18T19:04:32.028Z" },
+    { url = "https://files.pythonhosted.org/packages/68/b1/344fd14049ba5c94526540af7eb661871f9c54d5f5601ff41a959b9a0bbd/jiter-0.10.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31c50c40272e189d50006ad5c73883caabb73d4e9748a688b216e85a9a9ca3b9", size = 351723, upload-time = "2025-05-18T19:04:33.467Z" },
+    { url = "https://files.pythonhosted.org/packages/41/89/4c0e345041186f82a31aee7b9d4219a910df672b9fef26f129f0cda07a29/jiter-0.10.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fa3402a2ff9815960e0372a47b75c76979d74402448509ccd49a275fa983ef8a", size = 392215, upload-time = "2025-05-18T19:04:34.827Z" },
+    { url = "https://files.pythonhosted.org/packages/55/58/ee607863e18d3f895feb802154a2177d7e823a7103f000df182e0f718b38/jiter-0.10.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:1956f934dca32d7bb647ea21d06d93ca40868b505c228556d3373cbd255ce853", size = 522762, upload-time = "2025-05-18T19:04:36.19Z" },
+    { url = "https://files.pythonhosted.org/packages/15/d0/9123fb41825490d16929e73c212de9a42913d68324a8ce3c8476cae7ac9d/jiter-0.10.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:fcedb049bdfc555e261d6f65a6abe1d5ad68825b7202ccb9692636c70fcced86", size = 513427, upload-time = "2025-05-18T19:04:37.544Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/b3/2bd02071c5a2430d0b70403a34411fc519c2f227da7b03da9ba6a956f931/jiter-0.10.0-cp314-cp314-win32.whl", hash = "sha256:ac509f7eccca54b2a29daeb516fb95b6f0bd0d0d8084efaf8ed5dfc7b9f0b357", size = 210127, upload-time = "2025-05-18T19:04:38.837Z" },
+    { url = "https://files.pythonhosted.org/packages/03/0c/5fe86614ea050c3ecd728ab4035534387cd41e7c1855ef6c031f1ca93e3f/jiter-0.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5ed975b83a2b8639356151cef5c0d597c68376fc4922b45d0eb384ac058cfa00", size = 318527, upload-time = "2025-05-18T19:04:40.612Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" },
+]
+
+[[package]]
+name = "joblib"
+version = "1.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" },
+]
+
+[[package]]
+name = "jsonschema"
+version = "4.24.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "attrs" },
+    { name = "jsonschema-specifications" },
+    { name = "referencing" },
+    { name = "rpds-py" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/bf/d3/1cf5326b923a53515d8f3a2cd442e6d7e94fcc444716e879ea70a0ce3177/jsonschema-4.24.0.tar.gz", hash = "sha256:0b4e8069eb12aedfa881333004bccaec24ecef5a8a6a4b6df142b2cc9599d196", size = 353480, upload-time = "2025-05-26T18:48:10.459Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a2/3d/023389198f69c722d039351050738d6755376c8fd343e91dc493ea485905/jsonschema-4.24.0-py3-none-any.whl", hash = "sha256:a462455f19f5faf404a7902952b6f0e3ce868f3ee09a359b05eca6673bd8412d", size = 88709, upload-time = "2025-05-26T18:48:08.417Z" },
+]
+
+[[package]]
+name = "jsonschema-specifications"
+version = "2025.4.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "referencing" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/bf/ce/46fbd9c8119cfc3581ee5643ea49464d168028cfb5caff5fc0596d0cf914/jsonschema_specifications-2025.4.1.tar.gz", hash = "sha256:630159c9f4dbea161a6a2205c3011cc4f18ff381b189fff48bb39b9bf26ae608", size = 15513, upload-time = "2025-04-23T12:34:07.418Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/01/0e/b27cdbaccf30b890c40ed1da9fd4a3593a5cf94dae54fb34f8a4b74fcd3f/jsonschema_specifications-2025.4.1-py3-none-any.whl", hash = "sha256:4653bffbd6584f7de83a67e0d620ef16900b390ddc7939d56684d6c81e33f1af", size = 18437, upload-time = "2025-04-23T12:34:05.422Z" },
+]
+
+[[package]]
+name = "lark"
+version = "1.2.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" },
+]
+
+[[package]]
+name = "litellm"
+version = "1.74.0.post1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "aiohttp" },
+    { name = "click" },
+    { name = "httpx" },
+    { name = "importlib-metadata" },
+    { name = "jinja2" },
+    { name = "jsonschema" },
+    { name = "openai" },
+    { name = "pydantic" },
+    { name = "python-dotenv" },
+    { name = "tiktoken" },
+    { name = "tokenizers" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4d/68/a58b31f25c42fe052d7e7a0fc27d96d20c88b171070acc7b5d3cd553db0e/litellm-1.74.0.post1.tar.gz", hash = "sha256:417b08d0584ffc2788386261f1b3dea6e0b1b09b0231f48c46426c0c7a9b7278", size = 9032360, upload-time = "2025-07-07T17:48:08.376Z" }
+
+[[package]]
+name = "lxml"
+version = "5.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/76/3d/14e82fc7c8fb1b7761f7e748fd47e2ec8276d137b6acfe5a4bb73853e08f/lxml-5.4.0.tar.gz", hash = "sha256:d12832e1dbea4be280b22fd0ea7c9b87f0d8fc51ba06e92dc62d52f804f78ebd", size = 3679479, upload-time = "2025-04-23T01:50:29.322Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f5/1f/a3b6b74a451ceb84b471caa75c934d2430a4d84395d38ef201d539f38cd1/lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c", size = 8076838, upload-time = "2025-04-23T01:44:29.325Z" },
+    { url = "https://files.pythonhosted.org/packages/36/af/a567a55b3e47135b4d1f05a1118c24529104c003f95851374b3748139dc1/lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7", size = 4381827, upload-time = "2025-04-23T01:44:33.345Z" },
+    { url = "https://files.pythonhosted.org/packages/50/ba/4ee47d24c675932b3eb5b6de77d0f623c2db6dc466e7a1f199792c5e3e3a/lxml-5.4.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:696ea9e87442467819ac22394ca36cb3d01848dad1be6fac3fb612d3bd5a12cf", size = 5204098, upload-time = "2025-04-23T01:44:35.809Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/0f/b4db6dfebfefe3abafe360f42a3d471881687fd449a0b86b70f1f2683438/lxml-5.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef80aeac414f33c24b3815ecd560cee272786c3adfa5f31316d8b349bfade28", size = 4930261, upload-time = "2025-04-23T01:44:38.271Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/1f/0bb1bae1ce056910f8db81c6aba80fec0e46c98d77c0f59298c70cd362a3/lxml-5.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b9c2754cef6963f3408ab381ea55f47dabc6f78f4b8ebb0f0b25cf1ac1f7609", size = 5529621, upload-time = "2025-04-23T01:44:40.921Z" },
+    { url = "https://files.pythonhosted.org/packages/21/f5/e7b66a533fc4a1e7fa63dd22a1ab2ec4d10319b909211181e1ab3e539295/lxml-5.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a62cc23d754bb449d63ff35334acc9f5c02e6dae830d78dab4dd12b78a524f4", size = 4983231, upload-time = "2025-04-23T01:44:43.871Z" },
+    { url = "https://files.pythonhosted.org/packages/11/39/a38244b669c2d95a6a101a84d3c85ba921fea827e9e5483e93168bf1ccb2/lxml-5.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f82125bc7203c5ae8633a7d5d20bcfdff0ba33e436e4ab0abc026a53a8960b7", size = 5084279, upload-time = "2025-04-23T01:44:46.632Z" },
+    { url = "https://files.pythonhosted.org/packages/db/64/48cac242347a09a07740d6cee7b7fd4663d5c1abd65f2e3c60420e231b27/lxml-5.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b67319b4aef1a6c56576ff544b67a2a6fbd7eaee485b241cabf53115e8908b8f", size = 4927405, upload-time = "2025-04-23T01:44:49.843Z" },
+    { url = "https://files.pythonhosted.org/packages/98/89/97442835fbb01d80b72374f9594fe44f01817d203fa056e9906128a5d896/lxml-5.4.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:a8ef956fce64c8551221f395ba21d0724fed6b9b6242ca4f2f7beb4ce2f41997", size = 5550169, upload-time = "2025-04-23T01:44:52.791Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/97/164ca398ee654eb21f29c6b582685c6c6b9d62d5213abc9b8380278e9c0a/lxml-5.4.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:0a01ce7d8479dce84fc03324e3b0c9c90b1ece9a9bb6a1b6c9025e7e4520e78c", size = 5062691, upload-time = "2025-04-23T01:44:56.108Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/bc/712b96823d7feb53482d2e4f59c090fb18ec7b0d0b476f353b3085893cda/lxml-5.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:91505d3ddebf268bb1588eb0f63821f738d20e1e7f05d3c647a5ca900288760b", size = 5133503, upload-time = "2025-04-23T01:44:59.222Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/55/a62a39e8f9da2a8b6002603475e3c57c870cd9c95fd4b94d4d9ac9036055/lxml-5.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a3bcdde35d82ff385f4ede021df801b5c4a5bcdfb61ea87caabcebfc4945dc1b", size = 4999346, upload-time = "2025-04-23T01:45:02.088Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/47/a393728ae001b92bb1a9e095e570bf71ec7f7fbae7688a4792222e56e5b9/lxml-5.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aea7c06667b987787c7d1f5e1dfcd70419b711cdb47d6b4bb4ad4b76777a0563", size = 5627139, upload-time = "2025-04-23T01:45:04.582Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/5f/9dcaaad037c3e642a7ea64b479aa082968de46dd67a8293c541742b6c9db/lxml-5.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a7fb111eef4d05909b82152721a59c1b14d0f365e2be4c742a473c5d7372f4f5", size = 5465609, upload-time = "2025-04-23T01:45:07.649Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/0a/ebcae89edf27e61c45023005171d0ba95cb414ee41c045ae4caf1b8487fd/lxml-5.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:43d549b876ce64aa18b2328faff70f5877f8c6dede415f80a2f799d31644d776", size = 5192285, upload-time = "2025-04-23T01:45:10.456Z" },
+    { url = "https://files.pythonhosted.org/packages/42/ad/cc8140ca99add7d85c92db8b2354638ed6d5cc0e917b21d36039cb15a238/lxml-5.4.0-cp310-cp310-win32.whl", hash = "sha256:75133890e40d229d6c5837b0312abbe5bac1c342452cf0e12523477cd3aa21e7", size = 3477507, upload-time = "2025-04-23T01:45:12.474Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/39/597ce090da1097d2aabd2f9ef42187a6c9c8546d67c419ce61b88b336c85/lxml-5.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:de5b4e1088523e2b6f730d0509a9a813355b7f5659d70eb4f319c76beea2e250", size = 3805104, upload-time = "2025-04-23T01:45:15.104Z" },
+    { url = "https://files.pythonhosted.org/packages/81/2d/67693cc8a605a12e5975380d7ff83020dcc759351b5a066e1cced04f797b/lxml-5.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:98a3912194c079ef37e716ed228ae0dcb960992100461b704aea4e93af6b0bb9", size = 8083240, upload-time = "2025-04-23T01:45:18.566Z" },
+    { url = "https://files.pythonhosted.org/packages/73/53/b5a05ab300a808b72e848efd152fe9c022c0181b0a70b8bca1199f1bed26/lxml-5.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ea0252b51d296a75f6118ed0d8696888e7403408ad42345d7dfd0d1e93309a7", size = 4387685, upload-time = "2025-04-23T01:45:21.387Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/cb/1a3879c5f512bdcd32995c301886fe082b2edd83c87d41b6d42d89b4ea4d/lxml-5.4.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b92b69441d1bd39f4940f9eadfa417a25862242ca2c396b406f9272ef09cdcaa", size = 4991164, upload-time = "2025-04-23T01:45:23.849Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/94/bbc66e42559f9d04857071e3b3d0c9abd88579367fd2588a4042f641f57e/lxml-5.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20e16c08254b9b6466526bc1828d9370ee6c0d60a4b64836bc3ac2917d1e16df", size = 4746206, upload-time = "2025-04-23T01:45:26.361Z" },
+    { url = "https://files.pythonhosted.org/packages/66/95/34b0679bee435da2d7cae895731700e519a8dfcab499c21662ebe671603e/lxml-5.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7605c1c32c3d6e8c990dd28a0970a3cbbf1429d5b92279e37fda05fb0c92190e", size = 5342144, upload-time = "2025-04-23T01:45:28.939Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/5d/abfcc6ab2fa0be72b2ba938abdae1f7cad4c632f8d552683ea295d55adfb/lxml-5.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecf4c4b83f1ab3d5a7ace10bafcb6f11df6156857a3c418244cef41ca9fa3e44", size = 4825124, upload-time = "2025-04-23T01:45:31.361Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/78/6bd33186c8863b36e084f294fc0a5e5eefe77af95f0663ef33809cc1c8aa/lxml-5.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cef4feae82709eed352cd7e97ae062ef6ae9c7b5dbe3663f104cd2c0e8d94ba", size = 4876520, upload-time = "2025-04-23T01:45:34.191Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/74/4d7ad4839bd0fc64e3d12da74fc9a193febb0fae0ba6ebd5149d4c23176a/lxml-5.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:df53330a3bff250f10472ce96a9af28628ff1f4efc51ccba351a8820bca2a8ba", size = 4765016, upload-time = "2025-04-23T01:45:36.7Z" },
+    { url = "https://files.pythonhosted.org/packages/24/0d/0a98ed1f2471911dadfc541003ac6dd6879fc87b15e1143743ca20f3e973/lxml-5.4.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:aefe1a7cb852fa61150fcb21a8c8fcea7b58c4cb11fbe59c97a0a4b31cae3c8c", size = 5362884, upload-time = "2025-04-23T01:45:39.291Z" },
+    { url = "https://files.pythonhosted.org/packages/48/de/d4f7e4c39740a6610f0f6959052b547478107967362e8424e1163ec37ae8/lxml-5.4.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ef5a7178fcc73b7d8c07229e89f8eb45b2908a9238eb90dcfc46571ccf0383b8", size = 4902690, upload-time = "2025-04-23T01:45:42.386Z" },
+    { url = "https://files.pythonhosted.org/packages/07/8c/61763abd242af84f355ca4ef1ee096d3c1b7514819564cce70fd18c22e9a/lxml-5.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d2ed1b3cb9ff1c10e6e8b00941bb2e5bb568b307bfc6b17dffbbe8be5eecba86", size = 4944418, upload-time = "2025-04-23T01:45:46.051Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/c5/6d7e3b63e7e282619193961a570c0a4c8a57fe820f07ca3fe2f6bd86608a/lxml-5.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:72ac9762a9f8ce74c9eed4a4e74306f2f18613a6b71fa065495a67ac227b3056", size = 4827092, upload-time = "2025-04-23T01:45:48.943Z" },
+    { url = "https://files.pythonhosted.org/packages/71/4a/e60a306df54680b103348545706a98a7514a42c8b4fbfdcaa608567bb065/lxml-5.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f5cb182f6396706dc6cc1896dd02b1c889d644c081b0cdec38747573db88a7d7", size = 5418231, upload-time = "2025-04-23T01:45:51.481Z" },
+    { url = "https://files.pythonhosted.org/packages/27/f2/9754aacd6016c930875854f08ac4b192a47fe19565f776a64004aa167521/lxml-5.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3a3178b4873df8ef9457a4875703488eb1622632a9cee6d76464b60e90adbfcd", size = 5261798, upload-time = "2025-04-23T01:45:54.146Z" },
+    { url = "https://files.pythonhosted.org/packages/38/a2/0c49ec6941428b1bd4f280650d7b11a0f91ace9db7de32eb7aa23bcb39ff/lxml-5.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e094ec83694b59d263802ed03a8384594fcce477ce484b0cbcd0008a211ca751", size = 4988195, upload-time = "2025-04-23T01:45:56.685Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/75/87a3963a08eafc46a86c1131c6e28a4de103ba30b5ae903114177352a3d7/lxml-5.4.0-cp311-cp311-win32.whl", hash = "sha256:4329422de653cdb2b72afa39b0aa04252fca9071550044904b2e7036d9d97fe4", size = 3474243, upload-time = "2025-04-23T01:45:58.863Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/f9/1f0964c4f6c2be861c50db380c554fb8befbea98c6404744ce243a3c87ef/lxml-5.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd3be6481ef54b8cfd0e1e953323b7aa9d9789b94842d0e5b142ef4bb7999539", size = 3815197, upload-time = "2025-04-23T01:46:01.096Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/4c/d101ace719ca6a4ec043eb516fcfcb1b396a9fccc4fcd9ef593df34ba0d5/lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b5aff6f3e818e6bdbbb38e5967520f174b18f539c2b9de867b1e7fde6f8d95a4", size = 8127392, upload-time = "2025-04-23T01:46:04.09Z" },
+    { url = "https://files.pythonhosted.org/packages/11/84/beddae0cec4dd9ddf46abf156f0af451c13019a0fa25d7445b655ba5ccb7/lxml-5.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:942a5d73f739ad7c452bf739a62a0f83e2578afd6b8e5406308731f4ce78b16d", size = 4415103, upload-time = "2025-04-23T01:46:07.227Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/25/d0d93a4e763f0462cccd2b8a665bf1e4343dd788c76dcfefa289d46a38a9/lxml-5.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:460508a4b07364d6abf53acaa0a90b6d370fafde5693ef37602566613a9b0779", size = 5024224, upload-time = "2025-04-23T01:46:10.237Z" },
+    { url = "https://files.pythonhosted.org/packages/31/ce/1df18fb8f7946e7f3388af378b1f34fcf253b94b9feedb2cec5969da8012/lxml-5.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529024ab3a505fed78fe3cc5ddc079464e709f6c892733e3f5842007cec8ac6e", size = 4769913, upload-time = "2025-04-23T01:46:12.757Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/62/f4a6c60ae7c40d43657f552f3045df05118636be1165b906d3423790447f/lxml-5.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ca56ebc2c474e8f3d5761debfd9283b8b18c76c4fc0967b74aeafba1f5647f9", size = 5290441, upload-time = "2025-04-23T01:46:16.037Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/aa/04f00009e1e3a77838c7fc948f161b5d2d5de1136b2b81c712a263829ea4/lxml-5.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a81e1196f0a5b4167a8dafe3a66aa67c4addac1b22dc47947abd5d5c7a3f24b5", size = 4820165, upload-time = "2025-04-23T01:46:19.137Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/1f/e0b2f61fa2404bf0f1fdf1898377e5bd1b74cc9b2cf2c6ba8509b8f27990/lxml-5.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b8686694423ddae324cf614e1b9659c2edb754de617703c3d29ff568448df5", size = 4932580, upload-time = "2025-04-23T01:46:21.963Z" },
+    { url = "https://files.pythonhosted.org/packages/24/a2/8263f351b4ffe0ed3e32ea7b7830f845c795349034f912f490180d88a877/lxml-5.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c5681160758d3f6ac5b4fea370495c48aac0989d6a0f01bb9a72ad8ef5ab75c4", size = 4759493, upload-time = "2025-04-23T01:46:24.316Z" },
+    { url = "https://files.pythonhosted.org/packages/05/00/41db052f279995c0e35c79d0f0fc9f8122d5b5e9630139c592a0b58c71b4/lxml-5.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:2dc191e60425ad70e75a68c9fd90ab284df64d9cd410ba8d2b641c0c45bc006e", size = 5324679, upload-time = "2025-04-23T01:46:27.097Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/be/ee99e6314cdef4587617d3b3b745f9356d9b7dd12a9663c5f3b5734b64ba/lxml-5.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:67f779374c6b9753ae0a0195a892a1c234ce8416e4448fe1e9f34746482070a7", size = 4890691, upload-time = "2025-04-23T01:46:30.009Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/36/239820114bf1d71f38f12208b9c58dec033cbcf80101cde006b9bde5cffd/lxml-5.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:79d5bfa9c1b455336f52343130b2067164040604e41f6dc4d8313867ed540079", size = 4955075, upload-time = "2025-04-23T01:46:32.33Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/e1/1b795cc0b174efc9e13dbd078a9ff79a58728a033142bc6d70a1ee8fc34d/lxml-5.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d3c30ba1c9b48c68489dc1829a6eede9873f52edca1dda900066542528d6b20", size = 4838680, upload-time = "2025-04-23T01:46:34.852Z" },
+    { url = "https://files.pythonhosted.org/packages/72/48/3c198455ca108cec5ae3662ae8acd7fd99476812fd712bb17f1b39a0b589/lxml-5.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1af80c6316ae68aded77e91cd9d80648f7dd40406cef73df841aa3c36f6907c8", size = 5391253, upload-time = "2025-04-23T01:46:37.608Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/10/5bf51858971c51ec96cfc13e800a9951f3fd501686f4c18d7d84fe2d6352/lxml-5.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4d885698f5019abe0de3d352caf9466d5de2baded00a06ef3f1216c1a58ae78f", size = 5261651, upload-time = "2025-04-23T01:46:40.183Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/11/06710dd809205377da380546f91d2ac94bad9ff735a72b64ec029f706c85/lxml-5.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea53d51859b6c64e7c51d522c03cc2c48b9b5d6172126854cc7f01aa11f52bc", size = 5024315, upload-time = "2025-04-23T01:46:43.333Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/b0/15b6217834b5e3a59ebf7f53125e08e318030e8cc0d7310355e6edac98ef/lxml-5.4.0-cp312-cp312-win32.whl", hash = "sha256:d90b729fd2732df28130c064aac9bb8aff14ba20baa4aee7bd0795ff1187545f", size = 3486149, upload-time = "2025-04-23T01:46:45.684Z" },
+    { url = "https://files.pythonhosted.org/packages/91/1e/05ddcb57ad2f3069101611bd5f5084157d90861a2ef460bf42f45cced944/lxml-5.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1dc4ca99e89c335a7ed47d38964abcb36c5910790f9bd106f2a8fa2ee0b909d2", size = 3817095, upload-time = "2025-04-23T01:46:48.521Z" },
+    { url = "https://files.pythonhosted.org/packages/87/cb/2ba1e9dd953415f58548506fa5549a7f373ae55e80c61c9041b7fd09a38a/lxml-5.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:773e27b62920199c6197130632c18fb7ead3257fce1ffb7d286912e56ddb79e0", size = 8110086, upload-time = "2025-04-23T01:46:52.218Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/3e/6602a4dca3ae344e8609914d6ab22e52ce42e3e1638c10967568c5c1450d/lxml-5.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9c671845de9699904b1e9df95acfe8dfc183f2310f163cdaa91a3535af95de", size = 4404613, upload-time = "2025-04-23T01:46:55.281Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/72/bf00988477d3bb452bef9436e45aeea82bb40cdfb4684b83c967c53909c7/lxml-5.4.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9454b8d8200ec99a224df8854786262b1bd6461f4280064c807303c642c05e76", size = 5012008, upload-time = "2025-04-23T01:46:57.817Z" },
+    { url = "https://files.pythonhosted.org/packages/92/1f/93e42d93e9e7a44b2d3354c462cd784dbaaf350f7976b5d7c3f85d68d1b1/lxml-5.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cccd007d5c95279e529c146d095f1d39ac05139de26c098166c4beb9374b0f4d", size = 4760915, upload-time = "2025-04-23T01:47:00.745Z" },
+    { url = "https://files.pythonhosted.org/packages/45/0b/363009390d0b461cf9976a499e83b68f792e4c32ecef092f3f9ef9c4ba54/lxml-5.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0fce1294a0497edb034cb416ad3e77ecc89b313cff7adbee5334e4dc0d11f422", size = 5283890, upload-time = "2025-04-23T01:47:04.702Z" },
+    { url = "https://files.pythonhosted.org/packages/19/dc/6056c332f9378ab476c88e301e6549a0454dbee8f0ae16847414f0eccb74/lxml-5.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:24974f774f3a78ac12b95e3a20ef0931795ff04dbb16db81a90c37f589819551", size = 4812644, upload-time = "2025-04-23T01:47:07.833Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/8a/f8c66bbb23ecb9048a46a5ef9b495fd23f7543df642dabeebcb2eeb66592/lxml-5.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:497cab4d8254c2a90bf988f162ace2ddbfdd806fce3bda3f581b9d24c852e03c", size = 4921817, upload-time = "2025-04-23T01:47:10.317Z" },
+    { url = "https://files.pythonhosted.org/packages/04/57/2e537083c3f381f83d05d9b176f0d838a9e8961f7ed8ddce3f0217179ce3/lxml-5.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e794f698ae4c5084414efea0f5cc9f4ac562ec02d66e1484ff822ef97c2cadff", size = 4753916, upload-time = "2025-04-23T01:47:12.823Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/80/ea8c4072109a350848f1157ce83ccd9439601274035cd045ac31f47f3417/lxml-5.4.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:2c62891b1ea3094bb12097822b3d44b93fc6c325f2043c4d2736a8ff09e65f60", size = 5289274, upload-time = "2025-04-23T01:47:15.916Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/47/c4be287c48cdc304483457878a3f22999098b9a95f455e3c4bda7ec7fc72/lxml-5.4.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:142accb3e4d1edae4b392bd165a9abdee8a3c432a2cca193df995bc3886249c8", size = 4874757, upload-time = "2025-04-23T01:47:19.793Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/04/6ef935dc74e729932e39478e44d8cfe6a83550552eaa072b7c05f6f22488/lxml-5.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1a42b3a19346e5601d1b8296ff6ef3d76038058f311902edd574461e9c036982", size = 4947028, upload-time = "2025-04-23T01:47:22.401Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/f9/c33fc8daa373ef8a7daddb53175289024512b6619bc9de36d77dca3df44b/lxml-5.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4291d3c409a17febf817259cb37bc62cb7eb398bcc95c1356947e2871911ae61", size = 4834487, upload-time = "2025-04-23T01:47:25.513Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/30/fc92bb595bcb878311e01b418b57d13900f84c2b94f6eca9e5073ea756e6/lxml-5.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4f5322cf38fe0e21c2d73901abf68e6329dc02a4994e483adbcf92b568a09a54", size = 5381688, upload-time = "2025-04-23T01:47:28.454Z" },
+    { url = "https://files.pythonhosted.org/packages/43/d1/3ba7bd978ce28bba8e3da2c2e9d5ae3f8f521ad3f0ca6ea4788d086ba00d/lxml-5.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0be91891bdb06ebe65122aa6bf3fc94489960cf7e03033c6f83a90863b23c58b", size = 5242043, upload-time = "2025-04-23T01:47:31.208Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/cd/95fa2201041a610c4d08ddaf31d43b98ecc4b1d74b1e7245b1abdab443cb/lxml-5.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:15a665ad90054a3d4f397bc40f73948d48e36e4c09f9bcffc7d90c87410e478a", size = 5021569, upload-time = "2025-04-23T01:47:33.805Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/a6/31da006fead660b9512d08d23d31e93ad3477dd47cc42e3285f143443176/lxml-5.4.0-cp313-cp313-win32.whl", hash = "sha256:d5663bc1b471c79f5c833cffbc9b87d7bf13f87e055a5c86c363ccd2348d7e82", size = 3485270, upload-time = "2025-04-23T01:47:36.133Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/14/c115516c62a7d2499781d2d3d7215218c0731b2c940753bf9f9b7b73924d/lxml-5.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:bcb7a1096b4b6b24ce1ac24d4942ad98f983cd3810f9711bcd0293f43a9d8b9f", size = 3814606, upload-time = "2025-04-23T01:47:39.028Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/b0/e4d1cbb8c078bc4ae44de9c6a79fec4e2b4151b1b4d50af71d799e76b177/lxml-5.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1b717b00a71b901b4667226bba282dd462c42ccf618ade12f9ba3674e1fabc55", size = 3892319, upload-time = "2025-04-23T01:49:22.069Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/aa/e2bdefba40d815059bcb60b371a36fbfcce970a935370e1b367ba1cc8f74/lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27a9ded0f0b52098ff89dd4c418325b987feed2ea5cc86e8860b0f844285d740", size = 4211614, upload-time = "2025-04-23T01:49:24.599Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/5f/91ff89d1e092e7cfdd8453a939436ac116db0a665e7f4be0cd8e65c7dc5a/lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b7ce10634113651d6f383aa712a194179dcd496bd8c41e191cec2099fa09de5", size = 4306273, upload-time = "2025-04-23T01:49:27.355Z" },
+    { url = "https://files.pythonhosted.org/packages/be/7c/8c3f15df2ca534589717bfd19d1e3482167801caedfa4d90a575facf68a6/lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53370c26500d22b45182f98847243efb518d268374a9570409d2e2276232fd37", size = 4208552, upload-time = "2025-04-23T01:49:29.949Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/d8/9567afb1665f64d73fc54eb904e418d1138d7f011ed00647121b4dd60b38/lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c6364038c519dffdbe07e3cf42e6a7f8b90c275d4d1617a69bb59734c1a2d571", size = 4331091, upload-time = "2025-04-23T01:49:32.842Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/ab/fdbbd91d8d82bf1a723ba88ec3e3d76c022b53c391b0c13cad441cdb8f9e/lxml-5.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b12cb6527599808ada9eb2cd6e0e7d3d8f13fe7bbb01c6311255a15ded4c7ab4", size = 3487862, upload-time = "2025-04-23T01:49:36.296Z" },
+]
+
+[[package]]
+name = "markdown-it-py"
+version = "3.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mdurl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" },
+]
+
+[[package]]
+name = "markupsafe"
+version = "3.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/90/d08277ce111dd22f77149fd1a5d4653eeb3b3eaacbdfcbae5afb2600eebd/MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", size = 14357, upload-time = "2024-10-18T15:20:51.44Z" },
+    { url = "https://files.pythonhosted.org/packages/04/e1/6e2194baeae0bca1fae6629dc0cbbb968d4d941469cbab11a3872edff374/MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", size = 12393, upload-time = "2024-10-18T15:20:52.426Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/69/35fa85a8ece0a437493dc61ce0bb6d459dcba482c34197e3efc829aa357f/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", size = 21732, upload-time = "2024-10-18T15:20:53.578Z" },
+    { url = "https://files.pythonhosted.org/packages/22/35/137da042dfb4720b638d2937c38a9c2df83fe32d20e8c8f3185dbfef05f7/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", size = 20866, upload-time = "2024-10-18T15:20:55.06Z" },
+    { url = "https://files.pythonhosted.org/packages/29/28/6d029a903727a1b62edb51863232152fd335d602def598dade38996887f0/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", size = 20964, upload-time = "2024-10-18T15:20:55.906Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/cd/07438f95f83e8bc028279909d9c9bd39e24149b0d60053a97b2bc4f8aa51/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", size = 21977, upload-time = "2024-10-18T15:20:57.189Z" },
+    { url = "https://files.pythonhosted.org/packages/29/01/84b57395b4cc062f9c4c55ce0df7d3108ca32397299d9df00fedd9117d3d/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", size = 21366, upload-time = "2024-10-18T15:20:58.235Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/6e/61ebf08d8940553afff20d1fb1ba7294b6f8d279df9fd0c0db911b4bbcfd/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", size = 21091, upload-time = "2024-10-18T15:20:59.235Z" },
+    { url = "https://files.pythonhosted.org/packages/11/23/ffbf53694e8c94ebd1e7e491de185124277964344733c45481f32ede2499/MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50", size = 15065, upload-time = "2024-10-18T15:21:00.307Z" },
+    { url = "https://files.pythonhosted.org/packages/44/06/e7175d06dd6e9172d4a69a72592cb3f7a996a9c396eee29082826449bbc3/MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", size = 15514, upload-time = "2024-10-18T15:21:01.122Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353, upload-time = "2024-10-18T15:21:02.187Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392, upload-time = "2024-10-18T15:21:02.941Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984, upload-time = "2024-10-18T15:21:03.953Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120, upload-time = "2024-10-18T15:21:06.495Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032, upload-time = "2024-10-18T15:21:07.295Z" },
+    { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057, upload-time = "2024-10-18T15:21:08.073Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359, upload-time = "2024-10-18T15:21:09.318Z" },
+    { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306, upload-time = "2024-10-18T15:21:10.185Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094, upload-time = "2024-10-18T15:21:11.005Z" },
+    { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521, upload-time = "2024-10-18T15:21:12.911Z" },
+    { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" },
+    { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" },
+    { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" },
+    { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274, upload-time = "2024-10-18T15:21:24.577Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352, upload-time = "2024-10-18T15:21:25.382Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122, upload-time = "2024-10-18T15:21:26.199Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085, upload-time = "2024-10-18T15:21:27.029Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978, upload-time = "2024-10-18T15:21:27.846Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208, upload-time = "2024-10-18T15:21:28.744Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357, upload-time = "2024-10-18T15:21:29.545Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344, upload-time = "2024-10-18T15:21:30.366Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101, upload-time = "2024-10-18T15:21:31.207Z" },
+    { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603, upload-time = "2024-10-18T15:21:32.032Z" },
+    { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510, upload-time = "2024-10-18T15:21:33.625Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486, upload-time = "2024-10-18T15:21:34.611Z" },
+    { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480, upload-time = "2024-10-18T15:21:35.398Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914, upload-time = "2024-10-18T15:21:36.231Z" },
+    { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796, upload-time = "2024-10-18T15:21:37.073Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473, upload-time = "2024-10-18T15:21:37.932Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114, upload-time = "2024-10-18T15:21:39.799Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098, upload-time = "2024-10-18T15:21:40.813Z" },
+    { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208, upload-time = "2024-10-18T15:21:41.814Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739, upload-time = "2024-10-18T15:21:42.784Z" },
+]
+
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
+]
+
+[[package]]
+name = "mpmath"
+version = "1.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" },
+]
+
+[[package]]
+name = "multidict"
+version = "6.6.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3d/2c/5dad12e82fbdf7470f29bff2171484bf07cb3b16ada60a6589af8f376440/multidict-6.6.3.tar.gz", hash = "sha256:798a9eb12dab0a6c2e29c1de6f3468af5cb2da6053a20dfa3344907eed0937cc", size = 101006, upload-time = "2025-06-30T15:53:46.929Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0b/67/414933982bce2efce7cbcb3169eaaf901e0f25baec69432b4874dfb1f297/multidict-6.6.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a2be5b7b35271f7fff1397204ba6708365e3d773579fe2a30625e16c4b4ce817", size = 77017, upload-time = "2025-06-30T15:50:58.931Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/fe/d8a3ee1fad37dc2ef4f75488b0d9d4f25bf204aad8306cbab63d97bff64a/multidict-6.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:12f4581d2930840295c461764b9a65732ec01250b46c6b2c510d7ee68872b140", size = 44897, upload-time = "2025-06-30T15:51:00.999Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/e0/265d89af8c98240265d82b8cbcf35897f83b76cd59ee3ab3879050fd8c45/multidict-6.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dd7793bab517e706c9ed9d7310b06c8672fd0aeee5781bfad612f56b8e0f7d14", size = 44574, upload-time = "2025-06-30T15:51:02.449Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/05/6b759379f7e8e04ccc97cfb2a5dcc5cdbd44a97f072b2272dc51281e6a40/multidict-6.6.3-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:72d8815f2cd3cf3df0f83cac3f3ef801d908b2d90409ae28102e0553af85545a", size = 225729, upload-time = "2025-06-30T15:51:03.794Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/f5/8d5a15488edd9a91fa4aad97228d785df208ed6298580883aa3d9def1959/multidict-6.6.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:531e331a2ee53543ab32b16334e2deb26f4e6b9b28e41f8e0c87e99a6c8e2d69", size = 242515, upload-time = "2025-06-30T15:51:05.002Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/b5/a8f317d47d0ac5bb746d6d8325885c8967c2a8ce0bb57be5399e3642cccb/multidict-6.6.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:42ca5aa9329a63be8dc49040f63817d1ac980e02eeddba763a9ae5b4027b9c9c", size = 222224, upload-time = "2025-06-30T15:51:06.148Z" },
+    { url = "https://files.pythonhosted.org/packages/76/88/18b2a0d5e80515fa22716556061189c2853ecf2aa2133081ebbe85ebea38/multidict-6.6.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:208b9b9757060b9faa6f11ab4bc52846e4f3c2fb8b14d5680c8aac80af3dc751", size = 253124, upload-time = "2025-06-30T15:51:07.375Z" },
+    { url = "https://files.pythonhosted.org/packages/62/bf/ebfcfd6b55a1b05ef16d0775ae34c0fe15e8dab570d69ca9941073b969e7/multidict-6.6.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:acf6b97bd0884891af6a8b43d0f586ab2fcf8e717cbd47ab4bdddc09e20652d8", size = 251529, upload-time = "2025-06-30T15:51:08.691Z" },
+    { url = "https://files.pythonhosted.org/packages/44/11/780615a98fd3775fc309d0234d563941af69ade2df0bb82c91dda6ddaea1/multidict-6.6.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:68e9e12ed00e2089725669bdc88602b0b6f8d23c0c95e52b95f0bc69f7fe9b55", size = 241627, upload-time = "2025-06-30T15:51:10.605Z" },
+    { url = "https://files.pythonhosted.org/packages/28/3d/35f33045e21034b388686213752cabc3a1b9d03e20969e6fa8f1b1d82db1/multidict-6.6.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:05db2f66c9addb10cfa226e1acb363450fab2ff8a6df73c622fefe2f5af6d4e7", size = 239351, upload-time = "2025-06-30T15:51:12.18Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/cc/ff84c03b95b430015d2166d9aae775a3985d757b94f6635010d0038d9241/multidict-6.6.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:0db58da8eafb514db832a1b44f8fa7906fdd102f7d982025f816a93ba45e3dcb", size = 233429, upload-time = "2025-06-30T15:51:13.533Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/f0/8cd49a0b37bdea673a4b793c2093f2f4ba8e7c9d6d7c9bd672fd6d38cd11/multidict-6.6.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:14117a41c8fdb3ee19c743b1c027da0736fdb79584d61a766da53d399b71176c", size = 243094, upload-time = "2025-06-30T15:51:14.815Z" },
+    { url = "https://files.pythonhosted.org/packages/96/19/5d9a0cfdafe65d82b616a45ae950975820289069f885328e8185e64283c2/multidict-6.6.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:877443eaaabcd0b74ff32ebeed6f6176c71850feb7d6a1d2db65945256ea535c", size = 248957, upload-time = "2025-06-30T15:51:16.076Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/dc/c90066151da87d1e489f147b9b4327927241e65f1876702fafec6729c014/multidict-6.6.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:70b72e749a4f6e7ed8fb334fa8d8496384840319512746a5f42fa0aec79f4d61", size = 243590, upload-time = "2025-06-30T15:51:17.413Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/39/458afb0cccbb0ee9164365273be3e039efddcfcb94ef35924b7dbdb05db0/multidict-6.6.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:43571f785b86afd02b3855c5ac8e86ec921b760298d6f82ff2a61daf5a35330b", size = 237487, upload-time = "2025-06-30T15:51:19.039Z" },
+    { url = "https://files.pythonhosted.org/packages/35/38/0016adac3990426610a081787011177e661875546b434f50a26319dc8372/multidict-6.6.3-cp310-cp310-win32.whl", hash = "sha256:20c5a0c3c13a15fd5ea86c42311859f970070e4e24de5a550e99d7c271d76318", size = 41390, upload-time = "2025-06-30T15:51:20.362Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/d2/17897a8f3f2c5363d969b4c635aa40375fe1f09168dc09a7826780bfb2a4/multidict-6.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab0a34a007704c625e25a9116c6770b4d3617a071c8a7c30cd338dfbadfe6485", size = 45954, upload-time = "2025-06-30T15:51:21.383Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/5f/d4a717c1e457fe44072e33fa400d2b93eb0f2819c4d669381f925b7cba1f/multidict-6.6.3-cp310-cp310-win_arm64.whl", hash = "sha256:769841d70ca8bdd140a715746199fc6473414bd02efd678d75681d2d6a8986c5", size = 42981, upload-time = "2025-06-30T15:51:22.809Z" },
+    { url = "https://files.pythonhosted.org/packages/08/f0/1a39863ced51f639c81a5463fbfa9eb4df59c20d1a8769ab9ef4ca57ae04/multidict-6.6.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:18f4eba0cbac3546b8ae31e0bbc55b02c801ae3cbaf80c247fcdd89b456ff58c", size = 76445, upload-time = "2025-06-30T15:51:24.01Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/0e/a7cfa451c7b0365cd844e90b41e21fab32edaa1e42fc0c9f68461ce44ed7/multidict-6.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef43b5dd842382329e4797c46f10748d8c2b6e0614f46b4afe4aee9ac33159df", size = 44610, upload-time = "2025-06-30T15:51:25.158Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/bb/a14a4efc5ee748cc1904b0748be278c31b9295ce5f4d2ef66526f410b94d/multidict-6.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf9bd1fd5eec01494e0f2e8e446a74a85d5e49afb63d75a9934e4a5423dba21d", size = 44267, upload-time = "2025-06-30T15:51:26.326Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/f8/410677d563c2d55e063ef74fe578f9d53fe6b0a51649597a5861f83ffa15/multidict-6.6.3-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5bd8d6f793a787153956cd35e24f60485bf0651c238e207b9a54f7458b16d539", size = 230004, upload-time = "2025-06-30T15:51:27.491Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/df/2b787f80059314a98e1ec6a4cc7576244986df3e56b3c755e6fc7c99e038/multidict-6.6.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bf99b4daf908c73856bd87ee0a2499c3c9a3d19bb04b9c6025e66af3fd07462", size = 247196, upload-time = "2025-06-30T15:51:28.762Z" },
+    { url = "https://files.pythonhosted.org/packages/05/f2/f9117089151b9a8ab39f9019620d10d9718eec2ac89e7ca9d30f3ec78e96/multidict-6.6.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b9e59946b49dafaf990fd9c17ceafa62976e8471a14952163d10a7a630413a9", size = 225337, upload-time = "2025-06-30T15:51:30.025Z" },
+    { url = "https://files.pythonhosted.org/packages/93/2d/7115300ec5b699faa152c56799b089a53ed69e399c3c2d528251f0aeda1a/multidict-6.6.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e2db616467070d0533832d204c54eea6836a5e628f2cb1e6dfd8cd6ba7277cb7", size = 257079, upload-time = "2025-06-30T15:51:31.716Z" },
+    { url = "https://files.pythonhosted.org/packages/15/ea/ff4bab367623e39c20d3b07637225c7688d79e4f3cc1f3b9f89867677f9a/multidict-6.6.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7394888236621f61dcdd25189b2768ae5cc280f041029a5bcf1122ac63df79f9", size = 255461, upload-time = "2025-06-30T15:51:33.029Z" },
+    { url = "https://files.pythonhosted.org/packages/74/07/2c9246cda322dfe08be85f1b8739646f2c4c5113a1422d7a407763422ec4/multidict-6.6.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f114d8478733ca7388e7c7e0ab34b72547476b97009d643644ac33d4d3fe1821", size = 246611, upload-time = "2025-06-30T15:51:34.47Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/62/279c13d584207d5697a752a66ffc9bb19355a95f7659140cb1b3cf82180e/multidict-6.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cdf22e4db76d323bcdc733514bf732e9fb349707c98d341d40ebcc6e9318ef3d", size = 243102, upload-time = "2025-06-30T15:51:36.525Z" },
+    { url = "https://files.pythonhosted.org/packages/69/cc/e06636f48c6d51e724a8bc8d9e1db5f136fe1df066d7cafe37ef4000f86a/multidict-6.6.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e995a34c3d44ab511bfc11aa26869b9d66c2d8c799fa0e74b28a473a692532d6", size = 238693, upload-time = "2025-06-30T15:51:38.278Z" },
+    { url = "https://files.pythonhosted.org/packages/89/a4/66c9d8fb9acf3b226cdd468ed009537ac65b520aebdc1703dd6908b19d33/multidict-6.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:766a4a5996f54361d8d5a9050140aa5362fe48ce51c755a50c0bc3706460c430", size = 246582, upload-time = "2025-06-30T15:51:39.709Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/01/c69e0317be556e46257826d5449feb4e6aa0d18573e567a48a2c14156f1f/multidict-6.6.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:3893a0d7d28a7fe6ca7a1f760593bc13038d1d35daf52199d431b61d2660602b", size = 253355, upload-time = "2025-06-30T15:51:41.013Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/da/9cc1da0299762d20e626fe0042e71b5694f9f72d7d3f9678397cbaa71b2b/multidict-6.6.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:934796c81ea996e61914ba58064920d6cad5d99140ac3167901eb932150e2e56", size = 247774, upload-time = "2025-06-30T15:51:42.291Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/91/b22756afec99cc31105ddd4a52f95ab32b1a4a58f4d417979c570c4a922e/multidict-6.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9ed948328aec2072bc00f05d961ceadfd3e9bfc2966c1319aeaf7b7c21219183", size = 242275, upload-time = "2025-06-30T15:51:43.642Z" },
+    { url = "https://files.pythonhosted.org/packages/be/f1/adcc185b878036a20399d5be5228f3cbe7f823d78985d101d425af35c800/multidict-6.6.3-cp311-cp311-win32.whl", hash = "sha256:9f5b28c074c76afc3e4c610c488e3493976fe0e596dd3db6c8ddfbb0134dcac5", size = 41290, upload-time = "2025-06-30T15:51:45.264Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/d4/27652c1c6526ea6b4f5ddd397e93f4232ff5de42bea71d339bc6a6cc497f/multidict-6.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc7f6fbc61b1c16050a389c630da0b32fc6d4a3d191394ab78972bf5edc568c2", size = 45942, upload-time = "2025-06-30T15:51:46.377Z" },
+    { url = "https://files.pythonhosted.org/packages/16/18/23f4932019804e56d3c2413e237f866444b774b0263bcb81df2fdecaf593/multidict-6.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:d4e47d8faffaae822fb5cba20937c048d4f734f43572e7079298a6c39fb172cb", size = 42880, upload-time = "2025-06-30T15:51:47.561Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/a0/6b57988ea102da0623ea814160ed78d45a2645e4bbb499c2896d12833a70/multidict-6.6.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:056bebbeda16b2e38642d75e9e5310c484b7c24e3841dc0fb943206a72ec89d6", size = 76514, upload-time = "2025-06-30T15:51:48.728Z" },
+    { url = "https://files.pythonhosted.org/packages/07/7a/d1e92665b0850c6c0508f101f9cf0410c1afa24973e1115fe9c6a185ebf7/multidict-6.6.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e5f481cccb3c5c5e5de5d00b5141dc589c1047e60d07e85bbd7dea3d4580d63f", size = 45394, upload-time = "2025-06-30T15:51:49.986Z" },
+    { url = "https://files.pythonhosted.org/packages/52/6f/dd104490e01be6ef8bf9573705d8572f8c2d2c561f06e3826b081d9e6591/multidict-6.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:10bea2ee839a759ee368b5a6e47787f399b41e70cf0c20d90dfaf4158dfb4e55", size = 43590, upload-time = "2025-06-30T15:51:51.331Z" },
+    { url = "https://files.pythonhosted.org/packages/44/fe/06e0e01b1b0611e6581b7fd5a85b43dacc08b6cea3034f902f383b0873e5/multidict-6.6.3-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:2334cfb0fa9549d6ce2c21af2bfbcd3ac4ec3646b1b1581c88e3e2b1779ec92b", size = 237292, upload-time = "2025-06-30T15:51:52.584Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/71/4f0e558fb77696b89c233c1ee2d92f3e1d5459070a0e89153c9e9e804186/multidict-6.6.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8fee016722550a2276ca2cb5bb624480e0ed2bd49125b2b73b7010b9090e888", size = 258385, upload-time = "2025-06-30T15:51:53.913Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/25/cca0e68228addad24903801ed1ab42e21307a1b4b6dd2cf63da5d3ae082a/multidict-6.6.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5511cb35f5c50a2db21047c875eb42f308c5583edf96bd8ebf7d770a9d68f6d", size = 242328, upload-time = "2025-06-30T15:51:55.672Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/a3/46f2d420d86bbcb8fe660b26a10a219871a0fbf4d43cb846a4031533f3e0/multidict-6.6.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:712b348f7f449948e0a6c4564a21c7db965af900973a67db432d724619b3c680", size = 268057, upload-time = "2025-06-30T15:51:57.037Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/73/1c743542fe00794a2ec7466abd3f312ccb8fad8dff9f36d42e18fb1ec33e/multidict-6.6.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e4e15d2138ee2694e038e33b7c3da70e6b0ad8868b9f8094a72e1414aeda9c1a", size = 269341, upload-time = "2025-06-30T15:51:59.111Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/11/6ec9dcbe2264b92778eeb85407d1df18812248bf3506a5a1754bc035db0c/multidict-6.6.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8df25594989aebff8a130f7899fa03cbfcc5d2b5f4a461cf2518236fe6f15961", size = 256081, upload-time = "2025-06-30T15:52:00.533Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/2b/631b1e2afeb5f1696846d747d36cda075bfdc0bc7245d6ba5c319278d6c4/multidict-6.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:159ca68bfd284a8860f8d8112cf0521113bffd9c17568579e4d13d1f1dc76b65", size = 253581, upload-time = "2025-06-30T15:52:02.43Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/0e/7e3b93f79efeb6111d3bf9a1a69e555ba1d07ad1c11bceb56b7310d0d7ee/multidict-6.6.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e098c17856a8c9ade81b4810888c5ad1914099657226283cab3062c0540b0643", size = 250750, upload-time = "2025-06-30T15:52:04.26Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/9e/086846c1d6601948e7de556ee464a2d4c85e33883e749f46b9547d7b0704/multidict-6.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:67c92ed673049dec52d7ed39f8cf9ebbadf5032c774058b4406d18c8f8fe7063", size = 251548, upload-time = "2025-06-30T15:52:06.002Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/7b/86ec260118e522f1a31550e87b23542294880c97cfbf6fb18cc67b044c66/multidict-6.6.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:bd0578596e3a835ef451784053cfd327d607fc39ea1a14812139339a18a0dbc3", size = 262718, upload-time = "2025-06-30T15:52:07.707Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/bd/22ce8f47abb0be04692c9fc4638508b8340987b18691aa7775d927b73f72/multidict-6.6.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:346055630a2df2115cd23ae271910b4cae40f4e336773550dca4889b12916e75", size = 259603, upload-time = "2025-06-30T15:52:09.58Z" },
+    { url = "https://files.pythonhosted.org/packages/07/9c/91b7ac1691be95cd1f4a26e36a74b97cda6aa9820632d31aab4410f46ebd/multidict-6.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:555ff55a359302b79de97e0468e9ee80637b0de1fce77721639f7cd9440b3a10", size = 251351, upload-time = "2025-06-30T15:52:10.947Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/5c/4d7adc739884f7a9fbe00d1eac8c034023ef8bad71f2ebe12823ca2e3649/multidict-6.6.3-cp312-cp312-win32.whl", hash = "sha256:73ab034fb8d58ff85c2bcbadc470efc3fafeea8affcf8722855fb94557f14cc5", size = 41860, upload-time = "2025-06-30T15:52:12.334Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/a3/0fbc7afdf7cb1aa12a086b02959307848eb6bcc8f66fcb66c0cb57e2a2c1/multidict-6.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:04cbcce84f63b9af41bad04a54d4cc4e60e90c35b9e6ccb130be2d75b71f8c17", size = 45982, upload-time = "2025-06-30T15:52:13.6Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/95/8c825bd70ff9b02462dc18d1295dd08d3e9e4eb66856d292ffa62cfe1920/multidict-6.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:0f1130b896ecb52d2a1e615260f3ea2af55fa7dc3d7c3003ba0c3121a759b18b", size = 43210, upload-time = "2025-06-30T15:52:14.893Z" },
+    { url = "https://files.pythonhosted.org/packages/52/1d/0bebcbbb4f000751fbd09957257903d6e002943fc668d841a4cf2fb7f872/multidict-6.6.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:540d3c06d48507357a7d57721e5094b4f7093399a0106c211f33540fdc374d55", size = 75843, upload-time = "2025-06-30T15:52:16.155Z" },
+    { url = "https://files.pythonhosted.org/packages/07/8f/cbe241b0434cfe257f65c2b1bcf9e8d5fb52bc708c5061fb29b0fed22bdf/multidict-6.6.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9c19cea2a690f04247d43f366d03e4eb110a0dc4cd1bbeee4d445435428ed35b", size = 45053, upload-time = "2025-06-30T15:52:17.429Z" },
+    { url = "https://files.pythonhosted.org/packages/32/d2/0b3b23f9dbad5b270b22a3ac3ea73ed0a50ef2d9a390447061178ed6bdb8/multidict-6.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7af039820cfd00effec86bda5d8debef711a3e86a1d3772e85bea0f243a4bd65", size = 43273, upload-time = "2025-06-30T15:52:19.346Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/fe/6eb68927e823999e3683bc49678eb20374ba9615097d085298fd5b386564/multidict-6.6.3-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:500b84f51654fdc3944e936f2922114349bf8fdcac77c3092b03449f0e5bc2b3", size = 237124, upload-time = "2025-06-30T15:52:20.773Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/ab/320d8507e7726c460cb77117848b3834ea0d59e769f36fdae495f7669929/multidict-6.6.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3fc723ab8a5c5ed6c50418e9bfcd8e6dceba6c271cee6728a10a4ed8561520c", size = 256892, upload-time = "2025-06-30T15:52:22.242Z" },
+    { url = "https://files.pythonhosted.org/packages/76/60/38ee422db515ac69834e60142a1a69111ac96026e76e8e9aa347fd2e4591/multidict-6.6.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:94c47ea3ade005b5976789baaed66d4de4480d0a0bf31cef6edaa41c1e7b56a6", size = 240547, upload-time = "2025-06-30T15:52:23.736Z" },
+    { url = "https://files.pythonhosted.org/packages/27/fb/905224fde2dff042b030c27ad95a7ae744325cf54b890b443d30a789b80e/multidict-6.6.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dbc7cf464cc6d67e83e136c9f55726da3a30176f020a36ead246eceed87f1cd8", size = 266223, upload-time = "2025-06-30T15:52:25.185Z" },
+    { url = "https://files.pythonhosted.org/packages/76/35/dc38ab361051beae08d1a53965e3e1a418752fc5be4d3fb983c5582d8784/multidict-6.6.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:900eb9f9da25ada070f8ee4a23f884e0ee66fe4e1a38c3af644256a508ad81ca", size = 267262, upload-time = "2025-06-30T15:52:26.969Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/a3/0a485b7f36e422421b17e2bbb5a81c1af10eac1d4476f2ff92927c730479/multidict-6.6.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c6df517cf177da5d47ab15407143a89cd1a23f8b335f3a28d57e8b0a3dbb884", size = 254345, upload-time = "2025-06-30T15:52:28.467Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/59/bcdd52c1dab7c0e0d75ff19cac751fbd5f850d1fc39172ce809a74aa9ea4/multidict-6.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4ef421045f13879e21c994b36e728d8e7d126c91a64b9185810ab51d474f27e7", size = 252248, upload-time = "2025-06-30T15:52:29.938Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/a4/2d96aaa6eae8067ce108d4acee6f45ced5728beda55c0f02ae1072c730d1/multidict-6.6.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6c1e61bb4f80895c081790b6b09fa49e13566df8fbff817da3f85b3a8192e36b", size = 250115, upload-time = "2025-06-30T15:52:31.416Z" },
+    { url = "https://files.pythonhosted.org/packages/25/d2/ed9f847fa5c7d0677d4f02ea2c163d5e48573de3f57bacf5670e43a5ffaa/multidict-6.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e5e8523bb12d7623cd8300dbd91b9e439a46a028cd078ca695eb66ba31adee3c", size = 249649, upload-time = "2025-06-30T15:52:32.996Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/af/9155850372563fc550803d3f25373308aa70f59b52cff25854086ecb4a79/multidict-6.6.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ef58340cc896219e4e653dade08fea5c55c6df41bcc68122e3be3e9d873d9a7b", size = 261203, upload-time = "2025-06-30T15:52:34.521Z" },
+    { url = "https://files.pythonhosted.org/packages/36/2f/c6a728f699896252cf309769089568a33c6439626648843f78743660709d/multidict-6.6.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fc9dc435ec8699e7b602b94fe0cd4703e69273a01cbc34409af29e7820f777f1", size = 258051, upload-time = "2025-06-30T15:52:35.999Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/60/689880776d6b18fa2b70f6cc74ff87dd6c6b9b47bd9cf74c16fecfaa6ad9/multidict-6.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9e864486ef4ab07db5e9cb997bad2b681514158d6954dd1958dfb163b83d53e6", size = 249601, upload-time = "2025-06-30T15:52:37.473Z" },
+    { url = "https://files.pythonhosted.org/packages/75/5e/325b11f2222a549019cf2ef879c1f81f94a0d40ace3ef55cf529915ba6cc/multidict-6.6.3-cp313-cp313-win32.whl", hash = "sha256:5633a82fba8e841bc5c5c06b16e21529573cd654f67fd833650a215520a6210e", size = 41683, upload-time = "2025-06-30T15:52:38.927Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/ad/cf46e73f5d6e3c775cabd2a05976547f3f18b39bee06260369a42501f053/multidict-6.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:e93089c1570a4ad54c3714a12c2cef549dc9d58e97bcded193d928649cab78e9", size = 45811, upload-time = "2025-06-30T15:52:40.207Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/c9/2e3fe950db28fb7c62e1a5f46e1e38759b072e2089209bc033c2798bb5ec/multidict-6.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:c60b401f192e79caec61f166da9c924e9f8bc65548d4246842df91651e83d600", size = 43056, upload-time = "2025-06-30T15:52:41.575Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/58/aaf8114cf34966e084a8cc9517771288adb53465188843d5a19862cb6dc3/multidict-6.6.3-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:02fd8f32d403a6ff13864b0851f1f523d4c988051eea0471d4f1fd8010f11134", size = 82811, upload-time = "2025-06-30T15:52:43.281Z" },
+    { url = "https://files.pythonhosted.org/packages/71/af/5402e7b58a1f5b987a07ad98f2501fdba2a4f4b4c30cf114e3ce8db64c87/multidict-6.6.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:f3aa090106b1543f3f87b2041eef3c156c8da2aed90c63a2fbed62d875c49c37", size = 48304, upload-time = "2025-06-30T15:52:45.026Z" },
+    { url = "https://files.pythonhosted.org/packages/39/65/ab3c8cafe21adb45b24a50266fd747147dec7847425bc2a0f6934b3ae9ce/multidict-6.6.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e924fb978615a5e33ff644cc42e6aa241effcf4f3322c09d4f8cebde95aff5f8", size = 46775, upload-time = "2025-06-30T15:52:46.459Z" },
+    { url = "https://files.pythonhosted.org/packages/49/ba/9fcc1b332f67cc0c0c8079e263bfab6660f87fe4e28a35921771ff3eea0d/multidict-6.6.3-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:b9fe5a0e57c6dbd0e2ce81ca66272282c32cd11d31658ee9553849d91289e1c1", size = 229773, upload-time = "2025-06-30T15:52:47.88Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/14/0145a251f555f7c754ce2dcbcd012939bbd1f34f066fa5d28a50e722a054/multidict-6.6.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b24576f208793ebae00280c59927c3b7c2a3b1655e443a25f753c4611bc1c373", size = 250083, upload-time = "2025-06-30T15:52:49.366Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/d4/d5c0bd2bbb173b586c249a151a26d2fb3ec7d53c96e42091c9fef4e1f10c/multidict-6.6.3-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135631cb6c58eac37d7ac0df380294fecdc026b28837fa07c02e459c7fb9c54e", size = 228980, upload-time = "2025-06-30T15:52:50.903Z" },
+    { url = "https://files.pythonhosted.org/packages/21/32/c9a2d8444a50ec48c4733ccc67254100c10e1c8ae8e40c7a2d2183b59b97/multidict-6.6.3-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:274d416b0df887aef98f19f21578653982cfb8a05b4e187d4a17103322eeaf8f", size = 257776, upload-time = "2025-06-30T15:52:52.764Z" },
+    { url = "https://files.pythonhosted.org/packages/68/d0/14fa1699f4ef629eae08ad6201c6b476098f5efb051b296f4c26be7a9fdf/multidict-6.6.3-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e252017a817fad7ce05cafbe5711ed40faeb580e63b16755a3a24e66fa1d87c0", size = 256882, upload-time = "2025-06-30T15:52:54.596Z" },
+    { url = "https://files.pythonhosted.org/packages/da/88/84a27570fbe303c65607d517a5f147cd2fc046c2d1da02b84b17b9bdc2aa/multidict-6.6.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e4cc8d848cd4fe1cdee28c13ea79ab0ed37fc2e89dd77bac86a2e7959a8c3bc", size = 247816, upload-time = "2025-06-30T15:52:56.175Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/60/dca352a0c999ce96a5d8b8ee0b2b9f729dcad2e0b0c195f8286269a2074c/multidict-6.6.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9e236a7094b9c4c1b7585f6b9cca34b9d833cf079f7e4c49e6a4a6ec9bfdc68f", size = 245341, upload-time = "2025-06-30T15:52:57.752Z" },
+    { url = "https://files.pythonhosted.org/packages/50/ef/433fa3ed06028f03946f3993223dada70fb700f763f70c00079533c34578/multidict-6.6.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:e0cb0ab69915c55627c933f0b555a943d98ba71b4d1c57bc0d0a66e2567c7471", size = 235854, upload-time = "2025-06-30T15:52:59.74Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/1f/487612ab56fbe35715320905215a57fede20de7db40a261759690dc80471/multidict-6.6.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:81ef2f64593aba09c5212a3d0f8c906a0d38d710a011f2f42759704d4557d3f2", size = 243432, upload-time = "2025-06-30T15:53:01.602Z" },
+    { url = "https://files.pythonhosted.org/packages/da/6f/ce8b79de16cd885c6f9052c96a3671373d00c59b3ee635ea93e6e81b8ccf/multidict-6.6.3-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:b9cbc60010de3562545fa198bfc6d3825df430ea96d2cc509c39bd71e2e7d648", size = 252731, upload-time = "2025-06-30T15:53:03.517Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/fe/a2514a6aba78e5abefa1624ca85ae18f542d95ac5cde2e3815a9fbf369aa/multidict-6.6.3-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:70d974eaaa37211390cd02ef93b7e938de564bbffa866f0b08d07e5e65da783d", size = 247086, upload-time = "2025-06-30T15:53:05.48Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/22/b788718d63bb3cce752d107a57c85fcd1a212c6c778628567c9713f9345a/multidict-6.6.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3713303e4a6663c6d01d648a68f2848701001f3390a030edaaf3fc949c90bf7c", size = 243338, upload-time = "2025-06-30T15:53:07.522Z" },
+    { url = "https://files.pythonhosted.org/packages/22/d6/fdb3d0670819f2228f3f7d9af613d5e652c15d170c83e5f1c94fbc55a25b/multidict-6.6.3-cp313-cp313t-win32.whl", hash = "sha256:639ecc9fe7cd73f2495f62c213e964843826f44505a3e5d82805aa85cac6f89e", size = 47812, upload-time = "2025-06-30T15:53:09.263Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/d6/a9d2c808f2c489ad199723197419207ecbfbc1776f6e155e1ecea9c883aa/multidict-6.6.3-cp313-cp313t-win_amd64.whl", hash = "sha256:9f97e181f344a0ef3881b573d31de8542cc0dbc559ec68c8f8b5ce2c2e91646d", size = 53011, upload-time = "2025-06-30T15:53:11.038Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/40/b68001cba8188dd267590a111f9661b6256debc327137667e832bf5d66e8/multidict-6.6.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ce8b7693da41a3c4fde5871c738a81490cea5496c671d74374c8ab889e1834fb", size = 45254, upload-time = "2025-06-30T15:53:12.421Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/30/9aec301e9772b098c1f5c0ca0279237c9766d94b97802e9888010c64b0ed/multidict-6.6.3-py3-none-any.whl", hash = "sha256:8db10f29c7541fc5da4defd8cd697e1ca429db743fa716325f236079b96f775a", size = 12313, upload-time = "2025-06-30T15:53:45.437Z" },
+]
+
+[[package]]
+name = "networkx"
+version = "3.2.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c4/80/a84676339aaae2f1cfdf9f418701dd634aef9cc76f708ef55c36ff39c3ca/networkx-3.2.1.tar.gz", hash = "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6", size = 2073928, upload-time = "2023-10-28T08:41:39.364Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d5/f0/8fbc882ca80cf077f1b246c0e3c3465f7f415439bdea6b899f6b19f61f70/networkx-3.2.1-py3-none-any.whl", hash = "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2", size = 1647772, upload-time = "2023-10-28T08:41:36.945Z" },
+]
+
+[[package]]
+name = "nltk"
+version = "3.9.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "joblib" },
+    { name = "regex" },
+    { name = "tqdm" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3c/87/db8be88ad32c2d042420b6fd9ffd4a149f9a0d7f0e86b3f543be2eeeedd2/nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868", size = 2904691, upload-time = "2024-08-18T19:48:37.769Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4d/66/7d9e26593edda06e8cb531874633f7c2372279c3b0f46235539fe546df8b/nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1", size = 1505442, upload-time = "2024-08-18T19:48:21.909Z" },
+]
+
+[[package]]
+name = "numpy"
+version = "2.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015, upload-time = "2024-08-26T20:19:40.945Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/21/91/3495b3237510f79f5d81f2508f9f13fea78ebfdf07538fc7444badda173d/numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece", size = 21165245, upload-time = "2024-08-26T20:04:14.625Z" },
+    { url = "https://files.pythonhosted.org/packages/05/33/26178c7d437a87082d11019292dce6d3fe6f0e9026b7b2309cbf3e489b1d/numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04", size = 13738540, upload-time = "2024-08-26T20:04:36.784Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/31/cc46e13bf07644efc7a4bf68df2df5fb2a1a88d0cd0da9ddc84dc0033e51/numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66", size = 5300623, upload-time = "2024-08-26T20:04:46.491Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/16/7bfcebf27bb4f9d7ec67332ffebee4d1bf085c84246552d52dbb548600e7/numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b", size = 6901774, upload-time = "2024-08-26T20:04:58.173Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/a3/561c531c0e8bf082c5bef509d00d56f82e0ea7e1e3e3a7fc8fa78742a6e5/numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd", size = 13907081, upload-time = "2024-08-26T20:05:19.098Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/66/f7177ab331876200ac7563a580140643d1179c8b4b6a6b0fc9838de2a9b8/numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318", size = 19523451, upload-time = "2024-08-26T20:05:47.479Z" },
+    { url = "https://files.pythonhosted.org/packages/25/7f/0b209498009ad6453e4efc2c65bcdf0ae08a182b2b7877d7ab38a92dc542/numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8", size = 19927572, upload-time = "2024-08-26T20:06:17.137Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/df/2619393b1e1b565cd2d4c4403bdd979621e2c4dea1f8532754b2598ed63b/numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326", size = 14400722, upload-time = "2024-08-26T20:06:39.16Z" },
+    { url = "https://files.pythonhosted.org/packages/22/ad/77e921b9f256d5da36424ffb711ae79ca3f451ff8489eeca544d0701d74a/numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97", size = 6472170, upload-time = "2024-08-26T20:06:50.361Z" },
+    { url = "https://files.pythonhosted.org/packages/10/05/3442317535028bc29cf0c0dd4c191a4481e8376e9f0db6bcf29703cadae6/numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131", size = 15905558, upload-time = "2024-08-26T20:07:13.881Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/cf/034500fb83041aa0286e0fb16e7c76e5c8b67c0711bb6e9e9737a717d5fe/numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448", size = 21169137, upload-time = "2024-08-26T20:07:45.345Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/d9/32de45561811a4b87fbdee23b5797394e3d1504b4a7cf40c10199848893e/numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195", size = 13703552, upload-time = "2024-08-26T20:08:06.666Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/ca/2f384720020c7b244d22508cb7ab23d95f179fcfff33c31a6eeba8d6c512/numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57", size = 5298957, upload-time = "2024-08-26T20:08:15.83Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/78/a3e4f9fb6aa4e6fdca0c5428e8ba039408514388cf62d89651aade838269/numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a", size = 6905573, upload-time = "2024-08-26T20:08:27.185Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/72/cfc3a1beb2caf4efc9d0b38a15fe34025230da27e1c08cc2eb9bfb1c7231/numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669", size = 13914330, upload-time = "2024-08-26T20:08:48.058Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/a8/c17acf65a931ce551fee11b72e8de63bf7e8a6f0e21add4c937c83563538/numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951", size = 19534895, upload-time = "2024-08-26T20:09:16.536Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/86/8767f3d54f6ae0165749f84648da9dcc8cd78ab65d415494962c86fac80f/numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9", size = 19937253, upload-time = "2024-08-26T20:09:46.263Z" },
+    { url = "https://files.pythonhosted.org/packages/df/87/f76450e6e1c14e5bb1eae6836478b1028e096fd02e85c1c37674606ab752/numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15", size = 14414074, upload-time = "2024-08-26T20:10:08.483Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/ca/0f0f328e1e59f73754f06e1adfb909de43726d4f24c6a3f8805f34f2b0fa/numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4", size = 6470640, upload-time = "2024-08-26T20:10:19.732Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/57/3a3f14d3a759dcf9bf6e9eda905794726b758819df4663f217d658a58695/numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc", size = 15910230, upload-time = "2024-08-26T20:10:43.413Z" },
+    { url = "https://files.pythonhosted.org/packages/45/40/2e117be60ec50d98fa08c2f8c48e09b3edea93cfcabd5a9ff6925d54b1c2/numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b", size = 20895803, upload-time = "2024-08-26T20:11:13.916Z" },
+    { url = "https://files.pythonhosted.org/packages/46/92/1b8b8dee833f53cef3e0a3f69b2374467789e0bb7399689582314df02651/numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e", size = 13471835, upload-time = "2024-08-26T20:11:34.779Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/19/e2793bde475f1edaea6945be141aef6c8b4c669b90c90a300a8954d08f0a/numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c", size = 5038499, upload-time = "2024-08-26T20:11:43.902Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/ff/ddf6dac2ff0dd50a7327bcdba45cb0264d0e96bb44d33324853f781a8f3c/numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c", size = 6633497, upload-time = "2024-08-26T20:11:55.09Z" },
+    { url = "https://files.pythonhosted.org/packages/72/21/67f36eac8e2d2cd652a2e69595a54128297cdcb1ff3931cfc87838874bd4/numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692", size = 13621158, upload-time = "2024-08-26T20:12:14.95Z" },
+    { url = "https://files.pythonhosted.org/packages/39/68/e9f1126d757653496dbc096cb429014347a36b228f5a991dae2c6b6cfd40/numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a", size = 19236173, upload-time = "2024-08-26T20:12:44.049Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/e9/1f5333281e4ebf483ba1c888b1d61ba7e78d7e910fdd8e6499667041cc35/numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c", size = 19634174, upload-time = "2024-08-26T20:13:13.634Z" },
+    { url = "https://files.pythonhosted.org/packages/71/af/a469674070c8d8408384e3012e064299f7a2de540738a8e414dcfd639996/numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded", size = 14099701, upload-time = "2024-08-26T20:13:34.851Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/3d/08ea9f239d0e0e939b6ca52ad403c84a2bce1bde301a8eb4888c1c1543f1/numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5", size = 6174313, upload-time = "2024-08-26T20:13:45.653Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/b5/4ac39baebf1fdb2e72585c8352c56d063b6126be9fc95bd2bb5ef5770c20/numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a", size = 15606179, upload-time = "2024-08-26T20:14:08.786Z" },
+]
+
+[[package]]
+name = "nvidia-cublas-cu12"
+version = "12.6.4.1"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/af/eb/ff4b8c503fa1f1796679dce648854d58751982426e4e4b37d6fce49d259c/nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb", size = 393138322, upload-time = "2024-11-20T17:40:25.65Z" },
+]
+
+[[package]]
+name = "nvidia-cuda-cupti-cu12"
+version = "12.6.80"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/49/60/7b6497946d74bcf1de852a21824d63baad12cd417db4195fc1bfe59db953/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6768bad6cab4f19e8292125e5f1ac8aa7d1718704012a0e3272a6f61c4bce132", size = 8917980, upload-time = "2024-11-20T17:36:04.019Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/24/120ee57b218d9952c379d1e026c4479c9ece9997a4fb46303611ee48f038/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a3eff6cdfcc6a4c35db968a06fcadb061cbc7d6dde548609a941ff8701b98b73", size = 8917972, upload-time = "2024-10-01T16:58:06.036Z" },
+]
+
+[[package]]
+name = "nvidia-cuda-nvrtc-cu12"
+version = "12.6.77"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/75/2e/46030320b5a80661e88039f59060d1790298b4718944a65a7f2aeda3d9e9/nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53", size = 23650380, upload-time = "2024-10-01T17:00:14.643Z" },
+]
+
+[[package]]
+name = "nvidia-cuda-runtime-cu12"
+version = "12.6.77"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e1/23/e717c5ac26d26cf39a27fbc076240fad2e3b817e5889d671b67f4f9f49c5/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba3b56a4f896141e25e19ab287cd71e52a6a0f4b29d0d31609f60e3b4d5219b7", size = 897690, upload-time = "2024-11-20T17:35:30.697Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/62/65c05e161eeddbafeca24dc461f47de550d9fa8a7e04eb213e32b55cfd99/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a84d15d5e1da416dd4774cb42edf5e954a3e60cc945698dc1d5be02321c44dc8", size = 897678, upload-time = "2024-10-01T16:57:33.821Z" },
+]
+
+[[package]]
+name = "nvidia-cudnn-cu12"
+version = "9.5.1.17"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "nvidia-cublas-cu12" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2a/78/4535c9c7f859a64781e43c969a3a7e84c54634e319a996d43ef32ce46f83/nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2", size = 570988386, upload-time = "2024-10-25T19:54:26.39Z" },
+]
+
+[[package]]
+name = "nvidia-cufft-cu12"
+version = "11.3.0.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "nvidia-nvjitlink-cu12" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5", size = 200221632, upload-time = "2024-11-20T17:41:32.357Z" },
+    { url = "https://files.pythonhosted.org/packages/60/de/99ec247a07ea40c969d904fc14f3a356b3e2a704121675b75c366b694ee1/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.whl", hash = "sha256:768160ac89f6f7b459bee747e8d175dbf53619cfe74b2a5636264163138013ca", size = 200221622, upload-time = "2024-10-01T17:03:58.79Z" },
+]
+
+[[package]]
+name = "nvidia-cufile-cu12"
+version = "1.11.1.6"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b2/66/cc9876340ac68ae71b15c743ddb13f8b30d5244af344ec8322b449e35426/nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159", size = 1142103, upload-time = "2024-11-20T17:42:11.83Z" },
+]
+
+[[package]]
+name = "nvidia-curand-cu12"
+version = "10.3.7.77"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/73/1b/44a01c4e70933637c93e6e1a8063d1e998b50213a6b65ac5a9169c47e98e/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf", size = 56279010, upload-time = "2024-11-20T17:42:50.958Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/aa/2c7ff0b5ee02eaef890c0ce7d4f74bc30901871c5e45dee1ae6d0083cd80/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:99f1a32f1ac2bd134897fc7a203f779303261268a65762a623bf30cc9fe79117", size = 56279000, upload-time = "2024-10-01T17:04:45.274Z" },
+]
+
+[[package]]
+name = "nvidia-cusolver-cu12"
+version = "11.7.1.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "nvidia-cublas-cu12" },
+    { name = "nvidia-cusparse-cu12" },
+    { name = "nvidia-nvjitlink-cu12" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f0/6e/c2cf12c9ff8b872e92b4a5740701e51ff17689c4d726fca91875b07f655d/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c", size = 158229790, upload-time = "2024-11-20T17:43:43.211Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/81/baba53585da791d043c10084cf9553e074548408e04ae884cfe9193bd484/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6cf28f17f64107a0c4d7802be5ff5537b2130bfc112f25d5a30df227058ca0e6", size = 158229780, upload-time = "2024-10-01T17:05:39.875Z" },
+]
+
+[[package]]
+name = "nvidia-cusparse-cu12"
+version = "12.5.4.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "nvidia-nvjitlink-cu12" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73", size = 216561367, upload-time = "2024-11-20T17:44:54.824Z" },
+    { url = "https://files.pythonhosted.org/packages/43/ac/64c4316ba163e8217a99680c7605f779accffc6a4bcd0c778c12948d3707/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:23749a6571191a215cb74d1cdbff4a86e7b19f1200c071b3fcf844a5bea23a2f", size = 216561357, upload-time = "2024-10-01T17:06:29.861Z" },
+]
+
+[[package]]
+name = "nvidia-cusparselt-cu12"
+version = "0.6.3"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3b/9a/72ef35b399b0e183bc2e8f6f558036922d453c4d8237dab26c666a04244b/nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46", size = 156785796, upload-time = "2024-10-15T21:29:17.709Z" },
+]
+
+[[package]]
+name = "nvidia-nccl-cu12"
+version = "2.26.2"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/67/ca/f42388aed0fddd64ade7493dbba36e1f534d4e6fdbdd355c6a90030ae028/nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6", size = 201319755, upload-time = "2025-03-13T00:29:55.296Z" },
+]
+
+[[package]]
+name = "nvidia-nvjitlink-cu12"
+version = "12.6.85"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9d/d7/c5383e47c7e9bf1c99d5bd2a8c935af2b6d705ad831a7ec5c97db4d82f4f/nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a", size = 19744971, upload-time = "2024-11-20T17:46:53.366Z" },
+]
+
+[[package]]
+name = "nvidia-nvtx-cu12"
+version = "12.6.77"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/56/9a/fff8376f8e3d084cd1530e1ef7b879bb7d6d265620c95c1b322725c694f4/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b90bed3df379fa79afbd21be8e04a0314336b8ae16768b58f2d34cb1d04cd7d2", size = 89276, upload-time = "2024-11-20T17:38:27.621Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/4e/0d0c945463719429b7bd21dece907ad0bde437a2ff12b9b12fee94722ab0/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6574241a3ec5fdc9334353ab8c479fe75841dbe8f4532a8fc97ce63503330ba1", size = 89265, upload-time = "2024-10-01T17:00:38.172Z" },
+]
+
+[[package]]
+name = "openai"
+version = "1.93.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "distro" },
+    { name = "httpx" },
+    { name = "jiter" },
+    { name = "pydantic" },
+    { name = "sniffio" },
+    { name = "tqdm" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e0/66/fadc0cad6a229c6a85c3aa5f222a786ec4d9bf14c2a004f80ffa21dbaf21/openai-1.93.3.tar.gz", hash = "sha256:488b76399238c694af7e4e30c58170ea55e6f65038ab27dbe95b5077a00f8af8", size = 487595, upload-time = "2025-07-09T14:08:27.789Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8b/b9/0df6351b25c6bd494c534d2a8191dc9460fb5bb09c88b1427775d49fde05/openai-1.93.3-py3-none-any.whl", hash = "sha256:41aaa7594c7d141b46eed0a58dcd75d20edcc809fdd2c931ecbb4957dc98a892", size = 755132, upload-time = "2025-07-09T14:08:25.533Z" },
+]
+
+[[package]]
+name = "outcome"
+version = "1.3.0.post0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "attrs" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060, upload-time = "2023-10-26T04:26:04.361Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692, upload-time = "2023-10-26T04:26:02.532Z" },
+]
+
+[[package]]
+name = "packaging"
+version = "25.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },
+]
+
+[[package]]
+name = "patchright"
+version = "1.56.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "greenlet" },
+    { name = "pyee" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0d/d6/ac03353f1b3138541356a044c2d7d3e01b73f9555c7281dab3423c5d44da/patchright-1.56.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:9df1e0b2c35a298ad2807044699476d3e872f76117c319077b68d7b8ac8bffae", size = 40595069, upload-time = "2025-11-11T18:59:20.801Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/f2/e33cd3e6916ed959f5b9be1182363e19771be81541a77b2c5615a6434cc8/patchright-1.56.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d8e86d743f4da99f83e9962f9317bd3aaabe4e9db86d3e12cb1d0f39e2f469", size = 39383869, upload-time = "2025-11-11T18:59:23.829Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/49/510b8fb761a7e6a2cbd8381f2254fd9442055e27961c0565944d5634ccb6/patchright-1.56.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:e939a9651ebad6ed5afb84e6cf8b874fd80e4b03a912509dc5159f0fa2d4a752", size = 40595069, upload-time = "2025-11-11T18:59:26.9Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/ad/17f72b906a1d3d9750b26964d4481340defbfc02f039c888756061f41a33/patchright-1.56.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:cef0f31334dece118fa8dbe20e0c0ac97ea539a2a3e7dd007a67d05b4c3e0512", size = 46246766, upload-time = "2025-11-11T18:59:29.985Z" },
+    { url = "https://files.pythonhosted.org/packages/53/b4/6546fb868f464c888cee8e95d5b0cc4e276afc5e531bc3ca737e2a910f77/patchright-1.56.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02329457d4f9e92b49209adceec33097042b6a6f03de31185a2f99721aeded6a", size = 46094291, upload-time = "2025-11-11T18:59:32.883Z" },
+    { url = "https://files.pythonhosted.org/packages/11/4b/a43f86570cb72f732723c19ba71f1c02200ed1cc762d71daa2c35718a529/patchright-1.56.0-py3-none-win32.whl", hash = "sha256:5502899cd8764ccc7a18c3cb47a76e49b25ec4eae155ec5ebea9e5e0022cc811", size = 35623487, upload-time = "2025-11-11T18:59:36.156Z" },
+    { url = "https://files.pythonhosted.org/packages/95/8d/37cc3127c6085ffa6482832a07f84587f2f04dbbaf194cc395bb2afa811c/patchright-1.56.0-py3-none-win_amd64.whl", hash = "sha256:0fe85d8f6b541689185115f3ce69e87a7f5dd964ec6aaeac635d4db5a906aec1", size = 35623491, upload-time = "2025-11-11T18:59:38.853Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/37/0b79ddfc6a2ba16e9f258670ea40a74c791d925f1847cf4ad0e739bc7506/patchright-1.56.0-py3-none-win_arm64.whl", hash = "sha256:8bbf1c753e7b07b889a3ff43109ce3cea457605ea6b0477e65af4d9a2c5585cf", size = 31232602, upload-time = "2025-11-11T18:59:41.537Z" },
+]
+
+[[package]]
+name = "pillow"
+version = "10.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/cd/74/ad3d526f3bf7b6d3f408b73fde271ec69dfac8b81341a318ce825f2b3812/pillow-10.4.0.tar.gz", hash = "sha256:166c1cd4d24309b30d61f79f4a9114b7b2313d7450912277855ff5dfd7cd4a06", size = 46555059, upload-time = "2024-07-01T09:48:43.583Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0e/69/a31cccd538ca0b5272be2a38347f8839b97a14be104ea08b0db92f749c74/pillow-10.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e", size = 3509271, upload-time = "2024-07-01T09:45:22.07Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/9e/4143b907be8ea0bce215f2ae4f7480027473f8b61fcedfda9d851082a5d2/pillow-10.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d", size = 3375658, upload-time = "2024-07-01T09:45:25.292Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/25/1fc45761955f9359b1169aa75e241551e74ac01a09f487adaaf4c3472d11/pillow-10.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7928ecbf1ece13956b95d9cbcfc77137652b02763ba384d9ab508099a2eca856", size = 4332075, upload-time = "2024-07-01T09:45:27.94Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/dd/425b95d0151e1d6c951f45051112394f130df3da67363b6bc75dc4c27aba/pillow-10.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d49b85c4348ea0b31ea63bc75a9f3857869174e2bf17e7aba02945cd218e6f", size = 4444808, upload-time = "2024-07-01T09:45:30.305Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/84/9a15cc5726cbbfe7f9f90bfb11f5d028586595907cd093815ca6644932e3/pillow-10.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6c762a5b0997f5659a5ef2266abc1d8851ad7749ad9a6a5506eb23d314e4f46b", size = 4356290, upload-time = "2024-07-01T09:45:32.868Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/5b/6651c288b08df3b8c1e2f8c1152201e0b25d240e22ddade0f1e242fc9fa0/pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a985e028fc183bf12a77a8bbf36318db4238a3ded7fa9df1b9a133f1cb79f8fc", size = 4525163, upload-time = "2024-07-01T09:45:35.279Z" },
+    { url = "https://files.pythonhosted.org/packages/07/8b/34854bf11a83c248505c8cb0fcf8d3d0b459a2246c8809b967963b6b12ae/pillow-10.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:812f7342b0eee081eaec84d91423d1b4650bb9828eb53d8511bcef8ce5aecf1e", size = 4463100, upload-time = "2024-07-01T09:45:37.74Z" },
+    { url = "https://files.pythonhosted.org/packages/78/63/0632aee4e82476d9cbe5200c0cdf9ba41ee04ed77887432845264d81116d/pillow-10.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ac1452d2fbe4978c2eec89fb5a23b8387aba707ac72810d9490118817d9c0b46", size = 4592880, upload-time = "2024-07-01T09:45:39.89Z" },
+    { url = "https://files.pythonhosted.org/packages/df/56/b8663d7520671b4398b9d97e1ed9f583d4afcbefbda3c6188325e8c297bd/pillow-10.4.0-cp310-cp310-win32.whl", hash = "sha256:bcd5e41a859bf2e84fdc42f4edb7d9aba0a13d29a2abadccafad99de3feff984", size = 2235218, upload-time = "2024-07-01T09:45:42.771Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/72/0203e94a91ddb4a9d5238434ae6c1ca10e610e8487036132ea9bf806ca2a/pillow-10.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:ecd85a8d3e79cd7158dec1c9e5808e821feea088e2f69a974db5edf84dc53141", size = 2554487, upload-time = "2024-07-01T09:45:45.176Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/52/7e7e93d7a6e4290543f17dc6f7d3af4bd0b3dd9926e2e8a35ac2282bc5f4/pillow-10.4.0-cp310-cp310-win_arm64.whl", hash = "sha256:ff337c552345e95702c5fde3158acb0625111017d0e5f24bf3acdb9cc16b90d1", size = 2243219, upload-time = "2024-07-01T09:45:47.274Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/62/c9449f9c3043c37f73e7487ec4ef0c03eb9c9afc91a92b977a67b3c0bbc5/pillow-10.4.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:0a9ec697746f268507404647e531e92889890a087e03681a3606d9b920fbee3c", size = 3509265, upload-time = "2024-07-01T09:45:49.812Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/5f/491dafc7bbf5a3cc1845dc0430872e8096eb9e2b6f8161509d124594ec2d/pillow-10.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe91cb65544a1321e631e696759491ae04a2ea11d36715eca01ce07284738be", size = 3375655, upload-time = "2024-07-01T09:45:52.462Z" },
+    { url = "https://files.pythonhosted.org/packages/73/d5/c4011a76f4207a3c151134cd22a1415741e42fa5ddecec7c0182887deb3d/pillow-10.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dc6761a6efc781e6a1544206f22c80c3af4c8cf461206d46a1e6006e4429ff3", size = 4340304, upload-time = "2024-07-01T09:45:55.006Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/10/c67e20445a707f7a610699bba4fe050583b688d8cd2d202572b257f46600/pillow-10.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e84b6cc6a4a3d76c153a6b19270b3526a5a8ed6b09501d3af891daa2a9de7d6", size = 4452804, upload-time = "2024-07-01T09:45:58.437Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/83/6523837906d1da2b269dee787e31df3b0acb12e3d08f024965a3e7f64665/pillow-10.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:bbc527b519bd3aa9d7f429d152fea69f9ad37c95f0b02aebddff592688998abe", size = 4365126, upload-time = "2024-07-01T09:46:00.713Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/e5/8c68ff608a4203085158cff5cc2a3c534ec384536d9438c405ed6370d080/pillow-10.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:76a911dfe51a36041f2e756b00f96ed84677cdeb75d25c767f296c1c1eda1319", size = 4533541, upload-time = "2024-07-01T09:46:03.235Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/7c/01b8dbdca5bc6785573f4cee96e2358b0918b7b2c7b60d8b6f3abf87a070/pillow-10.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59291fb29317122398786c2d44427bbd1a6d7ff54017075b22be9d21aa59bd8d", size = 4471616, upload-time = "2024-07-01T09:46:05.356Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/57/2899b82394a35a0fbfd352e290945440e3b3785655a03365c0ca8279f351/pillow-10.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:416d3a5d0e8cfe4f27f574362435bc9bae57f679a7158e0096ad2beb427b8696", size = 4600802, upload-time = "2024-07-01T09:46:08.145Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/d7/a44f193d4c26e58ee5d2d9db3d4854b2cfb5b5e08d360a5e03fe987c0086/pillow-10.4.0-cp311-cp311-win32.whl", hash = "sha256:7086cc1d5eebb91ad24ded9f58bec6c688e9f0ed7eb3dbbf1e4800280a896496", size = 2235213, upload-time = "2024-07-01T09:46:10.211Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/d0/5866318eec2b801cdb8c82abf190c8343d8a1cd8bf5a0c17444a6f268291/pillow-10.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cbed61494057c0f83b83eb3a310f0bf774b09513307c434d4366ed64f4128a91", size = 2554498, upload-time = "2024-07-01T09:46:12.685Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/c8/310ac16ac2b97e902d9eb438688de0d961660a87703ad1561fd3dfbd2aa0/pillow-10.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:f5f0c3e969c8f12dd2bb7e0b15d5c468b51e5017e01e2e867335c81903046a22", size = 2243219, upload-time = "2024-07-01T09:46:14.83Z" },
+    { url = "https://files.pythonhosted.org/packages/05/cb/0353013dc30c02a8be34eb91d25e4e4cf594b59e5a55ea1128fde1e5f8ea/pillow-10.4.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:673655af3eadf4df6b5457033f086e90299fdd7a47983a13827acf7459c15d94", size = 3509350, upload-time = "2024-07-01T09:46:17.177Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/cf/5c558a0f247e0bf9cec92bff9b46ae6474dd736f6d906315e60e4075f737/pillow-10.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:866b6942a92f56300012f5fbac71f2d610312ee65e22f1aa2609e491284e5597", size = 3374980, upload-time = "2024-07-01T09:46:19.169Z" },
+    { url = "https://files.pythonhosted.org/packages/84/48/6e394b86369a4eb68b8a1382c78dc092245af517385c086c5094e3b34428/pillow-10.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29dbdc4207642ea6aad70fbde1a9338753d33fb23ed6956e706936706f52dd80", size = 4343799, upload-time = "2024-07-01T09:46:21.883Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/f3/a8c6c11fa84b59b9df0cd5694492da8c039a24cd159f0f6918690105c3be/pillow-10.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf2342ac639c4cf38799a44950bbc2dfcb685f052b9e262f446482afaf4bffca", size = 4459973, upload-time = "2024-07-01T09:46:24.321Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/1b/c14b4197b80150fb64453585247e6fb2e1d93761fa0fa9cf63b102fde822/pillow-10.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f5b92f4d70791b4a67157321c4e8225d60b119c5cc9aee8ecf153aace4aad4ef", size = 4370054, upload-time = "2024-07-01T09:46:26.825Z" },
+    { url = "https://files.pythonhosted.org/packages/55/77/40daddf677897a923d5d33329acd52a2144d54a9644f2a5422c028c6bf2d/pillow-10.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:86dcb5a1eb778d8b25659d5e4341269e8590ad6b4e8b44d9f4b07f8d136c414a", size = 4539484, upload-time = "2024-07-01T09:46:29.355Z" },
+    { url = "https://files.pythonhosted.org/packages/40/54/90de3e4256b1207300fb2b1d7168dd912a2fb4b2401e439ba23c2b2cabde/pillow-10.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:780c072c2e11c9b2c7ca37f9a2ee8ba66f44367ac3e5c7832afcfe5104fd6d1b", size = 4477375, upload-time = "2024-07-01T09:46:31.756Z" },
+    { url = "https://files.pythonhosted.org/packages/13/24/1bfba52f44193860918ff7c93d03d95e3f8748ca1de3ceaf11157a14cf16/pillow-10.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37fb69d905be665f68f28a8bba3c6d3223c8efe1edf14cc4cfa06c241f8c81d9", size = 4608773, upload-time = "2024-07-01T09:46:33.73Z" },
+    { url = "https://files.pythonhosted.org/packages/55/04/5e6de6e6120451ec0c24516c41dbaf80cce1b6451f96561235ef2429da2e/pillow-10.4.0-cp312-cp312-win32.whl", hash = "sha256:7dfecdbad5c301d7b5bde160150b4db4c659cee2b69589705b6f8a0c509d9f42", size = 2235690, upload-time = "2024-07-01T09:46:36.587Z" },
+    { url = "https://files.pythonhosted.org/packages/74/0a/d4ce3c44bca8635bd29a2eab5aa181b654a734a29b263ca8efe013beea98/pillow-10.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1d846aea995ad352d4bdcc847535bd56e0fd88d36829d2c90be880ef1ee4668a", size = 2554951, upload-time = "2024-07-01T09:46:38.777Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/ca/184349ee40f2e92439be9b3502ae6cfc43ac4b50bc4fc6b3de7957563894/pillow-10.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:e553cad5179a66ba15bb18b353a19020e73a7921296a7979c4a2b7f6a5cd57f9", size = 2243427, upload-time = "2024-07-01T09:46:43.15Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/00/706cebe7c2c12a6318aabe5d354836f54adff7156fd9e1bd6c89f4ba0e98/pillow-10.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8bc1a764ed8c957a2e9cacf97c8b2b053b70307cf2996aafd70e91a082e70df3", size = 3525685, upload-time = "2024-07-01T09:46:45.194Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/76/f658cbfa49405e5ecbfb9ba42d07074ad9792031267e782d409fd8fe7c69/pillow-10.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6209bb41dc692ddfee4942517c19ee81b86c864b626dbfca272ec0f7cff5d9fb", size = 3374883, upload-time = "2024-07-01T09:46:47.331Z" },
+    { url = "https://files.pythonhosted.org/packages/46/2b/99c28c4379a85e65378211971c0b430d9c7234b1ec4d59b2668f6299e011/pillow-10.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bee197b30783295d2eb680b311af15a20a8b24024a19c3a26431ff83eb8d1f70", size = 4339837, upload-time = "2024-07-01T09:46:49.647Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/74/b1ec314f624c0c43711fdf0d8076f82d9d802afd58f1d62c2a86878e8615/pillow-10.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ef61f5dd14c300786318482456481463b9d6b91ebe5ef12f405afbba77ed0be", size = 4455562, upload-time = "2024-07-01T09:46:51.811Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/2a/4b04157cb7b9c74372fa867096a1607e6fedad93a44deeff553ccd307868/pillow-10.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:297e388da6e248c98bc4a02e018966af0c5f92dfacf5a5ca22fa01cb3179bca0", size = 4366761, upload-time = "2024-07-01T09:46:53.961Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/7b/8f1d815c1a6a268fe90481232c98dd0e5fa8c75e341a75f060037bd5ceae/pillow-10.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e4db64794ccdf6cb83a59d73405f63adbe2a1887012e308828596100a0b2f6cc", size = 4536767, upload-time = "2024-07-01T09:46:56.664Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/77/05fa64d1f45d12c22c314e7b97398ffb28ef2813a485465017b7978b3ce7/pillow-10.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd2880a07482090a3bcb01f4265f1936a903d70bc740bfcb1fd4e8a2ffe5cf5a", size = 4477989, upload-time = "2024-07-01T09:46:58.977Z" },
+    { url = "https://files.pythonhosted.org/packages/12/63/b0397cfc2caae05c3fb2f4ed1b4fc4fc878f0243510a7a6034ca59726494/pillow-10.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b35b21b819ac1dbd1233317adeecd63495f6babf21b7b2512d244ff6c6ce309", size = 4610255, upload-time = "2024-07-01T09:47:01.189Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/f9/cfaa5082ca9bc4a6de66ffe1c12c2d90bf09c309a5f52b27759a596900e7/pillow-10.4.0-cp313-cp313-win32.whl", hash = "sha256:551d3fd6e9dc15e4c1eb6fc4ba2b39c0c7933fa113b220057a34f4bb3268a060", size = 2235603, upload-time = "2024-07-01T09:47:03.918Z" },
+    { url = "https://files.pythonhosted.org/packages/01/6a/30ff0eef6e0c0e71e55ded56a38d4859bf9d3634a94a88743897b5f96936/pillow-10.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:030abdbe43ee02e0de642aee345efa443740aa4d828bfe8e2eb11922ea6a21ea", size = 2554972, upload-time = "2024-07-01T09:47:06.152Z" },
+    { url = "https://files.pythonhosted.org/packages/48/2c/2e0a52890f269435eee38b21c8218e102c621fe8d8df8b9dd06fabf879ba/pillow-10.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:5b001114dd152cfd6b23befeb28d7aee43553e2402c9f159807bf55f33af8a8d", size = 2243375, upload-time = "2024-07-01T09:47:09.065Z" },
+    { url = "https://files.pythonhosted.org/packages/38/30/095d4f55f3a053392f75e2eae45eba3228452783bab3d9a920b951ac495c/pillow-10.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5b4815f2e65b30f5fbae9dfffa8636d992d49705723fe86a3661806e069352d4", size = 3493889, upload-time = "2024-07-01T09:48:04.815Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/e8/4ff79788803a5fcd5dc35efdc9386af153569853767bff74540725b45863/pillow-10.4.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8f0aef4ef59694b12cadee839e2ba6afeab89c0f39a3adc02ed51d109117b8da", size = 3346160, upload-time = "2024-07-01T09:48:07.206Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/ac/4184edd511b14f760c73f5bb8a5d6fd85c591c8aff7c2229677a355c4179/pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f4727572e2918acaa9077c919cbbeb73bd2b3ebcfe033b72f858fc9fbef0026", size = 3435020, upload-time = "2024-07-01T09:48:09.66Z" },
+    { url = "https://files.pythonhosted.org/packages/da/21/1749cd09160149c0a246a81d646e05f35041619ce76f6493d6a96e8d1103/pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff25afb18123cea58a591ea0244b92eb1e61a1fd497bf6d6384f09bc3262ec3e", size = 3490539, upload-time = "2024-07-01T09:48:12.529Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/f5/f71fe1888b96083b3f6dfa0709101f61fc9e972c0c8d04e9d93ccef2a045/pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dc3e2db6ba09ffd7d02ae9141cfa0ae23393ee7687248d46a7507b75d610f4f5", size = 3476125, upload-time = "2024-07-01T09:48:14.891Z" },
+    { url = "https://files.pythonhosted.org/packages/96/b9/c0362c54290a31866c3526848583a2f45a535aa9d725fd31e25d318c805f/pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02a2be69f9c9b8c1e97cf2713e789d4e398c751ecfd9967c18d0ce304efbf885", size = 3579373, upload-time = "2024-07-01T09:48:17.601Z" },
+    { url = "https://files.pythonhosted.org/packages/52/3b/ce7a01026a7cf46e5452afa86f97a5e88ca97f562cafa76570178ab56d8d/pillow-10.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0755ffd4a0c6f267cccbae2e9903d95477ca2f77c4fcf3a3a09570001856c8a5", size = 2554661, upload-time = "2024-07-01T09:48:20.293Z" },
+]
+
+[[package]]
+name = "playwright"
+version = "1.53.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "greenlet" },
+    { name = "pyee" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f5/e2/2f107be74419280749723bd1197c99351f4b8a0a25e974b9764affb940b2/playwright-1.53.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:48a1a15ce810f0ffe512b6050de9871ea193b41dd3cc1bbed87b8431012419ba", size = 40392498, upload-time = "2025-06-25T21:48:34.17Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/d5/e8c57a4f6fd46059fb2d51da2d22b47afc886b42400f06b742cd4a9ba131/playwright-1.53.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a701f9498a5b87e3f929ec01cea3109fbde75821b19c7ba4bba54f6127b94f76", size = 38647035, upload-time = "2025-06-25T21:48:38.414Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/f3/da18cd7c22398531316e58fd131243fd9156fe7765aae239ae542a5d07d2/playwright-1.53.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:f765498341c4037b4c01e742ae32dd335622f249488ccd77ca32d301d7c82c61", size = 40392502, upload-time = "2025-06-25T21:48:42.293Z" },
+    { url = "https://files.pythonhosted.org/packages/92/32/5d871c3753fbee5113eefc511b9e44c0006a27f2301b4c6bffa4346fbd94/playwright-1.53.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:db19cb5b58f3b15cad3e2419f4910c053e889202fc202461ee183f1530d1db60", size = 45848364, upload-time = "2025-06-25T21:48:45.849Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/6b/9942f86661ff41332f9299db4950623123e60ca71e4fb6e6942fc0212624/playwright-1.53.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9276c9c935fc062f51f4f5107e56420afd6d9a524348dc437793dc2e34c742e3", size = 45235174, upload-time = "2025-06-25T21:48:49.579Z" },
+    { url = "https://files.pythonhosted.org/packages/51/63/28b3f2d36e6a95e88f033d2aa7af06083f6f4aa0d9764759d96033cd053e/playwright-1.53.0-py3-none-win32.whl", hash = "sha256:36eedec101724ff5a000cddab87dd9a72a39f9b3e65a687169c465484e667c06", size = 35415131, upload-time = "2025-06-25T21:48:53.403Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/b5/4ca25974a90d16cfd4a9a953ee5a666cf484a0bdacb4eed484e5cab49e66/playwright-1.53.0-py3-none-win_amd64.whl", hash = "sha256:d68975807a0fd997433537f1dcf2893cda95884a39dc23c6f591b8d5f691e9e8", size = 35415138, upload-time = "2025-06-25T21:48:57.082Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/81/b42ff2116df5d07ccad2dc4eeb20af92c975a1fbc7cd3ed37b678468b813/playwright-1.53.0-py3-none-win_arm64.whl", hash = "sha256:fcfd481f76568d7b011571160e801b47034edd9e2383c43d83a5fb3f35c67885", size = 31188568, upload-time = "2025-06-25T21:49:00.194Z" },
+]
+
+[[package]]
+name = "propcache"
+version = "0.3.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139, upload-time = "2025-06-09T22:56:06.081Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ab/14/510deed325e262afeb8b360043c5d7c960da7d3ecd6d6f9496c9c56dc7f4/propcache-0.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:22d9962a358aedbb7a2e36187ff273adeaab9743373a272976d2e348d08c7770", size = 73178, upload-time = "2025-06-09T22:53:40.126Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/4e/ad52a7925ff01c1325653a730c7ec3175a23f948f08626a534133427dcff/propcache-0.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d0fda578d1dc3f77b6b5a5dce3b9ad69a8250a891760a548df850a5e8da87f3", size = 43133, upload-time = "2025-06-09T22:53:41.965Z" },
+    { url = "https://files.pythonhosted.org/packages/63/7c/e9399ba5da7780871db4eac178e9c2e204c23dd3e7d32df202092a1ed400/propcache-0.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3def3da3ac3ce41562d85db655d18ebac740cb3fa4367f11a52b3da9d03a5cc3", size = 43039, upload-time = "2025-06-09T22:53:43.268Z" },
+    { url = "https://files.pythonhosted.org/packages/22/e1/58da211eb8fdc6fc854002387d38f415a6ca5f5c67c1315b204a5d3e9d7a/propcache-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bec58347a5a6cebf239daba9bda37dffec5b8d2ce004d9fe4edef3d2815137e", size = 201903, upload-time = "2025-06-09T22:53:44.872Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/0a/550ea0f52aac455cb90111c8bab995208443e46d925e51e2f6ebdf869525/propcache-0.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55ffda449a507e9fbd4aca1a7d9aa6753b07d6166140e5a18d2ac9bc49eac220", size = 213362, upload-time = "2025-06-09T22:53:46.707Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/af/9893b7d878deda9bb69fcf54600b247fba7317761b7db11fede6e0f28bd0/propcache-0.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64a67fb39229a8a8491dd42f864e5e263155e729c2e7ff723d6e25f596b1e8cb", size = 210525, upload-time = "2025-06-09T22:53:48.547Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/bb/38fd08b278ca85cde36d848091ad2b45954bc5f15cce494bb300b9285831/propcache-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9da1cf97b92b51253d5b68cf5a2b9e0dafca095e36b7f2da335e27dc6172a614", size = 198283, upload-time = "2025-06-09T22:53:50.067Z" },
+    { url = "https://files.pythonhosted.org/packages/78/8c/9fe55bd01d362bafb413dfe508c48753111a1e269737fa143ba85693592c/propcache-0.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f559e127134b07425134b4065be45b166183fdcb433cb6c24c8e4149056ad50", size = 191872, upload-time = "2025-06-09T22:53:51.438Z" },
+    { url = "https://files.pythonhosted.org/packages/54/14/4701c33852937a22584e08abb531d654c8bcf7948a8f87ad0a4822394147/propcache-0.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aff2e4e06435d61f11a428360a932138d0ec288b0a31dd9bd78d200bd4a2b339", size = 199452, upload-time = "2025-06-09T22:53:53.229Z" },
+    { url = "https://files.pythonhosted.org/packages/16/44/447f2253d859602095356007657ee535e0093215ea0b3d1d6a41d16e5201/propcache-0.3.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4927842833830942a5d0a56e6f4839bc484785b8e1ce8d287359794818633ba0", size = 191567, upload-time = "2025-06-09T22:53:54.541Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/b3/e4756258749bb2d3b46defcff606a2f47410bab82be5824a67e84015b267/propcache-0.3.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6107ddd08b02654a30fb8ad7a132021759d750a82578b94cd55ee2772b6ebea2", size = 193015, upload-time = "2025-06-09T22:53:56.44Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/df/e6d3c7574233164b6330b9fd697beeac402afd367280e6dc377bb99b43d9/propcache-0.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:70bd8b9cd6b519e12859c99f3fc9a93f375ebd22a50296c3a295028bea73b9e7", size = 204660, upload-time = "2025-06-09T22:53:57.839Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/53/e4d31dd5170b4a0e2e6b730f2385a96410633b4833dc25fe5dffd1f73294/propcache-0.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2183111651d710d3097338dd1893fcf09c9f54e27ff1a8795495a16a469cc90b", size = 206105, upload-time = "2025-06-09T22:53:59.638Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/fe/74d54cf9fbe2a20ff786e5f7afcfde446588f0cf15fb2daacfbc267b866c/propcache-0.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fb075ad271405dcad8e2a7ffc9a750a3bf70e533bd86e89f0603e607b93aa64c", size = 196980, upload-time = "2025-06-09T22:54:01.071Z" },
+    { url = "https://files.pythonhosted.org/packages/22/ec/c469c9d59dada8a7679625e0440b544fe72e99311a4679c279562051f6fc/propcache-0.3.2-cp310-cp310-win32.whl", hash = "sha256:404d70768080d3d3bdb41d0771037da19d8340d50b08e104ca0e7f9ce55fce70", size = 37679, upload-time = "2025-06-09T22:54:03.003Z" },
+    { url = "https://files.pythonhosted.org/packages/38/35/07a471371ac89d418f8d0b699c75ea6dca2041fbda360823de21f6a9ce0a/propcache-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:7435d766f978b4ede777002e6b3b6641dd229cd1da8d3d3106a45770365f9ad9", size = 41459, upload-time = "2025-06-09T22:54:04.134Z" },
+    { url = "https://files.pythonhosted.org/packages/80/8d/e8b436717ab9c2cfc23b116d2c297305aa4cd8339172a456d61ebf5669b8/propcache-0.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0b8d2f607bd8f80ddc04088bc2a037fdd17884a6fcadc47a96e334d72f3717be", size = 74207, upload-time = "2025-06-09T22:54:05.399Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/29/1e34000e9766d112171764b9fa3226fa0153ab565d0c242c70e9945318a7/propcache-0.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06766d8f34733416e2e34f46fea488ad5d60726bb9481d3cddf89a6fa2d9603f", size = 43648, upload-time = "2025-06-09T22:54:08.023Z" },
+    { url = "https://files.pythonhosted.org/packages/46/92/1ad5af0df781e76988897da39b5f086c2bf0f028b7f9bd1f409bb05b6874/propcache-0.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2dc1f4a1df4fecf4e6f68013575ff4af84ef6f478fe5344317a65d38a8e6dc9", size = 43496, upload-time = "2025-06-09T22:54:09.228Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/ce/e96392460f9fb68461fabab3e095cb00c8ddf901205be4eae5ce246e5b7e/propcache-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be29c4f4810c5789cf10ddf6af80b041c724e629fa51e308a7a0fb19ed1ef7bf", size = 217288, upload-time = "2025-06-09T22:54:10.466Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/2a/866726ea345299f7ceefc861a5e782b045545ae6940851930a6adaf1fca6/propcache-0.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59d61f6970ecbd8ff2e9360304d5c8876a6abd4530cb752c06586849ac8a9dc9", size = 227456, upload-time = "2025-06-09T22:54:11.828Z" },
+    { url = "https://files.pythonhosted.org/packages/de/03/07d992ccb6d930398689187e1b3c718339a1c06b8b145a8d9650e4726166/propcache-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62180e0b8dbb6b004baec00a7983e4cc52f5ada9cd11f48c3528d8cfa7b96a66", size = 225429, upload-time = "2025-06-09T22:54:13.823Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/e6/116ba39448753b1330f48ab8ba927dcd6cf0baea8a0ccbc512dfb49ba670/propcache-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c144ca294a204c470f18cf4c9d78887810d04a3e2fbb30eea903575a779159df", size = 213472, upload-time = "2025-06-09T22:54:15.232Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/85/f01f5d97e54e428885a5497ccf7f54404cbb4f906688a1690cd51bf597dc/propcache-0.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5c2a784234c28854878d68978265617aa6dc0780e53d44b4d67f3651a17a9a2", size = 204480, upload-time = "2025-06-09T22:54:17.104Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/79/7bf5ab9033b8b8194cc3f7cf1aaa0e9c3256320726f64a3e1f113a812dce/propcache-0.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5745bc7acdafa978ca1642891b82c19238eadc78ba2aaa293c6863b304e552d7", size = 214530, upload-time = "2025-06-09T22:54:18.512Z" },
+    { url = "https://files.pythonhosted.org/packages/31/0b/bd3e0c00509b609317df4a18e6b05a450ef2d9a963e1d8bc9c9415d86f30/propcache-0.3.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c0075bf773d66fa8c9d41f66cc132ecc75e5bb9dd7cce3cfd14adc5ca184cb95", size = 205230, upload-time = "2025-06-09T22:54:19.947Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/23/fae0ff9b54b0de4e819bbe559508da132d5683c32d84d0dc2ccce3563ed4/propcache-0.3.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5f57aa0847730daceff0497f417c9de353c575d8da3579162cc74ac294c5369e", size = 206754, upload-time = "2025-06-09T22:54:21.716Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/7f/ad6a3c22630aaa5f618b4dc3c3598974a72abb4c18e45a50b3cdd091eb2f/propcache-0.3.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:eef914c014bf72d18efb55619447e0aecd5fb7c2e3fa7441e2e5d6099bddff7e", size = 218430, upload-time = "2025-06-09T22:54:23.17Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/2c/ba4f1c0e8a4b4c75910742f0d333759d441f65a1c7f34683b4a74c0ee015/propcache-0.3.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2a4092e8549031e82facf3decdbc0883755d5bbcc62d3aea9d9e185549936dcf", size = 223884, upload-time = "2025-06-09T22:54:25.539Z" },
+    { url = "https://files.pythonhosted.org/packages/88/e4/ebe30fc399e98572019eee82ad0caf512401661985cbd3da5e3140ffa1b0/propcache-0.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85871b050f174bc0bfb437efbdb68aaf860611953ed12418e4361bc9c392749e", size = 211480, upload-time = "2025-06-09T22:54:26.892Z" },
+    { url = "https://files.pythonhosted.org/packages/96/0a/7d5260b914e01d1d0906f7f38af101f8d8ed0dc47426219eeaf05e8ea7c2/propcache-0.3.2-cp311-cp311-win32.whl", hash = "sha256:36c8d9b673ec57900c3554264e630d45980fd302458e4ac801802a7fd2ef7897", size = 37757, upload-time = "2025-06-09T22:54:28.241Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/2d/89fe4489a884bc0da0c3278c552bd4ffe06a1ace559db5ef02ef24ab446b/propcache-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53af8cb6a781b02d2ea079b5b853ba9430fcbe18a8e3ce647d5982a3ff69f39", size = 41500, upload-time = "2025-06-09T22:54:29.4Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674, upload-time = "2025-06-09T22:54:30.551Z" },
+    { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570, upload-time = "2025-06-09T22:54:32.296Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094, upload-time = "2025-06-09T22:54:33.929Z" },
+    { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958, upload-time = "2025-06-09T22:54:35.186Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894, upload-time = "2025-06-09T22:54:36.708Z" },
+    { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672, upload-time = "2025-06-09T22:54:38.062Z" },
+    { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395, upload-time = "2025-06-09T22:54:39.634Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510, upload-time = "2025-06-09T22:54:41.565Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949, upload-time = "2025-06-09T22:54:43.038Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258, upload-time = "2025-06-09T22:54:44.376Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036, upload-time = "2025-06-09T22:54:46.243Z" },
+    { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684, upload-time = "2025-06-09T22:54:47.63Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562, upload-time = "2025-06-09T22:54:48.982Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142, upload-time = "2025-06-09T22:54:50.424Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711, upload-time = "2025-06-09T22:54:52.072Z" },
+    { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479, upload-time = "2025-06-09T22:54:53.234Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/d1/8c747fafa558c603c4ca19d8e20b288aa0c7cda74e9402f50f31eb65267e/propcache-0.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ca592ed634a73ca002967458187109265e980422116c0a107cf93d81f95af945", size = 71286, upload-time = "2025-06-09T22:54:54.369Z" },
+    { url = "https://files.pythonhosted.org/packages/61/99/d606cb7986b60d89c36de8a85d58764323b3a5ff07770a99d8e993b3fa73/propcache-0.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ecb0aad4020e275652ba3975740f241bd12a61f1a784df044cf7477a02bc252", size = 42425, upload-time = "2025-06-09T22:54:55.642Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/96/ef98f91bbb42b79e9bb82bdd348b255eb9d65f14dbbe3b1594644c4073f7/propcache-0.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f08f1cc28bd2eade7a8a3d2954ccc673bb02062e3e7da09bc75d843386b342f", size = 41846, upload-time = "2025-06-09T22:54:57.246Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/ad/3f0f9a705fb630d175146cd7b1d2bf5555c9beaed54e94132b21aac098a6/propcache-0.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a342c834734edb4be5ecb1e9fb48cb64b1e2320fccbd8c54bf8da8f2a84c33", size = 208871, upload-time = "2025-06-09T22:54:58.975Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/38/2085cda93d2c8b6ec3e92af2c89489a36a5886b712a34ab25de9fbca7992/propcache-0.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a544caaae1ac73f1fecfae70ded3e93728831affebd017d53449e3ac052ac1e", size = 215720, upload-time = "2025-06-09T22:55:00.471Z" },
+    { url = "https://files.pythonhosted.org/packages/61/c1/d72ea2dc83ac7f2c8e182786ab0fc2c7bd123a1ff9b7975bee671866fe5f/propcache-0.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310d11aa44635298397db47a3ebce7db99a4cc4b9bbdfcf6c98a60c8d5261cf1", size = 215203, upload-time = "2025-06-09T22:55:01.834Z" },
+    { url = "https://files.pythonhosted.org/packages/af/81/b324c44ae60c56ef12007105f1460d5c304b0626ab0cc6b07c8f2a9aa0b8/propcache-0.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1396592321ac83157ac03a2023aa6cc4a3cc3cfdecb71090054c09e5a7cce3", size = 206365, upload-time = "2025-06-09T22:55:03.199Z" },
+    { url = "https://files.pythonhosted.org/packages/09/73/88549128bb89e66d2aff242488f62869014ae092db63ccea53c1cc75a81d/propcache-0.3.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cabf5b5902272565e78197edb682017d21cf3b550ba0460ee473753f28d23c1", size = 196016, upload-time = "2025-06-09T22:55:04.518Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/3f/3bdd14e737d145114a5eb83cb172903afba7242f67c5877f9909a20d948d/propcache-0.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0a2f2235ac46a7aa25bdeb03a9e7060f6ecbd213b1f9101c43b3090ffb971ef6", size = 205596, upload-time = "2025-06-09T22:55:05.942Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/ca/2f4aa819c357d3107c3763d7ef42c03980f9ed5c48c82e01e25945d437c1/propcache-0.3.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:92b69e12e34869a6970fd2f3da91669899994b47c98f5d430b781c26f1d9f387", size = 200977, upload-time = "2025-06-09T22:55:07.792Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/4a/e65276c7477533c59085251ae88505caf6831c0e85ff8b2e31ebcbb949b1/propcache-0.3.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:54e02207c79968ebbdffc169591009f4474dde3b4679e16634d34c9363ff56b4", size = 197220, upload-time = "2025-06-09T22:55:09.173Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/54/fc7152e517cf5578278b242396ce4d4b36795423988ef39bb8cd5bf274c8/propcache-0.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4adfb44cb588001f68c5466579d3f1157ca07f7504fc91ec87862e2b8e556b88", size = 210642, upload-time = "2025-06-09T22:55:10.62Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/80/abeb4a896d2767bf5f1ea7b92eb7be6a5330645bd7fb844049c0e4045d9d/propcache-0.3.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fd3e6019dc1261cd0291ee8919dd91fbab7b169bb76aeef6c716833a3f65d206", size = 212789, upload-time = "2025-06-09T22:55:12.029Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/db/ea12a49aa7b2b6d68a5da8293dcf50068d48d088100ac016ad92a6a780e6/propcache-0.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c181cad81158d71c41a2bce88edce078458e2dd5ffee7eddd6b05da85079f43", size = 205880, upload-time = "2025-06-09T22:55:13.45Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/e5/9076a0bbbfb65d1198007059c65639dfd56266cf8e477a9707e4b1999ff4/propcache-0.3.2-cp313-cp313-win32.whl", hash = "sha256:8a08154613f2249519e549de2330cf8e2071c2887309a7b07fb56098f5170a02", size = 37220, upload-time = "2025-06-09T22:55:15.284Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/f5/b369e026b09a26cd77aa88d8fffd69141d2ae00a2abaaf5380d2603f4b7f/propcache-0.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e41671f1594fc4ab0a6dec1351864713cb3a279910ae8b58f884a88a0a632c05", size = 40678, upload-time = "2025-06-09T22:55:16.445Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/3a/6ece377b55544941a08d03581c7bc400a3c8cd3c2865900a68d5de79e21f/propcache-0.3.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:9a3cf035bbaf035f109987d9d55dc90e4b0e36e04bbbb95af3055ef17194057b", size = 76560, upload-time = "2025-06-09T22:55:17.598Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/da/64a2bb16418740fa634b0e9c3d29edff1db07f56d3546ca2d86ddf0305e1/propcache-0.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:156c03d07dc1323d8dacaa221fbe028c5c70d16709cdd63502778e6c3ccca1b0", size = 44676, upload-time = "2025-06-09T22:55:18.922Z" },
+    { url = "https://files.pythonhosted.org/packages/36/7b/f025e06ea51cb72c52fb87e9b395cced02786610b60a3ed51da8af017170/propcache-0.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74413c0ba02ba86f55cf60d18daab219f7e531620c15f1e23d95563f505efe7e", size = 44701, upload-time = "2025-06-09T22:55:20.106Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/00/faa1b1b7c3b74fc277f8642f32a4c72ba1d7b2de36d7cdfb676db7f4303e/propcache-0.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f066b437bb3fa39c58ff97ab2ca351db465157d68ed0440abecb21715eb24b28", size = 276934, upload-time = "2025-06-09T22:55:21.5Z" },
+    { url = "https://files.pythonhosted.org/packages/74/ab/935beb6f1756e0476a4d5938ff44bf0d13a055fed880caf93859b4f1baf4/propcache-0.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1304b085c83067914721e7e9d9917d41ad87696bf70f0bc7dee450e9c71ad0a", size = 278316, upload-time = "2025-06-09T22:55:22.918Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/9d/994a5c1ce4389610838d1caec74bdf0e98b306c70314d46dbe4fcf21a3e2/propcache-0.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab50cef01b372763a13333b4e54021bdcb291fc9a8e2ccb9c2df98be51bcde6c", size = 282619, upload-time = "2025-06-09T22:55:24.651Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/00/a10afce3d1ed0287cef2e09506d3be9822513f2c1e96457ee369adb9a6cd/propcache-0.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad3b2a085ec259ad2c2842666b2a0a49dea8463579c606426128925af1ed725", size = 265896, upload-time = "2025-06-09T22:55:26.049Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/a8/2aa6716ffa566ca57c749edb909ad27884680887d68517e4be41b02299f3/propcache-0.3.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:261fa020c1c14deafd54c76b014956e2f86991af198c51139faf41c4d5e83892", size = 252111, upload-time = "2025-06-09T22:55:27.381Z" },
+    { url = "https://files.pythonhosted.org/packages/36/4f/345ca9183b85ac29c8694b0941f7484bf419c7f0fea2d1e386b4f7893eed/propcache-0.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:46d7f8aa79c927e5f987ee3a80205c987717d3659f035c85cf0c3680526bdb44", size = 268334, upload-time = "2025-06-09T22:55:28.747Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/ca/fcd54f78b59e3f97b3b9715501e3147f5340167733d27db423aa321e7148/propcache-0.3.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:6d8f3f0eebf73e3c0ff0e7853f68be638b4043c65a70517bb575eff54edd8dbe", size = 255026, upload-time = "2025-06-09T22:55:30.184Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/95/8e6a6bbbd78ac89c30c225210a5c687790e532ba4088afb8c0445b77ef37/propcache-0.3.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:03c89c1b14a5452cf15403e291c0ccd7751d5b9736ecb2c5bab977ad6c5bcd81", size = 250724, upload-time = "2025-06-09T22:55:31.646Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/b0/0dd03616142baba28e8b2d14ce5df6631b4673850a3d4f9c0f9dd714a404/propcache-0.3.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:0cc17efde71e12bbaad086d679ce575268d70bc123a5a71ea7ad76f70ba30bba", size = 268868, upload-time = "2025-06-09T22:55:33.209Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/98/2c12407a7e4fbacd94ddd32f3b1e3d5231e77c30ef7162b12a60e2dd5ce3/propcache-0.3.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:acdf05d00696bc0447e278bb53cb04ca72354e562cf88ea6f9107df8e7fd9770", size = 271322, upload-time = "2025-06-09T22:55:35.065Z" },
+    { url = "https://files.pythonhosted.org/packages/35/91/9cb56efbb428b006bb85db28591e40b7736847b8331d43fe335acf95f6c8/propcache-0.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4445542398bd0b5d32df908031cb1b30d43ac848e20470a878b770ec2dcc6330", size = 265778, upload-time = "2025-06-09T22:55:36.45Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/4c/b0fe775a2bdd01e176b14b574be679d84fc83958335790f7c9a686c1f468/propcache-0.3.2-cp313-cp313t-win32.whl", hash = "sha256:f86e5d7cd03afb3a1db8e9f9f6eff15794e79e791350ac48a8c924e6f439f394", size = 41175, upload-time = "2025-06-09T22:55:38.436Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/ff/47f08595e3d9b5e149c150f88d9714574f1a7cbd89fe2817158a952674bf/propcache-0.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9704bedf6e7cbe3c65eca4379a9b53ee6a83749f047808cbb5044d40d7d72198", size = 44857, upload-time = "2025-06-09T22:55:39.687Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" },
+]
+
+[[package]]
+name = "psutil"
+version = "7.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003, upload-time = "2025-02-13T21:54:07.946Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051, upload-time = "2025-02-13T21:54:12.36Z" },
+    { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535, upload-time = "2025-02-13T21:54:16.07Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004, upload-time = "2025-02-13T21:54:18.662Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986, upload-time = "2025-02-13T21:54:21.811Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544, upload-time = "2025-02-13T21:54:24.68Z" },
+    { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053, upload-time = "2025-02-13T21:54:34.31Z" },
+    { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885, upload-time = "2025-02-13T21:54:37.486Z" },
+]
+
+[[package]]
+name = "pycparser"
+version = "2.22"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" },
+]
+
+[[package]]
+name = "pydantic"
+version = "2.11.7"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "annotated-types" },
+    { name = "pydantic-core" },
+    { name = "typing-extensions" },
+    { name = "typing-inspection" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/00/dd/4325abf92c39ba8623b5af936ddb36ffcfe0beae70405d456ab1fb2f5b8c/pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db", size = 788350, upload-time = "2025-06-14T08:33:17.137Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6a/c0/ec2b1c8712ca690e5d61979dee872603e92b8a32f94cc1b72d53beab008a/pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b", size = 444782, upload-time = "2025-06-14T08:33:14.905Z" },
+]
+
+[[package]]
+name = "pydantic-core"
+version = "2.33.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195, upload-time = "2025-04-23T18:33:52.104Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e5/92/b31726561b5dae176c2d2c2dc43a9c5bfba5d32f96f8b4c0a600dd492447/pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8", size = 2028817, upload-time = "2025-04-23T18:30:43.919Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/44/3f0b95fafdaca04a483c4e685fe437c6891001bf3ce8b2fded82b9ea3aa1/pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d", size = 1861357, upload-time = "2025-04-23T18:30:46.372Z" },
+    { url = "https://files.pythonhosted.org/packages/30/97/e8f13b55766234caae05372826e8e4b3b96e7b248be3157f53237682e43c/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d", size = 1898011, upload-time = "2025-04-23T18:30:47.591Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/a3/99c48cf7bafc991cc3ee66fd544c0aae8dc907b752f1dad2d79b1b5a471f/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572", size = 1982730, upload-time = "2025-04-23T18:30:49.328Z" },
+    { url = "https://files.pythonhosted.org/packages/de/8e/a5b882ec4307010a840fb8b58bd9bf65d1840c92eae7534c7441709bf54b/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02", size = 2136178, upload-time = "2025-04-23T18:30:50.907Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/bb/71e35fc3ed05af6834e890edb75968e2802fe98778971ab5cba20a162315/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b", size = 2736462, upload-time = "2025-04-23T18:30:52.083Z" },
+    { url = "https://files.pythonhosted.org/packages/31/0d/c8f7593e6bc7066289bbc366f2235701dcbebcd1ff0ef8e64f6f239fb47d/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2", size = 2005652, upload-time = "2025-04-23T18:30:53.389Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/7a/996d8bd75f3eda405e3dd219ff5ff0a283cd8e34add39d8ef9157e722867/pydantic_core-2.33.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a", size = 2113306, upload-time = "2025-04-23T18:30:54.661Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/84/daf2a6fb2db40ffda6578a7e8c5a6e9c8affb251a05c233ae37098118788/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac", size = 2073720, upload-time = "2025-04-23T18:30:56.11Z" },
+    { url = "https://files.pythonhosted.org/packages/77/fb/2258da019f4825128445ae79456a5499c032b55849dbd5bed78c95ccf163/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a", size = 2244915, upload-time = "2025-04-23T18:30:57.501Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/7a/925ff73756031289468326e355b6fa8316960d0d65f8b5d6b3a3e7866de7/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b", size = 2241884, upload-time = "2025-04-23T18:30:58.867Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/b0/249ee6d2646f1cdadcb813805fe76265745c4010cf20a8eba7b0e639d9b2/pydantic_core-2.33.2-cp310-cp310-win32.whl", hash = "sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22", size = 1910496, upload-time = "2025-04-23T18:31:00.078Z" },
+    { url = "https://files.pythonhosted.org/packages/66/ff/172ba8f12a42d4b552917aa65d1f2328990d3ccfc01d5b7c943ec084299f/pydantic_core-2.33.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640", size = 1955019, upload-time = "2025-04-23T18:31:01.335Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/8d/71db63483d518cbbf290261a1fc2839d17ff89fce7089e08cad07ccfce67/pydantic_core-2.33.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7", size = 2028584, upload-time = "2025-04-23T18:31:03.106Z" },
+    { url = "https://files.pythonhosted.org/packages/24/2f/3cfa7244ae292dd850989f328722d2aef313f74ffc471184dc509e1e4e5a/pydantic_core-2.33.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246", size = 1855071, upload-time = "2025-04-23T18:31:04.621Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/d3/4ae42d33f5e3f50dd467761304be2fa0a9417fbf09735bc2cce003480f2a/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f", size = 1897823, upload-time = "2025-04-23T18:31:06.377Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/f3/aa5976e8352b7695ff808599794b1fba2a9ae2ee954a3426855935799488/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc", size = 1983792, upload-time = "2025-04-23T18:31:07.93Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/7a/cda9b5a23c552037717f2b2a5257e9b2bfe45e687386df9591eff7b46d28/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de", size = 2136338, upload-time = "2025-04-23T18:31:09.283Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/9f/b8f9ec8dd1417eb9da784e91e1667d58a2a4a7b7b34cf4af765ef663a7e5/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a", size = 2730998, upload-time = "2025-04-23T18:31:11.7Z" },
+    { url = "https://files.pythonhosted.org/packages/47/bc/cd720e078576bdb8255d5032c5d63ee5c0bf4b7173dd955185a1d658c456/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef", size = 2003200, upload-time = "2025-04-23T18:31:13.536Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/22/3602b895ee2cd29d11a2b349372446ae9727c32e78a94b3d588a40fdf187/pydantic_core-2.33.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e", size = 2113890, upload-time = "2025-04-23T18:31:15.011Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/e6/e3c5908c03cf00d629eb38393a98fccc38ee0ce8ecce32f69fc7d7b558a7/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d", size = 2073359, upload-time = "2025-04-23T18:31:16.393Z" },
+    { url = "https://files.pythonhosted.org/packages/12/e7/6a36a07c59ebefc8777d1ffdaf5ae71b06b21952582e4b07eba88a421c79/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30", size = 2245883, upload-time = "2025-04-23T18:31:17.892Z" },
+    { url = "https://files.pythonhosted.org/packages/16/3f/59b3187aaa6cc0c1e6616e8045b284de2b6a87b027cce2ffcea073adf1d2/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf", size = 2241074, upload-time = "2025-04-23T18:31:19.205Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/ed/55532bb88f674d5d8f67ab121a2a13c385df382de2a1677f30ad385f7438/pydantic_core-2.33.2-cp311-cp311-win32.whl", hash = "sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51", size = 1910538, upload-time = "2025-04-23T18:31:20.541Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/1b/25b7cccd4519c0b23c2dd636ad39d381abf113085ce4f7bec2b0dc755eb1/pydantic_core-2.33.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab", size = 1952909, upload-time = "2025-04-23T18:31:22.371Z" },
+    { url = "https://files.pythonhosted.org/packages/49/a9/d809358e49126438055884c4366a1f6227f0f84f635a9014e2deb9b9de54/pydantic_core-2.33.2-cp311-cp311-win_arm64.whl", hash = "sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65", size = 1897786, upload-time = "2025-04-23T18:31:24.161Z" },
+    { url = "https://files.pythonhosted.org/packages/18/8a/2b41c97f554ec8c71f2a8a5f85cb56a8b0956addfe8b0efb5b3d77e8bdc3/pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc", size = 2009000, upload-time = "2025-04-23T18:31:25.863Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/02/6224312aacb3c8ecbaa959897af57181fb6cf3a3d7917fd44d0f2917e6f2/pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7", size = 1847996, upload-time = "2025-04-23T18:31:27.341Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/46/6dcdf084a523dbe0a0be59d054734b86a981726f221f4562aed313dbcb49/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025", size = 1880957, upload-time = "2025-04-23T18:31:28.956Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/6b/1ec2c03837ac00886ba8160ce041ce4e325b41d06a034adbef11339ae422/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011", size = 1964199, upload-time = "2025-04-23T18:31:31.025Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/1d/6bf34d6adb9debd9136bd197ca72642203ce9aaaa85cfcbfcf20f9696e83/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f", size = 2120296, upload-time = "2025-04-23T18:31:32.514Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/94/2bd0aaf5a591e974b32a9f7123f16637776c304471a0ab33cf263cf5591a/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88", size = 2676109, upload-time = "2025-04-23T18:31:33.958Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/41/4b043778cf9c4285d59742281a769eac371b9e47e35f98ad321349cc5d61/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1", size = 2002028, upload-time = "2025-04-23T18:31:39.095Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/d5/7bb781bf2748ce3d03af04d5c969fa1308880e1dca35a9bd94e1a96a922e/pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b", size = 2100044, upload-time = "2025-04-23T18:31:41.034Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/36/def5e53e1eb0ad896785702a5bbfd25eed546cdcf4087ad285021a90ed53/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1", size = 2058881, upload-time = "2025-04-23T18:31:42.757Z" },
+    { url = "https://files.pythonhosted.org/packages/01/6c/57f8d70b2ee57fc3dc8b9610315949837fa8c11d86927b9bb044f8705419/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6", size = 2227034, upload-time = "2025-04-23T18:31:44.304Z" },
+    { url = "https://files.pythonhosted.org/packages/27/b9/9c17f0396a82b3d5cbea4c24d742083422639e7bb1d5bf600e12cb176a13/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea", size = 2234187, upload-time = "2025-04-23T18:31:45.891Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/6a/adf5734ffd52bf86d865093ad70b2ce543415e0e356f6cacabbc0d9ad910/pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290", size = 1892628, upload-time = "2025-04-23T18:31:47.819Z" },
+    { url = "https://files.pythonhosted.org/packages/43/e4/5479fecb3606c1368d496a825d8411e126133c41224c1e7238be58b87d7e/pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2", size = 1955866, upload-time = "2025-04-23T18:31:49.635Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/24/8b11e8b3e2be9dd82df4b11408a67c61bb4dc4f8e11b5b0fc888b38118b5/pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab", size = 1888894, upload-time = "2025-04-23T18:31:51.609Z" },
+    { url = "https://files.pythonhosted.org/packages/46/8c/99040727b41f56616573a28771b1bfa08a3d3fe74d3d513f01251f79f172/pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f", size = 2015688, upload-time = "2025-04-23T18:31:53.175Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/cc/5999d1eb705a6cefc31f0b4a90e9f7fc400539b1a1030529700cc1b51838/pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6", size = 1844808, upload-time = "2025-04-23T18:31:54.79Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/5e/a0a7b8885c98889a18b6e376f344da1ef323d270b44edf8174d6bce4d622/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef", size = 1885580, upload-time = "2025-04-23T18:31:57.393Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/2a/953581f343c7d11a304581156618c3f592435523dd9d79865903272c256a/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a", size = 1973859, upload-time = "2025-04-23T18:31:59.065Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/55/f1a813904771c03a3f97f676c62cca0c0a4138654107c1b61f19c644868b/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916", size = 2120810, upload-time = "2025-04-23T18:32:00.78Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/c3/053389835a996e18853ba107a63caae0b9deb4a276c6b472931ea9ae6e48/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a", size = 2676498, upload-time = "2025-04-23T18:32:02.418Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/3c/f4abd740877a35abade05e437245b192f9d0ffb48bbbbd708df33d3cda37/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d", size = 2000611, upload-time = "2025-04-23T18:32:04.152Z" },
+    { url = "https://files.pythonhosted.org/packages/59/a7/63ef2fed1837d1121a894d0ce88439fe3e3b3e48c7543b2a4479eb99c2bd/pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56", size = 2107924, upload-time = "2025-04-23T18:32:06.129Z" },
+    { url = "https://files.pythonhosted.org/packages/04/8f/2551964ef045669801675f1cfc3b0d74147f4901c3ffa42be2ddb1f0efc4/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5", size = 2063196, upload-time = "2025-04-23T18:32:08.178Z" },
+    { url = "https://files.pythonhosted.org/packages/26/bd/d9602777e77fc6dbb0c7db9ad356e9a985825547dce5ad1d30ee04903918/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e", size = 2236389, upload-time = "2025-04-23T18:32:10.242Z" },
+    { url = "https://files.pythonhosted.org/packages/42/db/0e950daa7e2230423ab342ae918a794964b053bec24ba8af013fc7c94846/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162", size = 2239223, upload-time = "2025-04-23T18:32:12.382Z" },
+    { url = "https://files.pythonhosted.org/packages/58/4d/4f937099c545a8a17eb52cb67fe0447fd9a373b348ccfa9a87f141eeb00f/pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849", size = 1900473, upload-time = "2025-04-23T18:32:14.034Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/75/4a0a9bac998d78d889def5e4ef2b065acba8cae8c93696906c3a91f310ca/pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9", size = 1955269, upload-time = "2025-04-23T18:32:15.783Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/86/1beda0576969592f1497b4ce8e7bc8cbdf614c352426271b1b10d5f0aa64/pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9", size = 1893921, upload-time = "2025-04-23T18:32:18.473Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/7d/e09391c2eebeab681df2b74bfe6c43422fffede8dc74187b2b0bf6fd7571/pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac", size = 1806162, upload-time = "2025-04-23T18:32:20.188Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/3d/847b6b1fed9f8ed3bb95a9ad04fbd0b212e832d4f0f50ff4d9ee5a9f15cf/pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5", size = 1981560, upload-time = "2025-04-23T18:32:22.354Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/9a/e73262f6c6656262b5fdd723ad90f518f579b7bc8622e43a942eec53c938/pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9", size = 1935777, upload-time = "2025-04-23T18:32:25.088Z" },
+    { url = "https://files.pythonhosted.org/packages/30/68/373d55e58b7e83ce371691f6eaa7175e3a24b956c44628eb25d7da007917/pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa", size = 2023982, upload-time = "2025-04-23T18:32:53.14Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/16/145f54ac08c96a63d8ed6442f9dec17b2773d19920b627b18d4f10a061ea/pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29", size = 1858412, upload-time = "2025-04-23T18:32:55.52Z" },
+    { url = "https://files.pythonhosted.org/packages/41/b1/c6dc6c3e2de4516c0bb2c46f6a373b91b5660312342a0cf5826e38ad82fa/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d", size = 1892749, upload-time = "2025-04-23T18:32:57.546Z" },
+    { url = "https://files.pythonhosted.org/packages/12/73/8cd57e20afba760b21b742106f9dbdfa6697f1570b189c7457a1af4cd8a0/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e", size = 2067527, upload-time = "2025-04-23T18:32:59.771Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/d5/0bb5d988cc019b3cba4a78f2d4b3854427fc47ee8ec8e9eaabf787da239c/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c", size = 2108225, upload-time = "2025-04-23T18:33:04.51Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/c5/00c02d1571913d496aabf146106ad8239dc132485ee22efe08085084ff7c/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec", size = 2069490, upload-time = "2025-04-23T18:33:06.391Z" },
+    { url = "https://files.pythonhosted.org/packages/22/a8/dccc38768274d3ed3a59b5d06f59ccb845778687652daa71df0cab4040d7/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052", size = 2237525, upload-time = "2025-04-23T18:33:08.44Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/e7/4f98c0b125dda7cf7ccd14ba936218397b44f50a56dd8c16a3091df116c3/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c", size = 2238446, upload-time = "2025-04-23T18:33:10.313Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/91/2ec36480fdb0b783cd9ef6795753c1dea13882f2e68e73bce76ae8c21e6a/pydantic_core-2.33.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808", size = 2066678, upload-time = "2025-04-23T18:33:12.224Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/27/d4ae6487d73948d6f20dddcd94be4ea43e74349b56eba82e9bdee2d7494c/pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8", size = 2025200, upload-time = "2025-04-23T18:33:14.199Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/b8/b3cb95375f05d33801024079b9392a5ab45267a63400bf1866e7ce0f0de4/pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593", size = 1859123, upload-time = "2025-04-23T18:33:16.555Z" },
+    { url = "https://files.pythonhosted.org/packages/05/bc/0d0b5adeda59a261cd30a1235a445bf55c7e46ae44aea28f7bd6ed46e091/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612", size = 1892852, upload-time = "2025-04-23T18:33:18.513Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/11/d37bdebbda2e449cb3f519f6ce950927b56d62f0b84fd9cb9e372a26a3d5/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7", size = 2067484, upload-time = "2025-04-23T18:33:20.475Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/55/1f95f0a05ce72ecb02a8a8a1c3be0579bbc29b1d5ab68f1378b7bebc5057/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e", size = 2108896, upload-time = "2025-04-23T18:33:22.501Z" },
+    { url = "https://files.pythonhosted.org/packages/53/89/2b2de6c81fa131f423246a9109d7b2a375e83968ad0800d6e57d0574629b/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8", size = 2069475, upload-time = "2025-04-23T18:33:24.528Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/e9/1f7efbe20d0b2b10f6718944b5d8ece9152390904f29a78e68d4e7961159/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf", size = 2239013, upload-time = "2025-04-23T18:33:26.621Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/b2/5309c905a93811524a49b4e031e9851a6b00ff0fb668794472ea7746b448/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb", size = 2238715, upload-time = "2025-04-23T18:33:28.656Z" },
+    { url = "https://files.pythonhosted.org/packages/32/56/8a7ca5d2cd2cda1d245d34b1c9a942920a718082ae8e54e5f3e5a58b7add/pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1", size = 2066757, upload-time = "2025-04-23T18:33:30.645Z" },
+]
+
+[[package]]
+name = "pyee"
+version = "13.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/95/03/1fd98d5841cd7964a27d729ccf2199602fe05eb7a405c1462eb7277945ed/pyee-13.0.0.tar.gz", hash = "sha256:b391e3c5a434d1f5118a25615001dbc8f669cf410ab67d04c4d4e07c55481c37", size = 31250, upload-time = "2025-03-17T18:53:15.955Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9b/4d/b9add7c84060d4c1906abe9a7e5359f2a60f7a9a4f67268b2766673427d8/pyee-13.0.0-py3-none-any.whl", hash = "sha256:48195a3cddb3b1515ce0695ed76036b5ccc2ef3a9f963ff9f77aec0139845498", size = 15730, upload-time = "2025-03-17T18:53:14.532Z" },
+]
+
+[[package]]
+name = "pygments"
+version = "2.19.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
+]
+
+[[package]]
+name = "pyopenssl"
+version = "25.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cryptography" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/80/be/97b83a464498a79103036bc74d1038df4a7ef0e402cfaf4d5e113fb14759/pyopenssl-25.3.0.tar.gz", hash = "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329", size = 184073, upload-time = "2025-09-17T00:32:21.037Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/81/ef2b1dfd1862567d573a4fdbc9f969067621764fbb74338496840a1d2977/pyopenssl-25.3.0-py3-none-any.whl", hash = "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6", size = 57268, upload-time = "2025-09-17T00:32:19.474Z" },
+]
+
+[[package]]
+name = "pypdf"
+version = "6.4.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/0c/e0/57f914ae9fedbc91fe3ebe74b78c88903943ec9c232b6da15947bb3bf8ab/pypdf-6.4.1.tar.gz", hash = "sha256:36eb0b52730fc3077d2b8d4122751e696d46af9ef9e5383db492df1ab0cc4647", size = 5275322, upload-time = "2025-12-07T14:19:27.922Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/db/ef/68c0f473d8b8764b23f199450dfa035e6f2206e67e9bde5dd695bab9bdf0/pypdf-6.4.1-py3-none-any.whl", hash = "sha256:1782ee0766f0b77defc305f1eb2bafe738a2ef6313f3f3d2ee85b4542ba7e535", size = 328325, upload-time = "2025-12-07T14:19:26.286Z" },
+]
+
+[[package]]
+name = "pysocks"
+version = "1.7.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" },
+]
+
+[[package]]
+name = "python-dotenv"
+version = "1.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f6/b0/4bc07ccd3572a2f9df7e6782f52b0c6c90dcbb803ac4a167702d7d0dfe1e/python_dotenv-1.1.1.tar.gz", hash = "sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab", size = 41978, upload-time = "2025-06-24T04:21:07.341Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5f/ed/539768cf28c661b5b068d66d96a2f155c4971a5d55684a514c1a0e0dec2f/python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc", size = 20556, upload-time = "2025-06-24T04:21:06.073Z" },
+]
+
+[[package]]
+name = "pyyaml"
+version = "6.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" },
+    { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" },
+    { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" },
+    { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" },
+    { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" },
+    { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" },
+    { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" },
+    { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" },
+    { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" },
+    { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" },
+    { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" },
+    { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" },
+    { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" },
+]
+
+[[package]]
+name = "rank-bm25"
+version = "0.2.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fc/0a/f9579384aa017d8b4c15613f86954b92a95a93d641cc849182467cf0bb3b/rank_bm25-0.2.2.tar.gz", hash = "sha256:096ccef76f8188563419aaf384a02f0ea459503fdf77901378d4fd9d87e5e51d", size = 8347, upload-time = "2022-02-16T12:10:52.196Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2a/21/f691fb2613100a62b3fa91e9988c991e9ca5b89ea31c0d3152a3210344f9/rank_bm25-0.2.2-py3-none-any.whl", hash = "sha256:7bd4a95571adadfc271746fa146a4bcfd89c0cf731e49c3d1ad863290adbe8ae", size = 8584, upload-time = "2022-02-16T12:10:50.626Z" },
+]
+
+[[package]]
+name = "referencing"
+version = "0.36.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "attrs" },
+    { name = "rpds-py" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775, upload-time = "2025-01-25T08:48:14.241Z" },
+]
+
+[[package]]
+name = "regex"
+version = "2024.11.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8e/5f/bd69653fbfb76cf8604468d3b4ec4c403197144c7bfe0e6a5fc9e02a07cb/regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519", size = 399494, upload-time = "2024-11-06T20:12:31.635Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/95/3c/4651f6b130c6842a8f3df82461a8950f923925db8b6961063e82744bddcc/regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91", size = 482674, upload-time = "2024-11-06T20:08:57.575Z" },
+    { url = "https://files.pythonhosted.org/packages/15/51/9f35d12da8434b489c7b7bffc205c474a0a9432a889457026e9bc06a297a/regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0", size = 287684, upload-time = "2024-11-06T20:08:59.787Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/18/b731f5510d1b8fb63c6b6d3484bfa9a59b84cc578ac8b5172970e05ae07c/regex-2024.11.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164d8b7b3b4bcb2068b97428060b2a53be050085ef94eca7f240e7947f1b080e", size = 284589, upload-time = "2024-11-06T20:09:01.896Z" },
+    { url = "https://files.pythonhosted.org/packages/78/a2/6dd36e16341ab95e4c6073426561b9bfdeb1a9c9b63ab1b579c2e96cb105/regex-2024.11.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3660c82f209655a06b587d55e723f0b813d3a7db2e32e5e7dc64ac2a9e86fde", size = 782511, upload-time = "2024-11-06T20:09:04.062Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/2b/323e72d5d2fd8de0d9baa443e1ed70363ed7e7b2fb526f5950c5cb99c364/regex-2024.11.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d22326fcdef5e08c154280b71163ced384b428343ae16a5ab2b3354aed12436e", size = 821149, upload-time = "2024-11-06T20:09:06.237Z" },
+    { url = "https://files.pythonhosted.org/packages/90/30/63373b9ea468fbef8a907fd273e5c329b8c9535fee36fc8dba5fecac475d/regex-2024.11.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1ac758ef6aebfc8943560194e9fd0fa18bcb34d89fd8bd2af18183afd8da3a2", size = 809707, upload-time = "2024-11-06T20:09:07.715Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/98/26d3830875b53071f1f0ae6d547f1d98e964dd29ad35cbf94439120bb67a/regex-2024.11.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:997d6a487ff00807ba810e0f8332c18b4eb8d29463cfb7c820dc4b6e7562d0cf", size = 781702, upload-time = "2024-11-06T20:09:10.101Z" },
+    { url = "https://files.pythonhosted.org/packages/87/55/eb2a068334274db86208ab9d5599ffa63631b9f0f67ed70ea7c82a69bbc8/regex-2024.11.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02a02d2bb04fec86ad61f3ea7f49c015a0681bf76abb9857f945d26159d2968c", size = 771976, upload-time = "2024-11-06T20:09:11.566Z" },
+    { url = "https://files.pythonhosted.org/packages/74/c0/be707bcfe98254d8f9d2cff55d216e946f4ea48ad2fd8cf1428f8c5332ba/regex-2024.11.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f02f93b92358ee3f78660e43b4b0091229260c5d5c408d17d60bf26b6c900e86", size = 697397, upload-time = "2024-11-06T20:09:13.119Z" },
+    { url = "https://files.pythonhosted.org/packages/49/dc/bb45572ceb49e0f6509f7596e4ba7031f6819ecb26bc7610979af5a77f45/regex-2024.11.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:06eb1be98df10e81ebaded73fcd51989dcf534e3c753466e4b60c4697a003b67", size = 768726, upload-time = "2024-11-06T20:09:14.85Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/db/f43fd75dc4c0c2d96d0881967897926942e935d700863666f3c844a72ce6/regex-2024.11.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:040df6fe1a5504eb0f04f048e6d09cd7c7110fef851d7c567a6b6e09942feb7d", size = 775098, upload-time = "2024-11-06T20:09:16.504Z" },
+    { url = "https://files.pythonhosted.org/packages/99/d7/f94154db29ab5a89d69ff893159b19ada89e76b915c1293e98603d39838c/regex-2024.11.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabbfc59f2c6edba2a6622c647b716e34e8e3867e0ab975412c5c2f79b82da2", size = 839325, upload-time = "2024-11-06T20:09:18.698Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/17/3cbfab1f23356fbbf07708220ab438a7efa1e0f34195bf857433f79f1788/regex-2024.11.6-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8447d2d39b5abe381419319f942de20b7ecd60ce86f16a23b0698f22e1b70008", size = 843277, upload-time = "2024-11-06T20:09:21.725Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/f2/48b393b51900456155de3ad001900f94298965e1cad1c772b87f9cfea011/regex-2024.11.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:da8f5fc57d1933de22a9e23eec290a0d8a5927a5370d24bda9a6abe50683fe62", size = 773197, upload-time = "2024-11-06T20:09:24.092Z" },
+    { url = "https://files.pythonhosted.org/packages/45/3f/ef9589aba93e084cd3f8471fded352826dcae8489b650d0b9b27bc5bba8a/regex-2024.11.6-cp310-cp310-win32.whl", hash = "sha256:b489578720afb782f6ccf2840920f3a32e31ba28a4b162e13900c3e6bd3f930e", size = 261714, upload-time = "2024-11-06T20:09:26.36Z" },
+    { url = "https://files.pythonhosted.org/packages/42/7e/5f1b92c8468290c465fd50c5318da64319133231415a8aa6ea5ab995a815/regex-2024.11.6-cp310-cp310-win_amd64.whl", hash = "sha256:5071b2093e793357c9d8b2929dfc13ac5f0a6c650559503bb81189d0a3814519", size = 274042, upload-time = "2024-11-06T20:09:28.762Z" },
+    { url = "https://files.pythonhosted.org/packages/58/58/7e4d9493a66c88a7da6d205768119f51af0f684fe7be7bac8328e217a52c/regex-2024.11.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5478c6962ad548b54a591778e93cd7c456a7a29f8eca9c49e4f9a806dcc5d638", size = 482669, upload-time = "2024-11-06T20:09:31.064Z" },
+    { url = "https://files.pythonhosted.org/packages/34/4c/8f8e631fcdc2ff978609eaeef1d6994bf2f028b59d9ac67640ed051f1218/regex-2024.11.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c89a8cc122b25ce6945f0423dc1352cb9593c68abd19223eebbd4e56612c5b7", size = 287684, upload-time = "2024-11-06T20:09:32.915Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/1b/f0e4d13e6adf866ce9b069e191f303a30ab1277e037037a365c3aad5cc9c/regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94d87b689cdd831934fa3ce16cc15cd65748e6d689f5d2b8f4f4df2065c9fa20", size = 284589, upload-time = "2024-11-06T20:09:35.504Z" },
+    { url = "https://files.pythonhosted.org/packages/25/4d/ab21047f446693887f25510887e6820b93f791992994f6498b0318904d4a/regex-2024.11.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1062b39a0a2b75a9c694f7a08e7183a80c63c0d62b301418ffd9c35f55aaa114", size = 792121, upload-time = "2024-11-06T20:09:37.701Z" },
+    { url = "https://files.pythonhosted.org/packages/45/ee/c867e15cd894985cb32b731d89576c41a4642a57850c162490ea34b78c3b/regex-2024.11.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:167ed4852351d8a750da48712c3930b031f6efdaa0f22fa1933716bfcd6bf4a3", size = 831275, upload-time = "2024-11-06T20:09:40.371Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/12/b0f480726cf1c60f6536fa5e1c95275a77624f3ac8fdccf79e6727499e28/regex-2024.11.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d548dafee61f06ebdb584080621f3e0c23fff312f0de1afc776e2a2ba99a74f", size = 818257, upload-time = "2024-11-06T20:09:43.059Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/ce/0d0e61429f603bac433910d99ef1a02ce45a8967ffbe3cbee48599e62d88/regex-2024.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a19f302cd1ce5dd01a9099aaa19cae6173306d1302a43b627f62e21cf18ac0", size = 792727, upload-time = "2024-11-06T20:09:48.19Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/c1/243c83c53d4a419c1556f43777ccb552bccdf79d08fda3980e4e77dd9137/regex-2024.11.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bec9931dfb61ddd8ef2ebc05646293812cb6b16b60cf7c9511a832b6f1854b55", size = 780667, upload-time = "2024-11-06T20:09:49.828Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/f4/75eb0dd4ce4b37f04928987f1d22547ddaf6c4bae697623c1b05da67a8aa/regex-2024.11.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9714398225f299aa85267fd222f7142fcb5c769e73d7733344efc46f2ef5cf89", size = 776963, upload-time = "2024-11-06T20:09:51.819Z" },
+    { url = "https://files.pythonhosted.org/packages/16/5d/95c568574e630e141a69ff8a254c2f188b4398e813c40d49228c9bbd9875/regex-2024.11.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:202eb32e89f60fc147a41e55cb086db2a3f8cb82f9a9a88440dcfc5d37faae8d", size = 784700, upload-time = "2024-11-06T20:09:53.982Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/b5/f8495c7917f15cc6fee1e7f395e324ec3e00ab3c665a7dc9d27562fd5290/regex-2024.11.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4181b814e56078e9b00427ca358ec44333765f5ca1b45597ec7446d3a1ef6e34", size = 848592, upload-time = "2024-11-06T20:09:56.222Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/80/6dd7118e8cb212c3c60b191b932dc57db93fb2e36fb9e0e92f72a5909af9/regex-2024.11.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:068376da5a7e4da51968ce4c122a7cd31afaaec4fccc7856c92f63876e57b51d", size = 852929, upload-time = "2024-11-06T20:09:58.642Z" },
+    { url = "https://files.pythonhosted.org/packages/11/9b/5a05d2040297d2d254baf95eeeb6df83554e5e1df03bc1a6687fc4ba1f66/regex-2024.11.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f2c4184420d881a3475fb2c6f4d95d53a8d50209a2500723d831036f7c45", size = 781213, upload-time = "2024-11-06T20:10:00.867Z" },
+    { url = "https://files.pythonhosted.org/packages/26/b7/b14e2440156ab39e0177506c08c18accaf2b8932e39fb092074de733d868/regex-2024.11.6-cp311-cp311-win32.whl", hash = "sha256:c36f9b6f5f8649bb251a5f3f66564438977b7ef8386a52460ae77e6070d309d9", size = 261734, upload-time = "2024-11-06T20:10:03.361Z" },
+    { url = "https://files.pythonhosted.org/packages/80/32/763a6cc01d21fb3819227a1cc3f60fd251c13c37c27a73b8ff4315433a8e/regex-2024.11.6-cp311-cp311-win_amd64.whl", hash = "sha256:02e28184be537f0e75c1f9b2f8847dc51e08e6e171c6bde130b2687e0c33cf60", size = 274052, upload-time = "2024-11-06T20:10:05.179Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/30/9a87ce8336b172cc232a0db89a3af97929d06c11ceaa19d97d84fa90a8f8/regex-2024.11.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:52fb28f528778f184f870b7cf8f225f5eef0a8f6e3778529bdd40c7b3920796a", size = 483781, upload-time = "2024-11-06T20:10:07.07Z" },
+    { url = "https://files.pythonhosted.org/packages/01/e8/00008ad4ff4be8b1844786ba6636035f7ef926db5686e4c0f98093612add/regex-2024.11.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9", size = 288455, upload-time = "2024-11-06T20:10:09.117Z" },
+    { url = "https://files.pythonhosted.org/packages/60/85/cebcc0aff603ea0a201667b203f13ba75d9fc8668fab917ac5b2de3967bc/regex-2024.11.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805e6b60c54bf766b251e94526ebad60b7de0c70f70a4e6210ee2891acb70bf2", size = 284759, upload-time = "2024-11-06T20:10:11.155Z" },
+    { url = "https://files.pythonhosted.org/packages/94/2b/701a4b0585cb05472a4da28ee28fdfe155f3638f5e1ec92306d924e5faf0/regex-2024.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b85c2530be953a890eaffde05485238f07029600e8f098cdf1848d414a8b45e4", size = 794976, upload-time = "2024-11-06T20:10:13.24Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/bf/fa87e563bf5fee75db8915f7352e1887b1249126a1be4813837f5dbec965/regex-2024.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb26437975da7dc36b7efad18aa9dd4ea569d2357ae6b783bf1118dabd9ea577", size = 833077, upload-time = "2024-11-06T20:10:15.37Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/56/7295e6bad94b047f4d0834e4779491b81216583c00c288252ef625c01d23/regex-2024.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abfa5080c374a76a251ba60683242bc17eeb2c9818d0d30117b4486be10c59d3", size = 823160, upload-time = "2024-11-06T20:10:19.027Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/13/e3b075031a738c9598c51cfbc4c7879e26729c53aa9cca59211c44235314/regex-2024.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b7fa6606c2881c1db9479b0eaa11ed5dfa11c8d60a474ff0e095099f39d98e", size = 796896, upload-time = "2024-11-06T20:10:21.85Z" },
+    { url = "https://files.pythonhosted.org/packages/24/56/0b3f1b66d592be6efec23a795b37732682520b47c53da5a32c33ed7d84e3/regex-2024.11.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c32f75920cf99fe6b6c539c399a4a128452eaf1af27f39bce8909c9a3fd8cbe", size = 783997, upload-time = "2024-11-06T20:10:24.329Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/a1/eb378dada8b91c0e4c5f08ffb56f25fcae47bf52ad18f9b2f33b83e6d498/regex-2024.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:982e6d21414e78e1f51cf595d7f321dcd14de1f2881c5dc6a6e23bbbbd68435e", size = 781725, upload-time = "2024-11-06T20:10:28.067Z" },
+    { url = "https://files.pythonhosted.org/packages/83/f2/033e7dec0cfd6dda93390089864732a3409246ffe8b042e9554afa9bff4e/regex-2024.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a7c2155f790e2fb448faed6dd241386719802296ec588a8b9051c1f5c481bc29", size = 789481, upload-time = "2024-11-06T20:10:31.612Z" },
+    { url = "https://files.pythonhosted.org/packages/83/23/15d4552ea28990a74e7696780c438aadd73a20318c47e527b47a4a5a596d/regex-2024.11.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149f5008d286636e48cd0b1dd65018548944e495b0265b45e1bffecce1ef7f39", size = 852896, upload-time = "2024-11-06T20:10:34.054Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/39/ed4416bc90deedbfdada2568b2cb0bc1fdb98efe11f5378d9892b2a88f8f/regex-2024.11.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e5364a4502efca094731680e80009632ad6624084aff9a23ce8c8c6820de3e51", size = 860138, upload-time = "2024-11-06T20:10:36.142Z" },
+    { url = "https://files.pythonhosted.org/packages/93/2d/dd56bb76bd8e95bbce684326302f287455b56242a4f9c61f1bc76e28360e/regex-2024.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0a86e7eeca091c09e021db8eb72d54751e527fa47b8d5787caf96d9831bd02ad", size = 787692, upload-time = "2024-11-06T20:10:38.394Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/55/31877a249ab7a5156758246b9c59539abbeba22461b7d8adc9e8475ff73e/regex-2024.11.6-cp312-cp312-win32.whl", hash = "sha256:32f9a4c643baad4efa81d549c2aadefaeba12249b2adc5af541759237eee1c54", size = 262135, upload-time = "2024-11-06T20:10:40.367Z" },
+    { url = "https://files.pythonhosted.org/packages/38/ec/ad2d7de49a600cdb8dd78434a1aeffe28b9d6fc42eb36afab4a27ad23384/regex-2024.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:a93c194e2df18f7d264092dc8539b8ffb86b45b899ab976aa15d48214138e81b", size = 273567, upload-time = "2024-11-06T20:10:43.467Z" },
+    { url = "https://files.pythonhosted.org/packages/90/73/bcb0e36614601016552fa9344544a3a2ae1809dc1401b100eab02e772e1f/regex-2024.11.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a6ba92c0bcdf96cbf43a12c717eae4bc98325ca3730f6b130ffa2e3c3c723d84", size = 483525, upload-time = "2024-11-06T20:10:45.19Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/3f/f1a082a46b31e25291d830b369b6b0c5576a6f7fb89d3053a354c24b8a83/regex-2024.11.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:525eab0b789891ac3be914d36893bdf972d483fe66551f79d3e27146191a37d4", size = 288324, upload-time = "2024-11-06T20:10:47.177Z" },
+    { url = "https://files.pythonhosted.org/packages/09/c9/4e68181a4a652fb3ef5099e077faf4fd2a694ea6e0f806a7737aff9e758a/regex-2024.11.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:086a27a0b4ca227941700e0b31425e7a28ef1ae8e5e05a33826e17e47fbfdba0", size = 284617, upload-time = "2024-11-06T20:10:49.312Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/fd/37868b75eaf63843165f1d2122ca6cb94bfc0271e4428cf58c0616786dce/regex-2024.11.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bde01f35767c4a7899b7eb6e823b125a64de314a8ee9791367c9a34d56af18d0", size = 795023, upload-time = "2024-11-06T20:10:51.102Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/7c/d4cd9c528502a3dedb5c13c146e7a7a539a3853dc20209c8e75d9ba9d1b2/regex-2024.11.6-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b583904576650166b3d920d2bcce13971f6f9e9a396c673187f49811b2769dc7", size = 833072, upload-time = "2024-11-06T20:10:52.926Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/db/46f563a08f969159c5a0f0e722260568425363bea43bb7ae370becb66a67/regex-2024.11.6-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c4de13f06a0d54fa0d5ab1b7138bfa0d883220965a29616e3ea61b35d5f5fc7", size = 823130, upload-time = "2024-11-06T20:10:54.828Z" },
+    { url = "https://files.pythonhosted.org/packages/db/60/1eeca2074f5b87df394fccaa432ae3fc06c9c9bfa97c5051aed70e6e00c2/regex-2024.11.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3cde6e9f2580eb1665965ce9bf17ff4952f34f5b126beb509fee8f4e994f143c", size = 796857, upload-time = "2024-11-06T20:10:56.634Z" },
+    { url = "https://files.pythonhosted.org/packages/10/db/ac718a08fcee981554d2f7bb8402f1faa7e868c1345c16ab1ebec54b0d7b/regex-2024.11.6-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d7f453dca13f40a02b79636a339c5b62b670141e63efd511d3f8f73fba162b3", size = 784006, upload-time = "2024-11-06T20:10:59.369Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/41/7da3fe70216cea93144bf12da2b87367590bcf07db97604edeea55dac9ad/regex-2024.11.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59dfe1ed21aea057a65c6b586afd2a945de04fc7db3de0a6e3ed5397ad491b07", size = 781650, upload-time = "2024-11-06T20:11:02.042Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/d5/880921ee4eec393a4752e6ab9f0fe28009435417c3102fc413f3fe81c4e5/regex-2024.11.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b97c1e0bd37c5cd7902e65f410779d39eeda155800b65fc4d04cc432efa9bc6e", size = 789545, upload-time = "2024-11-06T20:11:03.933Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/96/53770115e507081122beca8899ab7f5ae28ae790bfcc82b5e38976df6a77/regex-2024.11.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f9d1e379028e0fc2ae3654bac3cbbef81bf3fd571272a42d56c24007979bafb6", size = 853045, upload-time = "2024-11-06T20:11:06.497Z" },
+    { url = "https://files.pythonhosted.org/packages/31/d3/1372add5251cc2d44b451bd94f43b2ec78e15a6e82bff6a290ef9fd8f00a/regex-2024.11.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:13291b39131e2d002a7940fb176e120bec5145f3aeb7621be6534e46251912c4", size = 860182, upload-time = "2024-11-06T20:11:09.06Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/e3/c446a64984ea9f69982ba1a69d4658d5014bc7a0ea468a07e1a1265db6e2/regex-2024.11.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f51f88c126370dcec4908576c5a627220da6c09d0bff31cfa89f2523843316d", size = 787733, upload-time = "2024-11-06T20:11:11.256Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/f1/e40c8373e3480e4f29f2692bd21b3e05f296d3afebc7e5dcf21b9756ca1c/regex-2024.11.6-cp313-cp313-win32.whl", hash = "sha256:63b13cfd72e9601125027202cad74995ab26921d8cd935c25f09c630436348ff", size = 262122, upload-time = "2024-11-06T20:11:13.161Z" },
+    { url = "https://files.pythonhosted.org/packages/45/94/bc295babb3062a731f52621cdc992d123111282e291abaf23faa413443ea/regex-2024.11.6-cp313-cp313-win_amd64.whl", hash = "sha256:2b3361af3198667e99927da8b84c1b010752fa4b1115ee30beaa332cabc3ef1a", size = 273545, upload-time = "2024-11-06T20:11:15Z" },
+]
+
+[[package]]
+name = "requests"
+version = "2.32.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "charset-normalizer" },
+    { name = "idna" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" },
+]
+
+[[package]]
+name = "rich"
+version = "14.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markdown-it-py" },
+    { name = "pygments" },
+    { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a1/53/830aa4c3066a8ab0ae9a9955976fb770fe9c6102117c8ec4ab3ea62d89e8/rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725", size = 224078, upload-time = "2025-03-30T14:15:14.23Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0d/9b/63f4c7ebc259242c89b3acafdb37b41d1185c07ff0011164674e9076b491/rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0", size = 243229, upload-time = "2025-03-30T14:15:12.283Z" },
+]
+
+[[package]]
+name = "rpds-py"
+version = "0.26.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a5/aa/4456d84bbb54adc6a916fb10c9b374f78ac840337644e4a5eda229c81275/rpds_py-0.26.0.tar.gz", hash = "sha256:20dae58a859b0906f0685642e591056f1e787f3a8b39c8e8749a45dc7d26bdb0", size = 27385, upload-time = "2025-07-01T15:57:13.958Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b9/31/1459645f036c3dfeacef89e8e5825e430c77dde8489f3b99eaafcd4a60f5/rpds_py-0.26.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4c70c70f9169692b36307a95f3d8c0a9fcd79f7b4a383aad5eaa0e9718b79b37", size = 372466, upload-time = "2025-07-01T15:53:40.55Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/ff/3d0727f35836cc8773d3eeb9a46c40cc405854e36a8d2e951f3a8391c976/rpds_py-0.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:777c62479d12395bfb932944e61e915741e364c843afc3196b694db3d669fcd0", size = 357825, upload-time = "2025-07-01T15:53:42.247Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/ce/badc5e06120a54099ae287fa96d82cbb650a5f85cf247ffe19c7b157fd1f/rpds_py-0.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec671691e72dff75817386aa02d81e708b5a7ec0dec6669ec05213ff6b77e1bd", size = 381530, upload-time = "2025-07-01T15:53:43.585Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/a5/fa5d96a66c95d06c62d7a30707b6a4cfec696ab8ae280ee7be14e961e118/rpds_py-0.26.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6a1cb5d6ce81379401bbb7f6dbe3d56de537fb8235979843f0d53bc2e9815a79", size = 396933, upload-time = "2025-07-01T15:53:45.78Z" },
+    { url = "https://files.pythonhosted.org/packages/00/a7/7049d66750f18605c591a9db47d4a059e112a0c9ff8de8daf8fa0f446bba/rpds_py-0.26.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f789e32fa1fb6a7bf890e0124e7b42d1e60d28ebff57fe806719abb75f0e9a3", size = 513973, upload-time = "2025-07-01T15:53:47.085Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/f1/528d02c7d6b29d29fac8fd784b354d3571cc2153f33f842599ef0cf20dd2/rpds_py-0.26.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c55b0a669976cf258afd718de3d9ad1b7d1fe0a91cd1ab36f38b03d4d4aeaaf", size = 402293, upload-time = "2025-07-01T15:53:48.117Z" },
+    { url = "https://files.pythonhosted.org/packages/15/93/fde36cd6e4685df2cd08508f6c45a841e82f5bb98c8d5ecf05649522acb5/rpds_py-0.26.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c70d9ec912802ecfd6cd390dadb34a9578b04f9bcb8e863d0a7598ba5e9e7ccc", size = 383787, upload-time = "2025-07-01T15:53:50.874Z" },
+    { url = "https://files.pythonhosted.org/packages/69/f2/5007553aaba1dcae5d663143683c3dfd03d9395289f495f0aebc93e90f24/rpds_py-0.26.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3021933c2cb7def39d927b9862292e0f4c75a13d7de70eb0ab06efed4c508c19", size = 416312, upload-time = "2025-07-01T15:53:52.046Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/a7/ce52c75c1e624a79e48a69e611f1c08844564e44c85db2b6f711d76d10ce/rpds_py-0.26.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8a7898b6ca3b7d6659e55cdac825a2e58c638cbf335cde41f4619e290dd0ad11", size = 558403, upload-time = "2025-07-01T15:53:53.192Z" },
+    { url = "https://files.pythonhosted.org/packages/79/d5/e119db99341cc75b538bf4cb80504129fa22ce216672fb2c28e4a101f4d9/rpds_py-0.26.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:12bff2ad9447188377f1b2794772f91fe68bb4bbfa5a39d7941fbebdbf8c500f", size = 588323, upload-time = "2025-07-01T15:53:54.336Z" },
+    { url = "https://files.pythonhosted.org/packages/93/94/d28272a0b02f5fe24c78c20e13bbcb95f03dc1451b68e7830ca040c60bd6/rpds_py-0.26.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:191aa858f7d4902e975d4cf2f2d9243816c91e9605070aeb09c0a800d187e323", size = 554541, upload-time = "2025-07-01T15:53:55.469Z" },
+    { url = "https://files.pythonhosted.org/packages/93/e0/8c41166602f1b791da892d976057eba30685486d2e2c061ce234679c922b/rpds_py-0.26.0-cp310-cp310-win32.whl", hash = "sha256:b37a04d9f52cb76b6b78f35109b513f6519efb481d8ca4c321f6a3b9580b3f45", size = 220442, upload-time = "2025-07-01T15:53:56.524Z" },
+    { url = "https://files.pythonhosted.org/packages/87/f0/509736bb752a7ab50fb0270c2a4134d671a7b3038030837e5536c3de0e0b/rpds_py-0.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:38721d4c9edd3eb6670437d8d5e2070063f305bfa2d5aa4278c51cedcd508a84", size = 231314, upload-time = "2025-07-01T15:53:57.842Z" },
+    { url = "https://files.pythonhosted.org/packages/09/4c/4ee8f7e512030ff79fda1df3243c88d70fc874634e2dbe5df13ba4210078/rpds_py-0.26.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9e8cb77286025bdb21be2941d64ac6ca016130bfdcd228739e8ab137eb4406ed", size = 372610, upload-time = "2025-07-01T15:53:58.844Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/9d/3dc16be00f14fc1f03c71b1d67c8df98263ab2710a2fbd65a6193214a527/rpds_py-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e09330b21d98adc8ccb2dbb9fc6cb434e8908d4c119aeaa772cb1caab5440a0", size = 358032, upload-time = "2025-07-01T15:53:59.985Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/5a/7f1bf8f045da2866324a08ae80af63e64e7bfaf83bd31f865a7b91a58601/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9c1b92b774b2e68d11193dc39620d62fd8ab33f0a3c77ecdabe19c179cdbc1", size = 381525, upload-time = "2025-07-01T15:54:01.162Z" },
+    { url = "https://files.pythonhosted.org/packages/45/8a/04479398c755a066ace10e3d158866beb600867cacae194c50ffa783abd0/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:824e6d3503ab990d7090768e4dfd9e840837bae057f212ff9f4f05ec6d1975e7", size = 397089, upload-time = "2025-07-01T15:54:02.319Z" },
+    { url = "https://files.pythonhosted.org/packages/72/88/9203f47268db488a1b6d469d69c12201ede776bb728b9d9f29dbfd7df406/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ad7fd2258228bf288f2331f0a6148ad0186b2e3643055ed0db30990e59817a6", size = 514255, upload-time = "2025-07-01T15:54:03.38Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/b4/01ce5d1e853ddf81fbbd4311ab1eff0b3cf162d559288d10fd127e2588b5/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0dc23bbb3e06ec1ea72d515fb572c1fea59695aefbffb106501138762e1e915e", size = 402283, upload-time = "2025-07-01T15:54:04.923Z" },
+    { url = "https://files.pythonhosted.org/packages/34/a2/004c99936997bfc644d590a9defd9e9c93f8286568f9c16cdaf3e14429a7/rpds_py-0.26.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d80bf832ac7b1920ee29a426cdca335f96a2b5caa839811803e999b41ba9030d", size = 383881, upload-time = "2025-07-01T15:54:06.482Z" },
+    { url = "https://files.pythonhosted.org/packages/05/1b/ef5fba4a8f81ce04c427bfd96223f92f05e6cd72291ce9d7523db3b03a6c/rpds_py-0.26.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0919f38f5542c0a87e7b4afcafab6fd2c15386632d249e9a087498571250abe3", size = 415822, upload-time = "2025-07-01T15:54:07.605Z" },
+    { url = "https://files.pythonhosted.org/packages/16/80/5c54195aec456b292f7bd8aa61741c8232964063fd8a75fdde9c1e982328/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d422b945683e409000c888e384546dbab9009bb92f7c0b456e217988cf316107", size = 558347, upload-time = "2025-07-01T15:54:08.591Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/1c/1845c1b1fd6d827187c43afe1841d91678d7241cbdb5420a4c6de180a538/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:77a7711fa562ba2da1aa757e11024ad6d93bad6ad7ede5afb9af144623e5f76a", size = 587956, upload-time = "2025-07-01T15:54:09.963Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/ff/9e979329dd131aa73a438c077252ddabd7df6d1a7ad7b9aacf6261f10faa/rpds_py-0.26.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:238e8c8610cb7c29460e37184f6799547f7e09e6a9bdbdab4e8edb90986a2318", size = 554363, upload-time = "2025-07-01T15:54:11.073Z" },
+    { url = "https://files.pythonhosted.org/packages/00/8b/d78cfe034b71ffbe72873a136e71acc7a831a03e37771cfe59f33f6de8a2/rpds_py-0.26.0-cp311-cp311-win32.whl", hash = "sha256:893b022bfbdf26d7bedb083efeea624e8550ca6eb98bf7fea30211ce95b9201a", size = 220123, upload-time = "2025-07-01T15:54:12.382Z" },
+    { url = "https://files.pythonhosted.org/packages/94/c1/3c8c94c7dd3905dbfde768381ce98778500a80db9924731d87ddcdb117e9/rpds_py-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:87a5531de9f71aceb8af041d72fc4cab4943648d91875ed56d2e629bef6d4c03", size = 231732, upload-time = "2025-07-01T15:54:13.434Z" },
+    { url = "https://files.pythonhosted.org/packages/67/93/e936fbed1b734eabf36ccb5d93c6a2e9246fbb13c1da011624b7286fae3e/rpds_py-0.26.0-cp311-cp311-win_arm64.whl", hash = "sha256:de2713f48c1ad57f89ac25b3cb7daed2156d8e822cf0eca9b96a6f990718cc41", size = 221917, upload-time = "2025-07-01T15:54:14.559Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/86/90eb87c6f87085868bd077c7a9938006eb1ce19ed4d06944a90d3560fce2/rpds_py-0.26.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:894514d47e012e794f1350f076c427d2347ebf82f9b958d554d12819849a369d", size = 363933, upload-time = "2025-07-01T15:54:15.734Z" },
+    { url = "https://files.pythonhosted.org/packages/63/78/4469f24d34636242c924626082b9586f064ada0b5dbb1e9d096ee7a8e0c6/rpds_py-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc921b96fa95a097add244da36a1d9e4f3039160d1d30f1b35837bf108c21136", size = 350447, upload-time = "2025-07-01T15:54:16.922Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/91/c448ed45efdfdade82348d5e7995e15612754826ea640afc20915119734f/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e1157659470aa42a75448b6e943c895be8c70531c43cb78b9ba990778955582", size = 384711, upload-time = "2025-07-01T15:54:18.101Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/43/e5c86fef4be7f49828bdd4ecc8931f0287b1152c0bb0163049b3218740e7/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:521ccf56f45bb3a791182dc6b88ae5f8fa079dd705ee42138c76deb1238e554e", size = 400865, upload-time = "2025-07-01T15:54:19.295Z" },
+    { url = "https://files.pythonhosted.org/packages/55/34/e00f726a4d44f22d5c5fe2e5ddd3ac3d7fd3f74a175607781fbdd06fe375/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9def736773fd56b305c0eef698be5192c77bfa30d55a0e5885f80126c4831a15", size = 517763, upload-time = "2025-07-01T15:54:20.858Z" },
+    { url = "https://files.pythonhosted.org/packages/52/1c/52dc20c31b147af724b16104500fba13e60123ea0334beba7b40e33354b4/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdad4ea3b4513b475e027be79e5a0ceac8ee1c113a1a11e5edc3c30c29f964d8", size = 406651, upload-time = "2025-07-01T15:54:22.508Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/77/87d7bfabfc4e821caa35481a2ff6ae0b73e6a391bb6b343db2c91c2b9844/rpds_py-0.26.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82b165b07f416bdccf5c84546a484cc8f15137ca38325403864bfdf2b5b72f6a", size = 386079, upload-time = "2025-07-01T15:54:23.987Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/d4/7f2200c2d3ee145b65b3cddc4310d51f7da6a26634f3ac87125fd789152a/rpds_py-0.26.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d04cab0a54b9dba4d278fe955a1390da3cf71f57feb78ddc7cb67cbe0bd30323", size = 421379, upload-time = "2025-07-01T15:54:25.073Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/13/9fdd428b9c820869924ab62236b8688b122baa22d23efdd1c566938a39ba/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:79061ba1a11b6a12743a2b0f72a46aa2758613d454aa6ba4f5a265cc48850158", size = 562033, upload-time = "2025-07-01T15:54:26.225Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/e1/b69686c3bcbe775abac3a4c1c30a164a2076d28df7926041f6c0eb5e8d28/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f405c93675d8d4c5ac87364bb38d06c988e11028a64b52a47158a355079661f3", size = 591639, upload-time = "2025-07-01T15:54:27.424Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/c9/1e3d8c8863c84a90197ac577bbc3d796a92502124c27092413426f670990/rpds_py-0.26.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dafd4c44b74aa4bed4b250f1aed165b8ef5de743bcca3b88fc9619b6087093d2", size = 557105, upload-time = "2025-07-01T15:54:29.93Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/c5/90c569649057622959f6dcc40f7b516539608a414dfd54b8d77e3b201ac0/rpds_py-0.26.0-cp312-cp312-win32.whl", hash = "sha256:3da5852aad63fa0c6f836f3359647870e21ea96cf433eb393ffa45263a170d44", size = 223272, upload-time = "2025-07-01T15:54:31.128Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/16/19f5d9f2a556cfed454eebe4d354c38d51c20f3db69e7b4ce6cff904905d/rpds_py-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf47cfdabc2194a669dcf7a8dbba62e37a04c5041d2125fae0233b720da6f05c", size = 234995, upload-time = "2025-07-01T15:54:32.195Z" },
+    { url = "https://files.pythonhosted.org/packages/83/f0/7935e40b529c0e752dfaa7880224771b51175fce08b41ab4a92eb2fbdc7f/rpds_py-0.26.0-cp312-cp312-win_arm64.whl", hash = "sha256:20ab1ae4fa534f73647aad289003f1104092890849e0266271351922ed5574f8", size = 223198, upload-time = "2025-07-01T15:54:33.271Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/67/bb62d0109493b12b1c6ab00de7a5566aa84c0e44217c2d94bee1bd370da9/rpds_py-0.26.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:696764a5be111b036256c0b18cd29783fab22154690fc698062fc1b0084b511d", size = 363917, upload-time = "2025-07-01T15:54:34.755Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/f3/34e6ae1925a5706c0f002a8d2d7f172373b855768149796af87bd65dcdb9/rpds_py-0.26.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1e6c15d2080a63aaed876e228efe4f814bc7889c63b1e112ad46fdc8b368b9e1", size = 350073, upload-time = "2025-07-01T15:54:36.292Z" },
+    { url = "https://files.pythonhosted.org/packages/75/83/1953a9d4f4e4de7fd0533733e041c28135f3c21485faaef56a8aadbd96b5/rpds_py-0.26.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:390e3170babf42462739a93321e657444f0862c6d722a291accc46f9d21ed04e", size = 384214, upload-time = "2025-07-01T15:54:37.469Z" },
+    { url = "https://files.pythonhosted.org/packages/48/0e/983ed1b792b3322ea1d065e67f4b230f3b96025f5ce3878cc40af09b7533/rpds_py-0.26.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7da84c2c74c0f5bc97d853d9e17bb83e2dcafcff0dc48286916001cc114379a1", size = 400113, upload-time = "2025-07-01T15:54:38.954Z" },
+    { url = "https://files.pythonhosted.org/packages/69/7f/36c0925fff6f660a80be259c5b4f5e53a16851f946eb080351d057698528/rpds_py-0.26.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c5fe114a6dd480a510b6d3661d09d67d1622c4bf20660a474507aaee7eeeee9", size = 515189, upload-time = "2025-07-01T15:54:40.57Z" },
+    { url = "https://files.pythonhosted.org/packages/13/45/cbf07fc03ba7a9b54662c9badb58294ecfb24f828b9732970bd1a431ed5c/rpds_py-0.26.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3100b3090269f3a7ea727b06a6080d4eb7439dca4c0e91a07c5d133bb1727ea7", size = 406998, upload-time = "2025-07-01T15:54:43.025Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/b0/8fa5e36e58657997873fd6a1cf621285ca822ca75b4b3434ead047daa307/rpds_py-0.26.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c03c9b0c64afd0320ae57de4c982801271c0c211aa2d37f3003ff5feb75bb04", size = 385903, upload-time = "2025-07-01T15:54:44.752Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/f7/b25437772f9f57d7a9fbd73ed86d0dcd76b4c7c6998348c070d90f23e315/rpds_py-0.26.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5963b72ccd199ade6ee493723d18a3f21ba7d5b957017607f815788cef50eaf1", size = 419785, upload-time = "2025-07-01T15:54:46.043Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/6b/63ffa55743dfcb4baf2e9e77a0b11f7f97ed96a54558fcb5717a4b2cd732/rpds_py-0.26.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9da4e873860ad5bab3291438525cae80169daecbfafe5657f7f5fb4d6b3f96b9", size = 561329, upload-time = "2025-07-01T15:54:47.64Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/07/1f4f5e2886c480a2346b1e6759c00278b8a69e697ae952d82ae2e6ee5db0/rpds_py-0.26.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:5afaddaa8e8c7f1f7b4c5c725c0070b6eed0228f705b90a1732a48e84350f4e9", size = 590875, upload-time = "2025-07-01T15:54:48.9Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/bc/e6639f1b91c3a55f8c41b47d73e6307051b6e246254a827ede730624c0f8/rpds_py-0.26.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4916dc96489616a6f9667e7526af8fa693c0fdb4f3acb0e5d9f4400eb06a47ba", size = 556636, upload-time = "2025-07-01T15:54:50.619Z" },
+    { url = "https://files.pythonhosted.org/packages/05/4c/b3917c45566f9f9a209d38d9b54a1833f2bb1032a3e04c66f75726f28876/rpds_py-0.26.0-cp313-cp313-win32.whl", hash = "sha256:2a343f91b17097c546b93f7999976fd6c9d5900617aa848c81d794e062ab302b", size = 222663, upload-time = "2025-07-01T15:54:52.023Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/0b/0851bdd6025775aaa2365bb8de0697ee2558184c800bfef8d7aef5ccde58/rpds_py-0.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:0a0b60701f2300c81b2ac88a5fb893ccfa408e1c4a555a77f908a2596eb875a5", size = 234428, upload-time = "2025-07-01T15:54:53.692Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/e8/a47c64ed53149c75fb581e14a237b7b7cd18217e969c30d474d335105622/rpds_py-0.26.0-cp313-cp313-win_arm64.whl", hash = "sha256:257d011919f133a4746958257f2c75238e3ff54255acd5e3e11f3ff41fd14256", size = 222571, upload-time = "2025-07-01T15:54:54.822Z" },
+    { url = "https://files.pythonhosted.org/packages/89/bf/3d970ba2e2bcd17d2912cb42874107390f72873e38e79267224110de5e61/rpds_py-0.26.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:529c8156d7506fba5740e05da8795688f87119cce330c244519cf706a4a3d618", size = 360475, upload-time = "2025-07-01T15:54:56.228Z" },
+    { url = "https://files.pythonhosted.org/packages/82/9f/283e7e2979fc4ec2d8ecee506d5a3675fce5ed9b4b7cb387ea5d37c2f18d/rpds_py-0.26.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f53ec51f9d24e9638a40cabb95078ade8c99251945dad8d57bf4aabe86ecee35", size = 346692, upload-time = "2025-07-01T15:54:58.561Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/03/7e50423c04d78daf391da3cc4330bdb97042fc192a58b186f2d5deb7befd/rpds_py-0.26.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab504c4d654e4a29558eaa5bb8cea5fdc1703ea60a8099ffd9c758472cf913f", size = 379415, upload-time = "2025-07-01T15:54:59.751Z" },
+    { url = "https://files.pythonhosted.org/packages/57/00/d11ee60d4d3b16808432417951c63df803afb0e0fc672b5e8d07e9edaaae/rpds_py-0.26.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd0641abca296bc1a00183fe44f7fced8807ed49d501f188faa642d0e4975b83", size = 391783, upload-time = "2025-07-01T15:55:00.898Z" },
+    { url = "https://files.pythonhosted.org/packages/08/b3/1069c394d9c0d6d23c5b522e1f6546b65793a22950f6e0210adcc6f97c3e/rpds_py-0.26.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b312fecc1d017b5327afa81d4da1480f51c68810963a7336d92203dbb3d4f1", size = 512844, upload-time = "2025-07-01T15:55:02.201Z" },
+    { url = "https://files.pythonhosted.org/packages/08/3b/c4fbf0926800ed70b2c245ceca99c49f066456755f5d6eb8863c2c51e6d0/rpds_py-0.26.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c741107203954f6fc34d3066d213d0a0c40f7bb5aafd698fb39888af277c70d8", size = 402105, upload-time = "2025-07-01T15:55:03.698Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/b0/db69b52ca07413e568dae9dc674627a22297abb144c4d6022c6d78f1e5cc/rpds_py-0.26.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc3e55a7db08dc9a6ed5fb7103019d2c1a38a349ac41901f9f66d7f95750942f", size = 383440, upload-time = "2025-07-01T15:55:05.398Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/e1/c65255ad5b63903e56b3bb3ff9dcc3f4f5c3badde5d08c741ee03903e951/rpds_py-0.26.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e851920caab2dbcae311fd28f4313c6953993893eb5c1bb367ec69d9a39e7ed", size = 412759, upload-time = "2025-07-01T15:55:08.316Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/22/bb731077872377a93c6e93b8a9487d0406c70208985831034ccdeed39c8e/rpds_py-0.26.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dfbf280da5f876d0b00c81f26bedce274e72a678c28845453885a9b3c22ae632", size = 556032, upload-time = "2025-07-01T15:55:09.52Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/8b/393322ce7bac5c4530fb96fc79cc9ea2f83e968ff5f6e873f905c493e1c4/rpds_py-0.26.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:1cc81d14ddfa53d7f3906694d35d54d9d3f850ef8e4e99ee68bc0d1e5fed9a9c", size = 585416, upload-time = "2025-07-01T15:55:11.216Z" },
+    { url = "https://files.pythonhosted.org/packages/49/ae/769dc372211835bf759319a7aae70525c6eb523e3371842c65b7ef41c9c6/rpds_py-0.26.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dca83c498b4650a91efcf7b88d669b170256bf8017a5db6f3e06c2bf031f57e0", size = 554049, upload-time = "2025-07-01T15:55:13.004Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/f9/4c43f9cc203d6ba44ce3146246cdc38619d92c7bd7bad4946a3491bd5b70/rpds_py-0.26.0-cp313-cp313t-win32.whl", hash = "sha256:4d11382bcaf12f80b51d790dee295c56a159633a8e81e6323b16e55d81ae37e9", size = 218428, upload-time = "2025-07-01T15:55:14.486Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/8b/9286b7e822036a4a977f2f1e851c7345c20528dbd56b687bb67ed68a8ede/rpds_py-0.26.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ff110acded3c22c033e637dd8896e411c7d3a11289b2edf041f86663dbc791e9", size = 231524, upload-time = "2025-07-01T15:55:15.745Z" },
+    { url = "https://files.pythonhosted.org/packages/55/07/029b7c45db910c74e182de626dfdae0ad489a949d84a468465cd0ca36355/rpds_py-0.26.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:da619979df60a940cd434084355c514c25cf8eb4cf9a508510682f6c851a4f7a", size = 364292, upload-time = "2025-07-01T15:55:17.001Z" },
+    { url = "https://files.pythonhosted.org/packages/13/d1/9b3d3f986216b4d1f584878dca15ce4797aaf5d372d738974ba737bf68d6/rpds_py-0.26.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ea89a2458a1a75f87caabefe789c87539ea4e43b40f18cff526052e35bbb4fdf", size = 350334, upload-time = "2025-07-01T15:55:18.922Z" },
+    { url = "https://files.pythonhosted.org/packages/18/98/16d5e7bc9ec715fa9668731d0cf97f6b032724e61696e2db3d47aeb89214/rpds_py-0.26.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feac1045b3327a45944e7dcbeb57530339f6b17baff154df51ef8b0da34c8c12", size = 384875, upload-time = "2025-07-01T15:55:20.399Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/13/aa5e2b1ec5ab0e86a5c464d53514c0467bec6ba2507027d35fc81818358e/rpds_py-0.26.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b818a592bd69bfe437ee8368603d4a2d928c34cffcdf77c2e761a759ffd17d20", size = 399993, upload-time = "2025-07-01T15:55:21.729Z" },
+    { url = "https://files.pythonhosted.org/packages/17/03/8021810b0e97923abdbab6474c8b77c69bcb4b2c58330777df9ff69dc559/rpds_py-0.26.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a8b0dd8648709b62d9372fc00a57466f5fdeefed666afe3fea5a6c9539a0331", size = 516683, upload-time = "2025-07-01T15:55:22.918Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/b1/da8e61c87c2f3d836954239fdbbfb477bb7b54d74974d8f6fcb34342d166/rpds_py-0.26.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6d3498ad0df07d81112aa6ec6c95a7e7b1ae00929fb73e7ebee0f3faaeabad2f", size = 408825, upload-time = "2025-07-01T15:55:24.207Z" },
+    { url = "https://files.pythonhosted.org/packages/38/bc/1fc173edaaa0e52c94b02a655db20697cb5fa954ad5a8e15a2c784c5cbdd/rpds_py-0.26.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24a4146ccb15be237fdef10f331c568e1b0e505f8c8c9ed5d67759dac58ac246", size = 387292, upload-time = "2025-07-01T15:55:25.554Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/eb/3a9bb4bd90867d21916f253caf4f0d0be7098671b6715ad1cead9fe7bab9/rpds_py-0.26.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a9a63785467b2d73635957d32a4f6e73d5e4df497a16a6392fa066b753e87387", size = 420435, upload-time = "2025-07-01T15:55:27.798Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/16/e066dcdb56f5632713445271a3f8d3d0b426d51ae9c0cca387799df58b02/rpds_py-0.26.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:de4ed93a8c91debfd5a047be327b7cc8b0cc6afe32a716bbbc4aedca9e2a83af", size = 562410, upload-time = "2025-07-01T15:55:29.057Z" },
+    { url = "https://files.pythonhosted.org/packages/60/22/ddbdec7eb82a0dc2e455be44c97c71c232983e21349836ce9f272e8a3c29/rpds_py-0.26.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:caf51943715b12af827696ec395bfa68f090a4c1a1d2509eb4e2cb69abbbdb33", size = 590724, upload-time = "2025-07-01T15:55:30.719Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/b4/95744085e65b7187d83f2fcb0bef70716a1ea0a9e5d8f7f39a86e5d83424/rpds_py-0.26.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4a59e5bc386de021f56337f757301b337d7ab58baa40174fb150accd480bc953", size = 558285, upload-time = "2025-07-01T15:55:31.981Z" },
+    { url = "https://files.pythonhosted.org/packages/37/37/6309a75e464d1da2559446f9c811aa4d16343cebe3dbb73701e63f760caa/rpds_py-0.26.0-cp314-cp314-win32.whl", hash = "sha256:92c8db839367ef16a662478f0a2fe13e15f2227da3c1430a782ad0f6ee009ec9", size = 223459, upload-time = "2025-07-01T15:55:33.312Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/6f/8e9c11214c46098b1d1391b7e02b70bb689ab963db3b19540cba17315291/rpds_py-0.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:b0afb8cdd034150d4d9f53926226ed27ad15b7f465e93d7468caaf5eafae0d37", size = 236083, upload-time = "2025-07-01T15:55:34.933Z" },
+    { url = "https://files.pythonhosted.org/packages/47/af/9c4638994dd623d51c39892edd9d08e8be8220a4b7e874fa02c2d6e91955/rpds_py-0.26.0-cp314-cp314-win_arm64.whl", hash = "sha256:ca3f059f4ba485d90c8dc75cb5ca897e15325e4e609812ce57f896607c1c0867", size = 223291, upload-time = "2025-07-01T15:55:36.202Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/db/669a241144460474aab03e254326b32c42def83eb23458a10d163cb9b5ce/rpds_py-0.26.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:5afea17ab3a126006dc2f293b14ffc7ef3c85336cf451564a0515ed7648033da", size = 361445, upload-time = "2025-07-01T15:55:37.483Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/2d/133f61cc5807c6c2fd086a46df0eb8f63a23f5df8306ff9f6d0fd168fecc/rpds_py-0.26.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:69f0c0a3df7fd3a7eec50a00396104bb9a843ea6d45fcc31c2d5243446ffd7a7", size = 347206, upload-time = "2025-07-01T15:55:38.828Z" },
+    { url = "https://files.pythonhosted.org/packages/05/bf/0e8fb4c05f70273469eecf82f6ccf37248558526a45321644826555db31b/rpds_py-0.26.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:801a71f70f9813e82d2513c9a96532551fce1e278ec0c64610992c49c04c2dad", size = 380330, upload-time = "2025-07-01T15:55:40.175Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/a8/060d24185d8b24d3923322f8d0ede16df4ade226a74e747b8c7c978e3dd3/rpds_py-0.26.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:df52098cde6d5e02fa75c1f6244f07971773adb4a26625edd5c18fee906fa84d", size = 392254, upload-time = "2025-07-01T15:55:42.015Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/7b/7c2e8a9ee3e6bc0bae26bf29f5219955ca2fbb761dca996a83f5d2f773fe/rpds_py-0.26.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bc596b30f86dc6f0929499c9e574601679d0341a0108c25b9b358a042f51bca", size = 516094, upload-time = "2025-07-01T15:55:43.603Z" },
+    { url = "https://files.pythonhosted.org/packages/75/d6/f61cafbed8ba1499b9af9f1777a2a199cd888f74a96133d8833ce5eaa9c5/rpds_py-0.26.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9dfbe56b299cf5875b68eb6f0ebaadc9cac520a1989cac0db0765abfb3709c19", size = 402889, upload-time = "2025-07-01T15:55:45.275Z" },
+    { url = "https://files.pythonhosted.org/packages/92/19/c8ac0a8a8df2dd30cdec27f69298a5c13e9029500d6d76718130f5e5be10/rpds_py-0.26.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac64f4b2bdb4ea622175c9ab7cf09444e412e22c0e02e906978b3b488af5fde8", size = 384301, upload-time = "2025-07-01T15:55:47.098Z" },
+    { url = "https://files.pythonhosted.org/packages/41/e1/6b1859898bc292a9ce5776016c7312b672da00e25cec74d7beced1027286/rpds_py-0.26.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:181ef9b6bbf9845a264f9aa45c31836e9f3c1f13be565d0d010e964c661d1e2b", size = 412891, upload-time = "2025-07-01T15:55:48.412Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/b9/ceb39af29913c07966a61367b3c08b4f71fad841e32c6b59a129d5974698/rpds_py-0.26.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:49028aa684c144ea502a8e847d23aed5e4c2ef7cadfa7d5eaafcb40864844b7a", size = 557044, upload-time = "2025-07-01T15:55:49.816Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/27/35637b98380731a521f8ec4f3fd94e477964f04f6b2f8f7af8a2d889a4af/rpds_py-0.26.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e5d524d68a474a9688336045bbf76cb0def88549c1b2ad9dbfec1fb7cfbe9170", size = 585774, upload-time = "2025-07-01T15:55:51.192Z" },
+    { url = "https://files.pythonhosted.org/packages/52/d9/3f0f105420fecd18551b678c9a6ce60bd23986098b252a56d35781b3e7e9/rpds_py-0.26.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1851f429b822831bd2edcbe0cfd12ee9ea77868f8d3daf267b189371671c80e", size = 554886, upload-time = "2025-07-01T15:55:52.541Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/c5/347c056a90dc8dd9bc240a08c527315008e1b5042e7a4cf4ac027be9d38a/rpds_py-0.26.0-cp314-cp314t-win32.whl", hash = "sha256:7bdb17009696214c3b66bb3590c6d62e14ac5935e53e929bcdbc5a495987a84f", size = 219027, upload-time = "2025-07-01T15:55:53.874Z" },
+    { url = "https://files.pythonhosted.org/packages/75/04/5302cea1aa26d886d34cadbf2dc77d90d7737e576c0065f357b96dc7a1a6/rpds_py-0.26.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f14440b9573a6f76b4ee4770c13f0b5921f71dde3b6fcb8dabbefd13b7fe05d7", size = 232821, upload-time = "2025-07-01T15:55:55.167Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/9a/1f033b0b31253d03d785b0cd905bc127e555ab496ea6b4c7c2e1f951f2fd/rpds_py-0.26.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3c0909c5234543ada2515c05dc08595b08d621ba919629e94427e8e03539c958", size = 373226, upload-time = "2025-07-01T15:56:16.578Z" },
+    { url = "https://files.pythonhosted.org/packages/58/29/5f88023fd6aaaa8ca3c4a6357ebb23f6f07da6079093ccf27c99efce87db/rpds_py-0.26.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c1fb0cda2abcc0ac62f64e2ea4b4e64c57dfd6b885e693095460c61bde7bb18e", size = 359230, upload-time = "2025-07-01T15:56:17.978Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/6c/13eaebd28b439da6964dde22712b52e53fe2824af0223b8e403249d10405/rpds_py-0.26.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84d142d2d6cf9b31c12aa4878d82ed3b2324226270b89b676ac62ccd7df52d08", size = 382363, upload-time = "2025-07-01T15:56:19.977Z" },
+    { url = "https://files.pythonhosted.org/packages/55/fc/3bb9c486b06da19448646f96147796de23c5811ef77cbfc26f17307b6a9d/rpds_py-0.26.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a547e21c5610b7e9093d870be50682a6a6cf180d6da0f42c47c306073bfdbbf6", size = 397146, upload-time = "2025-07-01T15:56:21.39Z" },
+    { url = "https://files.pythonhosted.org/packages/15/18/9d1b79eb4d18e64ba8bba9e7dec6f9d6920b639f22f07ee9368ca35d4673/rpds_py-0.26.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:35e9a70a0f335371275cdcd08bc5b8051ac494dd58bff3bbfb421038220dc871", size = 514804, upload-time = "2025-07-01T15:56:22.78Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/5a/175ad7191bdbcd28785204621b225ad70e85cdfd1e09cc414cb554633b21/rpds_py-0.26.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0dfa6115c6def37905344d56fb54c03afc49104e2ca473d5dedec0f6606913b4", size = 402820, upload-time = "2025-07-01T15:56:24.584Z" },
+    { url = "https://files.pythonhosted.org/packages/11/45/6a67ecf6d61c4d4aff4bc056e864eec4b2447787e11d1c2c9a0242c6e92a/rpds_py-0.26.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:313cfcd6af1a55a286a3c9a25f64af6d0e46cf60bc5798f1db152d97a216ff6f", size = 384567, upload-time = "2025-07-01T15:56:26.064Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/ba/16589da828732b46454c61858950a78fe4c931ea4bf95f17432ffe64b241/rpds_py-0.26.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f7bf2496fa563c046d05e4d232d7b7fd61346e2402052064b773e5c378bf6f73", size = 416520, upload-time = "2025-07-01T15:56:27.608Z" },
+    { url = "https://files.pythonhosted.org/packages/81/4b/00092999fc7c0c266045e984d56b7314734cc400a6c6dc4d61a35f135a9d/rpds_py-0.26.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:aa81873e2c8c5aa616ab8e017a481a96742fdf9313c40f14338ca7dbf50cb55f", size = 559362, upload-time = "2025-07-01T15:56:29.078Z" },
+    { url = "https://files.pythonhosted.org/packages/96/0c/43737053cde1f93ac4945157f7be1428724ab943e2132a0d235a7e161d4e/rpds_py-0.26.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:68ffcf982715f5b5b7686bdd349ff75d422e8f22551000c24b30eaa1b7f7ae84", size = 588113, upload-time = "2025-07-01T15:56:30.485Z" },
+    { url = "https://files.pythonhosted.org/packages/46/46/8e38f6161466e60a997ed7e9951ae5de131dedc3cf778ad35994b4af823d/rpds_py-0.26.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6188de70e190847bb6db3dc3981cbadff87d27d6fe9b4f0e18726d55795cee9b", size = 555429, upload-time = "2025-07-01T15:56:31.956Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/ac/65da605e9f1dd643ebe615d5bbd11b6efa1d69644fc4bf623ea5ae385a82/rpds_py-0.26.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:1c962145c7473723df9722ba4c058de12eb5ebedcb4e27e7d902920aa3831ee8", size = 231950, upload-time = "2025-07-01T15:56:33.337Z" },
+    { url = "https://files.pythonhosted.org/packages/51/f2/b5c85b758a00c513bb0389f8fc8e61eb5423050c91c958cdd21843faa3e6/rpds_py-0.26.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f61a9326f80ca59214d1cceb0a09bb2ece5b2563d4e0cd37bfd5515c28510674", size = 373505, upload-time = "2025-07-01T15:56:34.716Z" },
+    { url = "https://files.pythonhosted.org/packages/23/e0/25db45e391251118e915e541995bb5f5ac5691a3b98fb233020ba53afc9b/rpds_py-0.26.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:183f857a53bcf4b1b42ef0f57ca553ab56bdd170e49d8091e96c51c3d69ca696", size = 359468, upload-time = "2025-07-01T15:56:36.219Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/73/dd5ee6075bb6491be3a646b301dfd814f9486d924137a5098e61f0487e16/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:941c1cfdf4799d623cf3aa1d326a6b4fdb7a5799ee2687f3516738216d2262fb", size = 382680, upload-time = "2025-07-01T15:56:37.644Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/10/84b522ff58763a5c443f5bcedc1820240e454ce4e620e88520f04589e2ea/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72a8d9564a717ee291f554eeb4bfeafe2309d5ec0aa6c475170bdab0f9ee8e88", size = 397035, upload-time = "2025-07-01T15:56:39.241Z" },
+    { url = "https://files.pythonhosted.org/packages/06/ea/8667604229a10a520fcbf78b30ccc278977dcc0627beb7ea2c96b3becef0/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:511d15193cbe013619dd05414c35a7dedf2088fcee93c6bbb7c77859765bd4e8", size = 514922, upload-time = "2025-07-01T15:56:40.645Z" },
+    { url = "https://files.pythonhosted.org/packages/24/e6/9ed5b625c0661c4882fc8cdf302bf8e96c73c40de99c31e0b95ed37d508c/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aea1f9741b603a8d8fedb0ed5502c2bc0accbc51f43e2ad1337fe7259c2b77a5", size = 402822, upload-time = "2025-07-01T15:56:42.137Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/58/212c7b6fd51946047fb45d3733da27e2fa8f7384a13457c874186af691b1/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4019a9d473c708cf2f16415688ef0b4639e07abaa569d72f74745bbeffafa2c7", size = 384336, upload-time = "2025-07-01T15:56:44.239Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/f5/a40ba78748ae8ebf4934d4b88e77b98497378bc2c24ba55ebe87a4e87057/rpds_py-0.26.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:093d63b4b0f52d98ebae33b8c50900d3d67e0666094b1be7a12fffd7f65de74b", size = 416871, upload-time = "2025-07-01T15:56:46.284Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/a6/33b1fc0c9f7dcfcfc4a4353daa6308b3ece22496ceece348b3e7a7559a09/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2abe21d8ba64cded53a2a677e149ceb76dcf44284202d737178afe7ba540c1eb", size = 559439, upload-time = "2025-07-01T15:56:48.549Z" },
+    { url = "https://files.pythonhosted.org/packages/71/2d/ceb3f9c12f8cfa56d34995097f6cd99da1325642c60d1b6680dd9df03ed8/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:4feb7511c29f8442cbbc28149a92093d32e815a28aa2c50d333826ad2a20fdf0", size = 588380, upload-time = "2025-07-01T15:56:50.086Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/ed/9de62c2150ca8e2e5858acf3f4f4d0d180a38feef9fdab4078bea63d8dba/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:e99685fc95d386da368013e7fb4269dd39c30d99f812a8372d62f244f662709c", size = 555334, upload-time = "2025-07-01T15:56:51.703Z" },
+]
+
+[[package]]
+name = "rtree"
+version = "1.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/18/b8/0091f020acafcb034daa5b062f0626f6a73c7e0d64826af23861390a9585/rtree-1.4.0.tar.gz", hash = "sha256:9d97c7c5dcf25f6c0599c76d9933368c6a8d7238f2c1d00e76f1a69369ca82a0", size = 50789, upload-time = "2025-03-05T23:31:45.962Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f6/4c/8d54d6dc5ff8ba8ced1fad9378f89f9dd60addcc4cf0e525ee0e67b1769f/rtree-1.4.0-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:4d1bebc418101480aabf41767e772dd2155d3b27b1376cccbd93e4509485e091", size = 482755, upload-time = "2025-03-05T23:31:29.884Z" },
+    { url = "https://files.pythonhosted.org/packages/20/29/045e700d2135e9a67896086c831fde80fd4105971b443d5727a4093fcbf1/rtree-1.4.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:997f8c38d5dffa3949ea8adb4c8b291ea5cd4ef5ee69455d642dd171baf9991d", size = 439796, upload-time = "2025-03-05T23:31:31.517Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/fc/c3bd8cd67b10a12a6b9e2d06796779128c3e6968922dbf29fcd53af68d81/rtree-1.4.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0133d9c54ab3ffe874ba6d411dbe0254765c5e68d92da5b91362c370f16fd997", size = 497549, upload-time = "2025-03-05T23:31:33.722Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/dd/49dc9ab037d0cb288ed40f8b7f498f69d44243e4745e241c05d5e457ea8b/rtree-1.4.0-py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:d3b7bf1fe6463139377995ebe22a01a7005d134707f43672a3c09305e12f5f43", size = 568787, upload-time = "2025-03-05T23:31:35.478Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/e7/57737dff73ce789bdadd916d48ac12e977d8578176e1e890b1b8d89b9dbf/rtree-1.4.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:27e4a6d617d63dcb82fcd4c2856134b8a3741bd1af3b1a0d98e886054f394da5", size = 541090, upload-time = "2025-03-05T23:31:37.712Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/8f/1f3f716c4e8388670cfd5d0a3578e2354a1e6a3403648e234e1540e3e3bd/rtree-1.4.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5258e826064eab82439760201e9421ce6d4340789d6d080c1b49367ddd03f61f", size = 1454194, upload-time = "2025-03-05T23:31:39.851Z" },
+    { url = "https://files.pythonhosted.org/packages/22/ec/b42052b10e63a1c5d5d61ce234332f689736053644ba1756f7a632ea7659/rtree-1.4.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:20d5b3f9cf8bbbcc9fec42ab837c603c5dd86103ef29134300c8da2495c1248b", size = 1692814, upload-time = "2025-03-05T23:31:41.617Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/5b/a9920e9a2dc43b066ff13b7fde2e7bffcca315cfa43ae6f4cc15970e39eb/rtree-1.4.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a67bee1233370a4c72c0969a96d2a1df1ba404ddd9f146849c53ab420eab361b", size = 1554860, upload-time = "2025-03-05T23:31:43.091Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/c2/362f2cc36a7a57b47380061c23fc109c7222c1a544ffd24cda289ba19673/rtree-1.4.0-py3-none-win_amd64.whl", hash = "sha256:ba83efc7b7563905b1bfdfc14490c4bfb59e92e5e6156bdeb6ec5df5117252f4", size = 385221, upload-time = "2025-03-05T23:31:44.537Z" },
+]
+
+[[package]]
+name = "safetensors"
+version = "0.5.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/71/7e/2d5d6ee7b40c0682315367ec7475693d110f512922d582fef1bd4a63adc3/safetensors-0.5.3.tar.gz", hash = "sha256:b6b0d6ecacec39a4fdd99cc19f4576f5219ce858e6fd8dbe7609df0b8dc56965", size = 67210, upload-time = "2025-02-26T09:15:13.155Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/18/ae/88f6c49dbd0cc4da0e08610019a3c78a7d390879a919411a410a1876d03a/safetensors-0.5.3-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd20eb133db8ed15b40110b7c00c6df51655a2998132193de2f75f72d99c7073", size = 436917, upload-time = "2025-02-26T09:15:03.702Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/3b/11f1b4a2f5d2ab7da34ecc062b0bc301f2be024d110a6466726bec8c055c/safetensors-0.5.3-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:21d01c14ff6c415c485616b8b0bf961c46b3b343ca59110d38d744e577f9cce7", size = 418419, upload-time = "2025-02-26T09:15:01.765Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/9a/add3e6fef267658075c5a41573c26d42d80c935cdc992384dfae435feaef/safetensors-0.5.3-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11bce6164887cd491ca75c2326a113ba934be596e22b28b1742ce27b1d076467", size = 459493, upload-time = "2025-02-26T09:14:51.812Z" },
+    { url = "https://files.pythonhosted.org/packages/df/5c/bf2cae92222513cc23b3ff85c4a1bb2811a2c3583ac0f8e8d502751de934/safetensors-0.5.3-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4a243be3590bc3301c821da7a18d87224ef35cbd3e5f5727e4e0728b8172411e", size = 472400, upload-time = "2025-02-26T09:14:53.549Z" },
+    { url = "https://files.pythonhosted.org/packages/58/11/7456afb740bd45782d0f4c8e8e1bb9e572f1bf82899fb6ace58af47b4282/safetensors-0.5.3-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8bd84b12b1670a6f8e50f01e28156422a2bc07fb16fc4e98bded13039d688a0d", size = 522891, upload-time = "2025-02-26T09:14:55.717Z" },
+    { url = "https://files.pythonhosted.org/packages/57/3d/fe73a9d2ace487e7285f6e157afee2383bd1ddb911b7cb44a55cf812eae3/safetensors-0.5.3-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:391ac8cab7c829452175f871fcaf414aa1e292b5448bd02620f675a7f3e7abb9", size = 537694, upload-time = "2025-02-26T09:14:57.036Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/f8/dae3421624fcc87a89d42e1898a798bc7ff72c61f38973a65d60df8f124c/safetensors-0.5.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cead1fa41fc54b1e61089fa57452e8834f798cb1dc7a09ba3524f1eb08e0317a", size = 471642, upload-time = "2025-02-26T09:15:00.544Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/20/1fbe16f9b815f6c5a672f5b760951e20e17e43f67f231428f871909a37f6/safetensors-0.5.3-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1077f3e94182d72618357b04b5ced540ceb71c8a813d3319f1aba448e68a770d", size = 502241, upload-time = "2025-02-26T09:14:58.303Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/18/8e108846b506487aa4629fe4116b27db65c3dde922de2c8e0cc1133f3f29/safetensors-0.5.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:799021e78287bac619c7b3f3606730a22da4cda27759ddf55d37c8db7511c74b", size = 638001, upload-time = "2025-02-26T09:15:05.79Z" },
+    { url = "https://files.pythonhosted.org/packages/82/5a/c116111d8291af6c8c8a8b40628fe833b9db97d8141c2a82359d14d9e078/safetensors-0.5.3-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:df26da01aaac504334644e1b7642fa000bfec820e7cef83aeac4e355e03195ff", size = 734013, upload-time = "2025-02-26T09:15:07.892Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/ff/41fcc4d3b7de837963622e8610d998710705bbde9a8a17221d85e5d0baad/safetensors-0.5.3-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:32c3ef2d7af8b9f52ff685ed0bc43913cdcde135089ae322ee576de93eae5135", size = 670687, upload-time = "2025-02-26T09:15:09.979Z" },
+    { url = "https://files.pythonhosted.org/packages/40/ad/2b113098e69c985a3d8fbda4b902778eae4a35b7d5188859b4a63d30c161/safetensors-0.5.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:37f1521be045e56fc2b54c606d4455573e717b2d887c579ee1dbba5f868ece04", size = 643147, upload-time = "2025-02-26T09:15:11.185Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/0c/95aeb51d4246bd9a3242d3d8349c1112b4ee7611a4b40f0c5c93b05f001d/safetensors-0.5.3-cp38-abi3-win32.whl", hash = "sha256:cfc0ec0846dcf6763b0ed3d1846ff36008c6e7290683b61616c4b040f6a54ace", size = 296677, upload-time = "2025-02-26T09:15:16.554Z" },
+    { url = "https://files.pythonhosted.org/packages/69/e2/b011c38e5394c4c18fb5500778a55ec43ad6106126e74723ffaee246f56e/safetensors-0.5.3-cp38-abi3-win_amd64.whl", hash = "sha256:836cbbc320b47e80acd40e44c8682db0e8ad7123209f69b093def21ec7cafd11", size = 308878, upload-time = "2025-02-26T09:15:14.99Z" },
+]
+
+[[package]]
+name = "scikit-learn"
+version = "1.6.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "joblib" },
+    { name = "numpy" },
+    { name = "scipy" },
+    { name = "threadpoolctl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9e/a5/4ae3b3a0755f7b35a280ac90b28817d1f380318973cff14075ab41ef50d9/scikit_learn-1.6.1.tar.gz", hash = "sha256:b4fc2525eca2c69a59260f583c56a7557c6ccdf8deafdba6e060f94c1c59738e", size = 7068312, upload-time = "2025-01-10T08:07:55.348Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2e/3a/f4597eb41049110b21ebcbb0bcb43e4035017545daa5eedcfeb45c08b9c5/scikit_learn-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d056391530ccd1e501056160e3c9673b4da4805eb67eb2bdf4e983e1f9c9204e", size = 12067702, upload-time = "2025-01-10T08:05:56.515Z" },
+    { url = "https://files.pythonhosted.org/packages/37/19/0423e5e1fd1c6ec5be2352ba05a537a473c1677f8188b9306097d684b327/scikit_learn-1.6.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0c8d036eb937dbb568c6242fa598d551d88fb4399c0344d95c001980ec1c7d36", size = 11112765, upload-time = "2025-01-10T08:06:00.272Z" },
+    { url = "https://files.pythonhosted.org/packages/70/95/d5cb2297a835b0f5fc9a77042b0a2d029866379091ab8b3f52cc62277808/scikit_learn-1.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8634c4bd21a2a813e0a7e3900464e6d593162a29dd35d25bdf0103b3fce60ed5", size = 12643991, upload-time = "2025-01-10T08:06:04.813Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/91/ab3c697188f224d658969f678be86b0968ccc52774c8ab4a86a07be13c25/scikit_learn-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:775da975a471c4f6f467725dff0ced5c7ac7bda5e9316b260225b48475279a1b", size = 13497182, upload-time = "2025-01-10T08:06:08.42Z" },
+    { url = "https://files.pythonhosted.org/packages/17/04/d5d556b6c88886c092cc989433b2bab62488e0f0dafe616a1d5c9cb0efb1/scikit_learn-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:8a600c31592bd7dab31e1c61b9bbd6dea1b3433e67d264d17ce1017dbdce8002", size = 11125517, upload-time = "2025-01-10T08:06:12.783Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/2a/e291c29670795406a824567d1dfc91db7b699799a002fdaa452bceea8f6e/scikit_learn-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:72abc587c75234935e97d09aa4913a82f7b03ee0b74111dcc2881cba3c5a7b33", size = 12102620, upload-time = "2025-01-10T08:06:16.675Z" },
+    { url = "https://files.pythonhosted.org/packages/25/92/ee1d7a00bb6b8c55755d4984fd82608603a3cc59959245068ce32e7fb808/scikit_learn-1.6.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b3b00cdc8f1317b5f33191df1386c0befd16625f49d979fe77a8d44cae82410d", size = 11116234, upload-time = "2025-01-10T08:06:21.83Z" },
+    { url = "https://files.pythonhosted.org/packages/30/cd/ed4399485ef364bb25f388ab438e3724e60dc218c547a407b6e90ccccaef/scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc4765af3386811c3ca21638f63b9cf5ecf66261cc4815c1db3f1e7dc7b79db2", size = 12592155, upload-time = "2025-01-10T08:06:27.309Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/f3/62fc9a5a659bb58a03cdd7e258956a5824bdc9b4bb3c5d932f55880be569/scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25fc636bdaf1cc2f4a124a116312d837148b5e10872147bdaf4887926b8c03d8", size = 13497069, upload-time = "2025-01-10T08:06:32.515Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/a6/c5b78606743a1f28eae8f11973de6613a5ee87366796583fb74c67d54939/scikit_learn-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:fa909b1a36e000a03c382aade0bd2063fd5680ff8b8e501660c0f59f021a6415", size = 11139809, upload-time = "2025-01-10T08:06:35.514Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/18/c797c9b8c10380d05616db3bfb48e2a3358c767affd0857d56c2eb501caa/scikit_learn-1.6.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:926f207c804104677af4857b2c609940b743d04c4c35ce0ddc8ff4f053cddc1b", size = 12104516, upload-time = "2025-01-10T08:06:40.009Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/b7/2e35f8e289ab70108f8cbb2e7a2208f0575dc704749721286519dcf35f6f/scikit_learn-1.6.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2c2cae262064e6a9b77eee1c8e768fc46aa0b8338c6a8297b9b6759720ec0ff2", size = 11167837, upload-time = "2025-01-10T08:06:43.305Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/f6/ff7beaeb644bcad72bcfd5a03ff36d32ee4e53a8b29a639f11bcb65d06cd/scikit_learn-1.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1061b7c028a8663fb9a1a1baf9317b64a257fcb036dae5c8752b2abef31d136f", size = 12253728, upload-time = "2025-01-10T08:06:47.618Z" },
+    { url = "https://files.pythonhosted.org/packages/29/7a/8bce8968883e9465de20be15542f4c7e221952441727c4dad24d534c6d99/scikit_learn-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e69fab4ebfc9c9b580a7a80111b43d214ab06250f8a7ef590a4edf72464dd86", size = 13147700, upload-time = "2025-01-10T08:06:50.888Z" },
+    { url = "https://files.pythonhosted.org/packages/62/27/585859e72e117fe861c2079bcba35591a84f801e21bc1ab85bce6ce60305/scikit_learn-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:70b1d7e85b1c96383f872a519b3375f92f14731e279a7b4c6cfd650cf5dffc52", size = 11110613, upload-time = "2025-01-10T08:06:54.115Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/59/8eb1872ca87009bdcdb7f3cdc679ad557b992c12f4b61f9250659e592c63/scikit_learn-1.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2ffa1e9e25b3d93990e74a4be2c2fc61ee5af85811562f1288d5d055880c4322", size = 12010001, upload-time = "2025-01-10T08:06:58.613Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/05/f2fc4effc5b32e525408524c982c468c29d22f828834f0625c5ef3d601be/scikit_learn-1.6.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:dc5cf3d68c5a20ad6d571584c0750ec641cc46aeef1c1507be51300e6003a7e1", size = 11096360, upload-time = "2025-01-10T08:07:01.556Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/e4/4195d52cf4f113573fb8ebc44ed5a81bd511a92c0228889125fac2f4c3d1/scikit_learn-1.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c06beb2e839ecc641366000ca84f3cf6fa9faa1777e29cf0c04be6e4d096a348", size = 12209004, upload-time = "2025-01-10T08:07:06.931Z" },
+    { url = "https://files.pythonhosted.org/packages/94/be/47e16cdd1e7fcf97d95b3cb08bde1abb13e627861af427a3651fcb80b517/scikit_learn-1.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8ca8cb270fee8f1f76fa9bfd5c3507d60c6438bbee5687f81042e2bb98e5a97", size = 13171776, upload-time = "2025-01-10T08:07:11.715Z" },
+    { url = "https://files.pythonhosted.org/packages/34/b0/ca92b90859070a1487827dbc672f998da95ce83edce1270fc23f96f1f61a/scikit_learn-1.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:7a1c43c8ec9fde528d664d947dc4c0789be4077a3647f232869f41d9bf50e0fb", size = 11071865, upload-time = "2025-01-10T08:07:16.088Z" },
+    { url = "https://files.pythonhosted.org/packages/12/ae/993b0fb24a356e71e9a894e42b8a9eec528d4c70217353a1cd7a48bc25d4/scikit_learn-1.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a17c1dea1d56dcda2fac315712f3651a1fea86565b64b48fa1bc090249cbf236", size = 11955804, upload-time = "2025-01-10T08:07:20.385Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/54/32fa2ee591af44507eac86406fa6bba968d1eb22831494470d0a2e4a1eb1/scikit_learn-1.6.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a7aa5f9908f0f28f4edaa6963c0a6183f1911e63a69aa03782f0d924c830a35", size = 11100530, upload-time = "2025-01-10T08:07:23.675Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/58/55856da1adec655bdce77b502e94a267bf40a8c0b89f8622837f89503b5a/scikit_learn-1.6.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0650e730afb87402baa88afbf31c07b84c98272622aaba002559b614600ca691", size = 12433852, upload-time = "2025-01-10T08:07:26.817Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/4f/c83853af13901a574f8f13b645467285a48940f185b690936bb700a50863/scikit_learn-1.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:3f59fe08dc03ea158605170eb52b22a105f238a5d512c4470ddeca71feae8e5f", size = 11337256, upload-time = "2025-01-10T08:07:31.084Z" },
+]
+
+[[package]]
+name = "scipy"
+version = "1.13.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ae/00/48c2f661e2816ccf2ecd77982f6605b2950afe60f60a52b4cbbc2504aa8f/scipy-1.13.1.tar.gz", hash = "sha256:095a87a0312b08dfd6a6155cbbd310a8c51800fc931b8c0b84003014b874ed3c", size = 57210720, upload-time = "2024-05-23T03:29:26.079Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/33/59/41b2529908c002ade869623b87eecff3e11e3ce62e996d0bdcb536984187/scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca", size = 39328076, upload-time = "2024-05-23T03:19:01.687Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/33/f1307601f492f764062ce7dd471a14750f3360e33cd0f8c614dae208492c/scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f", size = 30306232, upload-time = "2024-05-23T03:19:09.089Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/66/9cd4f501dd5ea03e4a4572ecd874936d0da296bd04d1c45ae1a4a75d9c3a/scipy-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfa31f1def5c819b19ecc3a8b52d28ffdcc7ed52bb20c9a7589669dd3c250989", size = 33743202, upload-time = "2024-05-23T03:19:15.138Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/ba/7255e5dc82a65adbe83771c72f384d99c43063648456796436c9a5585ec3/scipy-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26264b282b9da0952a024ae34710c2aff7d27480ee91a2e82b7b7073c24722f", size = 38577335, upload-time = "2024-05-23T03:19:21.984Z" },
+    { url = "https://files.pythonhosted.org/packages/49/a5/bb9ded8326e9f0cdfdc412eeda1054b914dfea952bda2097d174f8832cc0/scipy-1.13.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eccfa1906eacc02de42d70ef4aecea45415f5be17e72b61bafcfd329bdc52e94", size = 38820728, upload-time = "2024-05-23T03:19:28.225Z" },
+    { url = "https://files.pythonhosted.org/packages/12/30/df7a8fcc08f9b4a83f5f27cfaaa7d43f9a2d2ad0b6562cced433e5b04e31/scipy-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:2831f0dc9c5ea9edd6e51e6e769b655f08ec6db6e2e10f86ef39bd32eb11da54", size = 46210588, upload-time = "2024-05-23T03:19:35.661Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/15/4a4bb1b15bbd2cd2786c4f46e76b871b28799b67891f23f455323a0cdcfb/scipy-1.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:27e52b09c0d3a1d5b63e1105f24177e544a222b43611aaf5bc44d4a0979e32f9", size = 39333805, upload-time = "2024-05-23T03:19:43.081Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/92/42476de1af309c27710004f5cdebc27bec62c204db42e05b23a302cb0c9a/scipy-1.13.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:54f430b00f0133e2224c3ba42b805bfd0086fe488835effa33fa291561932326", size = 30317687, upload-time = "2024-05-23T03:19:48.799Z" },
+    { url = "https://files.pythonhosted.org/packages/80/ba/8be64fe225360a4beb6840f3cbee494c107c0887f33350d0a47d55400b01/scipy-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e89369d27f9e7b0884ae559a3a956e77c02114cc60a6058b4e5011572eea9299", size = 33694638, upload-time = "2024-05-23T03:19:55.104Z" },
+    { url = "https://files.pythonhosted.org/packages/36/07/035d22ff9795129c5a847c64cb43c1fa9188826b59344fee28a3ab02e283/scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78b4b3345f1b6f68a763c6e25c0c9a23a9fd0f39f5f3d200efe8feda560a5fa", size = 38569931, upload-time = "2024-05-23T03:20:01.82Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/10/f9b43de37e5ed91facc0cfff31d45ed0104f359e4f9a68416cbf4e790241/scipy-1.13.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45484bee6d65633752c490404513b9ef02475b4284c4cfab0ef946def50b3f59", size = 38838145, upload-time = "2024-05-23T03:20:09.173Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/48/4513a1a5623a23e95f94abd675ed91cfb19989c58e9f6f7d03990f6caf3d/scipy-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:5713f62f781eebd8d597eb3f88b8bf9274e79eeabf63afb4a737abc6c84ad37b", size = 46196227, upload-time = "2024-05-23T03:20:16.433Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/7b/fb6b46fbee30fc7051913068758414f2721003a89dd9a707ad49174e3843/scipy-1.13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5d72782f39716b2b3509cd7c33cdc08c96f2f4d2b06d51e52fb45a19ca0c86a1", size = 39357301, upload-time = "2024-05-23T03:20:23.538Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/5a/2043a3bde1443d94014aaa41e0b50c39d046dda8360abd3b2a1d3f79907d/scipy-1.13.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:017367484ce5498445aade74b1d5ab377acdc65e27095155e448c88497755a5d", size = 30363348, upload-time = "2024-05-23T03:20:29.885Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/cb/26e4a47364bbfdb3b7fb3363be6d8a1c543bcd70a7753ab397350f5f189a/scipy-1.13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:949ae67db5fa78a86e8fa644b9a6b07252f449dcf74247108c50e1d20d2b4627", size = 33406062, upload-time = "2024-05-23T03:20:36.012Z" },
+    { url = "https://files.pythonhosted.org/packages/88/ab/6ecdc526d509d33814835447bbbeedbebdec7cca46ef495a61b00a35b4bf/scipy-1.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de3ade0e53bc1f21358aa74ff4830235d716211d7d077e340c7349bc3542e884", size = 38218311, upload-time = "2024-05-23T03:20:42.086Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/00/9f54554f0f8318100a71515122d8f4f503b1a2c4b4cfab3b4b68c0eb08fa/scipy-1.13.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2ac65fb503dad64218c228e2dc2d0a0193f7904747db43014645ae139c8fad16", size = 38442493, upload-time = "2024-05-23T03:20:48.292Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/df/963384e90733e08eac978cd103c34df181d1fec424de383cdc443f418dd4/scipy-1.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:cdd7dacfb95fea358916410ec61bbc20440f7860333aee6d882bb8046264e949", size = 45910955, upload-time = "2024-05-23T03:20:55.091Z" },
+]
+
+[[package]]
+name = "selenium"
+version = "4.34.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "trio" },
+    { name = "trio-websocket" },
+    { name = "typing-extensions" },
+    { name = "urllib3", extra = ["socks"] },
+    { name = "websocket-client" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b0/e6/646d0a41fb9a64572043c3de80be2a4941f2aeb578f273cf3dae54fc9437/selenium-4.34.2.tar.gz", hash = "sha256:0f6d147595f08c6d4bad87b34c39dcacb4650aedc78e3956c8eac1bb752a3854", size = 896309, upload-time = "2025-07-08T12:54:54.785Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f3/2b/dee1c58bde0a747b2d75fa7282a190885a726fe95b18b8ce1dc52f9c0983/selenium-4.34.2-py3-none-any.whl", hash = "sha256:ea208f7db9e3b26e58c4a817ea9dd29454576d6ea55937d754df079ad588e1ad", size = 9410676, upload-time = "2025-07-08T12:54:48.725Z" },
+]
+
+[[package]]
+name = "sentence-transformers"
+version = "5.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "huggingface-hub" },
+    { name = "pillow" },
+    { name = "scikit-learn" },
+    { name = "scipy" },
+    { name = "torch" },
+    { name = "tqdm" },
+    { name = "transformers" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/99/69/2a29773b43a24ee04eb26af492d85d520b30a86cfef22a0885e77e9c4a16/sentence_transformers-5.0.0.tar.gz", hash = "sha256:e5a411845910275fd166bacb01d28b7f79537d3550628ae42309dbdd3d5670d1", size = 366847, upload-time = "2025-07-01T13:01:33.04Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6f/ff/178f08ea5ebc1f9193d9de7f601efe78c01748347875c8438f66f5cecc19/sentence_transformers-5.0.0-py3-none-any.whl", hash = "sha256:346240f9cc6b01af387393f03e103998190dfb0826a399d0c38a81a05c7a5d76", size = 470191, upload-time = "2025-07-01T13:01:31.619Z" },
+]
+
+[[package]]
+name = "setuptools"
+version = "80.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" },
+]
+
+[[package]]
+name = "shapely"
+version = "2.0.7"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/21/c0/a911d1fd765d07a2b6769ce155219a281bfbe311584ebe97340d75c5bdb1/shapely-2.0.7.tar.gz", hash = "sha256:28fe2997aab9a9dc026dc6a355d04e85841546b2a5d232ed953e3321ab958ee5", size = 283413, upload-time = "2025-01-31T01:10:20.787Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/15/2e/02c694d6ddacd4f13b625722d313d2838f23c5b988cbc680132983f73ce3/shapely-2.0.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:33fb10e50b16113714ae40adccf7670379e9ccf5b7a41d0002046ba2b8f0f691", size = 1478310, upload-time = "2025-01-31T02:42:18.134Z" },
+    { url = "https://files.pythonhosted.org/packages/87/69/b54a08bcd25e561bdd5183c008ace4424c25e80506e80674032504800efd/shapely-2.0.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f44eda8bd7a4bccb0f281264b34bf3518d8c4c9a8ffe69a1a05dabf6e8461147", size = 1336082, upload-time = "2025-01-31T02:42:19.986Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/f9/40473fcb5b66ff849e563ca523d2a26dafd6957d52dd876ffd0eded39f1c/shapely-2.0.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf6c50cd879831955ac47af9c907ce0310245f9d162e298703f82e1785e38c98", size = 2371047, upload-time = "2025-01-31T02:42:22.724Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/f3/c9cc07a7a03b5f5e83bd059f9adf3e21cf086b0e41d7f95e6464b151e798/shapely-2.0.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04a65d882456e13c8b417562c36324c0cd1e5915f3c18ad516bb32ee3f5fc895", size = 2469112, upload-time = "2025-01-31T02:42:26.739Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/b9/fc63d6b0b25063a3ff806857a5dc88851d54d1c278288f18cef1b322b449/shapely-2.0.7-cp310-cp310-win32.whl", hash = "sha256:7e97104d28e60b69f9b6a957c4d3a2a893b27525bc1fc96b47b3ccef46726bf2", size = 1296057, upload-time = "2025-01-31T02:42:29.156Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/d1/8df43f94cf4cda0edbab4545f7cdd67d3f1d02910eaff152f9f45c6d00d8/shapely-2.0.7-cp310-cp310-win_amd64.whl", hash = "sha256:35524cc8d40ee4752520819f9894b9f28ba339a42d4922e92c99b148bed3be39", size = 1441787, upload-time = "2025-01-31T02:42:31.412Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/ad/21798c2fec013e289f8ab91d42d4d3299c315b8c4460c08c75fef0901713/shapely-2.0.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5cf23400cb25deccf48c56a7cdda8197ae66c0e9097fcdd122ac2007e320bc34", size = 1473091, upload-time = "2025-01-31T02:42:33.595Z" },
+    { url = "https://files.pythonhosted.org/packages/15/63/eef4f180f1b5859c70e7f91d2f2570643e5c61e7d7c40743d15f8c6cbc42/shapely-2.0.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8f1da01c04527f7da59ee3755d8ee112cd8967c15fab9e43bba936b81e2a013", size = 1332921, upload-time = "2025-01-31T02:42:34.993Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/67/77851dd17738bbe7762a0ef1acf7bc499d756f68600dd68a987d78229412/shapely-2.0.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f623b64bb219d62014781120f47499a7adc30cf7787e24b659e56651ceebcb0", size = 2427949, upload-time = "2025-01-31T02:42:37.578Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/a5/2c8dbb0f383519771df19164e3bf3a8895d195d2edeab4b6040f176ee28e/shapely-2.0.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6d95703efaa64aaabf278ced641b888fc23d9c6dd71f8215091afd8a26a66e3", size = 2529282, upload-time = "2025-01-31T02:42:39.504Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/4e/e1d608773c7fe4cde36d48903c0d6298e3233dc69412403783ac03fa5205/shapely-2.0.7-cp311-cp311-win32.whl", hash = "sha256:2f6e4759cf680a0f00a54234902415f2fa5fe02f6b05546c662654001f0793a2", size = 1295751, upload-time = "2025-01-31T02:42:41.107Z" },
+    { url = "https://files.pythonhosted.org/packages/27/57/8ec7c62012bed06731f7ee979da7f207bbc4b27feed5f36680b6a70df54f/shapely-2.0.7-cp311-cp311-win_amd64.whl", hash = "sha256:b52f3ab845d32dfd20afba86675c91919a622f4627182daec64974db9b0b4608", size = 1442684, upload-time = "2025-01-31T02:42:43.181Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/3e/ea100eec5811bafd0175eb21828a3be5b0960f65250f4474391868be7c0f/shapely-2.0.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4c2b9859424facbafa54f4a19b625a752ff958ab49e01bc695f254f7db1835fa", size = 1482451, upload-time = "2025-01-31T02:42:44.902Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/53/c6a3487716fd32e1f813d2a9608ba7b72a8a52a6966e31c6443480a1d016/shapely-2.0.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5aed1c6764f51011d69a679fdf6b57e691371ae49ebe28c3edb5486537ffbd51", size = 1345765, upload-time = "2025-01-31T02:42:46.625Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/dd/b35d7891d25cc11066a70fb8d8169a6a7fca0735dd9b4d563a84684969a3/shapely-2.0.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73c9ae8cf443187d784d57202199bf9fd2d4bb7d5521fe8926ba40db1bc33e8e", size = 2421540, upload-time = "2025-01-31T02:42:49.971Z" },
+    { url = "https://files.pythonhosted.org/packages/62/de/8dbd7df60eb23cb983bb698aac982944b3d602ef0ce877a940c269eae34e/shapely-2.0.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9469f49ff873ef566864cb3516091881f217b5d231c8164f7883990eec88b73", size = 2525741, upload-time = "2025-01-31T02:42:53.882Z" },
+    { url = "https://files.pythonhosted.org/packages/96/64/faf0413ebc7a84fe7a0790bf39ec0b02b40132b68e57aba985c0b6e4e7b6/shapely-2.0.7-cp312-cp312-win32.whl", hash = "sha256:6bca5095e86be9d4ef3cb52d56bdd66df63ff111d580855cb8546f06c3c907cd", size = 1296552, upload-time = "2025-01-31T02:42:55.714Z" },
+    { url = "https://files.pythonhosted.org/packages/63/05/8a1c279c226d6ad7604d9e237713dd21788eab96db97bf4ce0ea565e5596/shapely-2.0.7-cp312-cp312-win_amd64.whl", hash = "sha256:f86e2c0259fe598c4532acfcf638c1f520fa77c1275912bbc958faecbf00b108", size = 1443464, upload-time = "2025-01-31T02:42:57.696Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/21/abea43effbfe11f792e44409ee9ad7635aa93ef1c8ada0ef59b3c1c3abad/shapely-2.0.7-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a0c09e3e02f948631c7763b4fd3dd175bc45303a0ae04b000856dedebefe13cb", size = 1481618, upload-time = "2025-01-31T02:42:59.915Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/71/af688798da36fe355a6e6ffe1d4628449cb5fa131d57fc169bcb614aeee7/shapely-2.0.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:06ff6020949b44baa8fc2e5e57e0f3d09486cd5c33b47d669f847c54136e7027", size = 1345159, upload-time = "2025-01-31T02:43:01.611Z" },
+    { url = "https://files.pythonhosted.org/packages/67/47/f934fe2b70d31bb9774ad4376e34f81666deed6b811306ff574faa3d115e/shapely-2.0.7-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d6dbf096f961ca6bec5640e22e65ccdec11e676344e8157fe7d636e7904fd36", size = 2410267, upload-time = "2025-01-31T02:43:05.83Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/8a/2545cc2a30afc63fc6176c1da3b76af28ef9c7358ed4f68f7c6a9d86cf5b/shapely-2.0.7-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adeddfb1e22c20548e840403e5e0b3d9dc3daf66f05fa59f1fcf5b5f664f0e98", size = 2514128, upload-time = "2025-01-31T02:43:08.427Z" },
+    { url = "https://files.pythonhosted.org/packages/87/54/2344ce7da39676adec94e84fbaba92a8f1664e4ae2d33bd404dafcbe607f/shapely-2.0.7-cp313-cp313-win32.whl", hash = "sha256:a7f04691ce1c7ed974c2f8b34a1fe4c3c5dfe33128eae886aa32d730f1ec1913", size = 1295783, upload-time = "2025-01-31T02:43:10.608Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/1e/6461e5cfc8e73ae165b8cff6eb26a4d65274fad0e1435137c5ba34fe4e88/shapely-2.0.7-cp313-cp313-win_amd64.whl", hash = "sha256:aaaf5f7e6cc234c1793f2a2760da464b604584fb58c6b6d7d94144fd2692d67e", size = 1442300, upload-time = "2025-01-31T02:43:12.299Z" },
+]
+
+[[package]]
+name = "sniffio"
+version = "1.3.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
+]
+
+[[package]]
+name = "snowballstemmer"
+version = "2.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/44/7b/af302bebf22c749c56c9c3e8ae13190b5b5db37a33d9068652e8f73b7089/snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1", size = 86699, upload-time = "2021-11-16T18:38:38.009Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a", size = 93002, upload-time = "2021-11-16T18:38:34.792Z" },
+]
+
+[[package]]
+name = "sortedcontainers"
+version = "2.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" },
+]
+
+[[package]]
+name = "soupsieve"
+version = "2.7"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3f/f4/4a80cd6ef364b2e8b65b15816a843c0980f7a5a2b4dc701fc574952aa19f/soupsieve-2.7.tar.gz", hash = "sha256:ad282f9b6926286d2ead4750552c8a6142bc4c783fd66b0293547c8fe6ae126a", size = 103418, upload-time = "2025-04-20T18:50:08.518Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e7/9c/0e6afc12c269578be5c0c1c9f4b49a8d32770a080260c333ac04cc1c832d/soupsieve-2.7-py3-none-any.whl", hash = "sha256:6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4", size = 36677, upload-time = "2025-04-20T18:50:07.196Z" },
+]
+
+[[package]]
+name = "sympy"
+version = "1.14.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mpmath" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" },
+]
+
+[[package]]
+name = "tf-playwright-stealth"
+version = "1.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "fake-http-header" },
+    { name = "playwright" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d6/6b/32bb58c65991f91aeaaf7473b650175d9d4af5dd383983d177d49ccba08d/tf_playwright_stealth-1.2.0.tar.gz", hash = "sha256:7bb8d32d3e60324fbf6b9eeae540b8cd9f3b9e07baeb33b025dbc98ad47658ba", size = 23362, upload-time = "2025-06-13T04:51:04.97Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/11/3d/2653f4cf49660bb44eeac8270617cc4c0287d61716f249f55053f0af0724/tf_playwright_stealth-1.2.0-py3-none-any.whl", hash = "sha256:26ee47ee89fa0f43c606fe37c188ea3ccd36f96ea90c01d167b768df457e7886", size = 33151, upload-time = "2025-06-13T04:51:03.769Z" },
+]
+
+[[package]]
+name = "threadpoolctl"
+version = "3.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" },
+]
+
+[[package]]
+name = "tiktoken"
+version = "0.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "regex" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ea/cf/756fedf6981e82897f2d570dd25fa597eb3f4459068ae0572d7e888cfd6f/tiktoken-0.9.0.tar.gz", hash = "sha256:d02a5ca6a938e0490e1ff957bc48c8b078c88cb83977be1625b1fd8aac792c5d", size = 35991, upload-time = "2025-02-14T06:03:01.003Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/64/f3/50ec5709fad61641e4411eb1b9ac55b99801d71f1993c29853f256c726c9/tiktoken-0.9.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:586c16358138b96ea804c034b8acf3f5d3f0258bd2bc3b0227af4af5d622e382", size = 1065770, upload-time = "2025-02-14T06:02:01.251Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/f8/5a9560a422cf1755b6e0a9a436e14090eeb878d8ec0f80e0cd3d45b78bf4/tiktoken-0.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9c59ccc528c6c5dd51820b3474402f69d9a9e1d656226848ad68a8d5b2e5108", size = 1009314, upload-time = "2025-02-14T06:02:02.869Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/20/3ed4cfff8f809cb902900ae686069e029db74567ee10d017cb254df1d598/tiktoken-0.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0968d5beeafbca2a72c595e8385a1a1f8af58feaebb02b227229b69ca5357fd", size = 1143140, upload-time = "2025-02-14T06:02:04.165Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/95/cc2c6d79df8f113bdc6c99cdec985a878768120d87d839a34da4bd3ff90a/tiktoken-0.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92a5fb085a6a3b7350b8fc838baf493317ca0e17bd95e8642f95fc69ecfed1de", size = 1197860, upload-time = "2025-02-14T06:02:06.268Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/6c/9c1a4cc51573e8867c9381db1814223c09ebb4716779c7f845d48688b9c8/tiktoken-0.9.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:15a2752dea63d93b0332fb0ddb05dd909371ededa145fe6a3242f46724fa7990", size = 1259661, upload-time = "2025-02-14T06:02:08.889Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/4c/22eb8e9856a2b1808d0a002d171e534eac03f96dbe1161978d7389a59498/tiktoken-0.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:26113fec3bd7a352e4b33dbaf1bd8948de2507e30bd95a44e2b1156647bc01b4", size = 894026, upload-time = "2025-02-14T06:02:12.841Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/ae/4613a59a2a48e761c5161237fc850eb470b4bb93696db89da51b79a871f1/tiktoken-0.9.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f32cc56168eac4851109e9b5d327637f15fd662aa30dd79f964b7c39fbadd26e", size = 1065987, upload-time = "2025-02-14T06:02:14.174Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/86/55d9d1f5b5a7e1164d0f1538a85529b5fcba2b105f92db3622e5d7de6522/tiktoken-0.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:45556bc41241e5294063508caf901bf92ba52d8ef9222023f83d2483a3055348", size = 1009155, upload-time = "2025-02-14T06:02:15.384Z" },
+    { url = "https://files.pythonhosted.org/packages/03/58/01fb6240df083b7c1916d1dcb024e2b761213c95d576e9f780dfb5625a76/tiktoken-0.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33", size = 1142898, upload-time = "2025-02-14T06:02:16.666Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/73/41591c525680cd460a6becf56c9b17468d3711b1df242c53d2c7b2183d16/tiktoken-0.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b3d80aad8d2c6b9238fc1a5524542087c52b860b10cbf952429ffb714bc1136", size = 1197535, upload-time = "2025-02-14T06:02:18.595Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/7c/1069f25521c8f01a1a182f362e5c8e0337907fae91b368b7da9c3e39b810/tiktoken-0.9.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b2a21133be05dc116b1d0372af051cd2c6aa1d2188250c9b553f9fa49301b336", size = 1259548, upload-time = "2025-02-14T06:02:20.729Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/07/c67ad1724b8e14e2b4c8cca04b15da158733ac60136879131db05dda7c30/tiktoken-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:11a20e67fdf58b0e2dea7b8654a288e481bb4fc0289d3ad21291f8d0849915fb", size = 893895, upload-time = "2025-02-14T06:02:22.67Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/e5/21ff33ecfa2101c1bb0f9b6df750553bd873b7fb532ce2cb276ff40b197f/tiktoken-0.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e88f121c1c22b726649ce67c089b90ddda8b9662545a8aeb03cfef15967ddd03", size = 1065073, upload-time = "2025-02-14T06:02:24.768Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/03/a95e7b4863ee9ceec1c55983e4cc9558bcfd8f4f80e19c4f8a99642f697d/tiktoken-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a6600660f2f72369acb13a57fb3e212434ed38b045fd8cc6cdd74947b4b5d210", size = 1008075, upload-time = "2025-02-14T06:02:26.92Z" },
+    { url = "https://files.pythonhosted.org/packages/40/10/1305bb02a561595088235a513ec73e50b32e74364fef4de519da69bc8010/tiktoken-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95e811743b5dfa74f4b227927ed86cbc57cad4df859cb3b643be797914e41794", size = 1140754, upload-time = "2025-02-14T06:02:28.124Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/40/da42522018ca496432ffd02793c3a72a739ac04c3794a4914570c9bb2925/tiktoken-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99376e1370d59bcf6935c933cb9ba64adc29033b7e73f5f7569f3aad86552b22", size = 1196678, upload-time = "2025-02-14T06:02:29.845Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/41/1e59dddaae270ba20187ceb8aa52c75b24ffc09f547233991d5fd822838b/tiktoken-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:badb947c32739fb6ddde173e14885fb3de4d32ab9d8c591cbd013c22b4c31dd2", size = 1259283, upload-time = "2025-02-14T06:02:33.838Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/64/b16003419a1d7728d0d8c0d56a4c24325e7b10a21a9dd1fc0f7115c02f0a/tiktoken-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:5a62d7a25225bafed786a524c1b9f0910a1128f4232615bf3f8257a73aaa3b16", size = 894897, upload-time = "2025-02-14T06:02:36.265Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/11/09d936d37f49f4f494ffe660af44acd2d99eb2429d60a57c71318af214e0/tiktoken-0.9.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b0e8e05a26eda1249e824156d537015480af7ae222ccb798e5234ae0285dbdb", size = 1064919, upload-time = "2025-02-14T06:02:37.494Z" },
+    { url = "https://files.pythonhosted.org/packages/80/0e/f38ba35713edb8d4197ae602e80837d574244ced7fb1b6070b31c29816e0/tiktoken-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:27d457f096f87685195eea0165a1807fae87b97b2161fe8c9b1df5bd74ca6f63", size = 1007877, upload-time = "2025-02-14T06:02:39.516Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/82/9197f77421e2a01373e27a79dd36efdd99e6b4115746ecc553318ecafbf0/tiktoken-0.9.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cf8ded49cddf825390e36dd1ad35cd49589e8161fdcb52aa25f0583e90a3e01", size = 1140095, upload-time = "2025-02-14T06:02:41.791Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/bb/4513da71cac187383541facd0291c4572b03ec23c561de5811781bbd988f/tiktoken-0.9.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc156cb314119a8bb9748257a2eaebd5cc0753b6cb491d26694ed42fc7cb3139", size = 1195649, upload-time = "2025-02-14T06:02:43Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/5c/74e4c137530dd8504e97e3a41729b1103a4ac29036cbfd3250b11fd29451/tiktoken-0.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cd69372e8c9dd761f0ab873112aba55a0e3e506332dd9f7522ca466e817b1b7a", size = 1258465, upload-time = "2025-02-14T06:02:45.046Z" },
+    { url = "https://files.pythonhosted.org/packages/de/a8/8f499c179ec900783ffe133e9aab10044481679bb9aad78436d239eee716/tiktoken-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:5ea0edb6f83dc56d794723286215918c1cde03712cbbafa0348b33448faf5b95", size = 894669, upload-time = "2025-02-14T06:02:47.341Z" },
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.21.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "huggingface-hub" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ab/2d/b0fce2b8201635f60e8c95990080f58461cc9ca3d5026de2e900f38a7f21/tokenizers-0.21.2.tar.gz", hash = "sha256:fdc7cffde3e2113ba0e6cc7318c40e3438a4d74bbc62bf04bcc63bdfb082ac77", size = 351545, upload-time = "2025-06-24T10:24:52.449Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1d/cc/2936e2d45ceb130a21d929743f1e9897514691bec123203e10837972296f/tokenizers-0.21.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:342b5dfb75009f2255ab8dec0041287260fed5ce00c323eb6bab639066fef8ec", size = 2875206, upload-time = "2025-06-24T10:24:42.755Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/e6/33f41f2cc7861faeba8988e7a77601407bf1d9d28fc79c5903f8f77df587/tokenizers-0.21.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:126df3205d6f3a93fea80c7a8a266a78c1bd8dd2fe043386bafdd7736a23e45f", size = 2732655, upload-time = "2025-06-24T10:24:41.56Z" },
+    { url = "https://files.pythonhosted.org/packages/33/2b/1791eb329c07122a75b01035b1a3aa22ad139f3ce0ece1b059b506d9d9de/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a32cd81be21168bd0d6a0f0962d60177c447a1aa1b1e48fa6ec9fc728ee0b12", size = 3019202, upload-time = "2025-06-24T10:24:31.791Z" },
+    { url = "https://files.pythonhosted.org/packages/05/15/fd2d8104faa9f86ac68748e6f7ece0b5eb7983c7efc3a2c197cb98c99030/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8bd8999538c405133c2ab999b83b17c08b7fc1b48c1ada2469964605a709ef91", size = 2934539, upload-time = "2025-06-24T10:24:34.567Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/2e/53e8fd053e1f3ffbe579ca5f9546f35ac67cf0039ed357ad7ec57f5f5af0/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e9944e61239b083a41cf8fc42802f855e1dca0f499196df37a8ce219abac6eb", size = 3248665, upload-time = "2025-06-24T10:24:39.024Z" },
+    { url = "https://files.pythonhosted.org/packages/00/15/79713359f4037aa8f4d1f06ffca35312ac83629da062670e8830917e2153/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:514cd43045c5d546f01142ff9c79a96ea69e4b5cda09e3027708cb2e6d5762ab", size = 3451305, upload-time = "2025-06-24T10:24:36.133Z" },
+    { url = "https://files.pythonhosted.org/packages/38/5f/959f3a8756fc9396aeb704292777b84f02a5c6f25c3fc3ba7530db5feb2c/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1b9405822527ec1e0f7d8d2fdb287a5730c3a6518189c968254a8441b21faae", size = 3214757, upload-time = "2025-06-24T10:24:37.784Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/74/f41a432a0733f61f3d21b288de6dfa78f7acff309c6f0f323b2833e9189f/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed9a4d51c395103ad24f8e7eb976811c57fbec2af9f133df471afcd922e5020", size = 3121887, upload-time = "2025-06-24T10:24:40.293Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/6a/bc220a11a17e5d07b0dfb3b5c628621d4dcc084bccd27cfaead659963016/tokenizers-0.21.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c41862df3d873665ec78b6be36fcc30a26e3d4902e9dd8608ed61d49a48bc19", size = 9091965, upload-time = "2025-06-24T10:24:44.431Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/bd/ac386d79c4ef20dc6f39c4706640c24823dca7ebb6f703bfe6b5f0292d88/tokenizers-0.21.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:ed21dc7e624e4220e21758b2e62893be7101453525e3d23264081c9ef9a6d00d", size = 9053372, upload-time = "2025-06-24T10:24:46.455Z" },
+    { url = "https://files.pythonhosted.org/packages/63/7b/5440bf203b2a5358f074408f7f9c42884849cd9972879e10ee6b7a8c3b3d/tokenizers-0.21.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:0e73770507e65a0e0e2a1affd6b03c36e3bc4377bd10c9ccf51a82c77c0fe365", size = 9298632, upload-time = "2025-06-24T10:24:48.446Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/d2/faa1acac3f96a7427866e94ed4289949b2524f0c1878512516567d80563c/tokenizers-0.21.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:106746e8aa9014a12109e58d540ad5465b4c183768ea96c03cbc24c44d329958", size = 9470074, upload-time = "2025-06-24T10:24:50.378Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/a5/896e1ef0707212745ae9f37e84c7d50269411aef2e9ccd0de63623feecdf/tokenizers-0.21.2-cp39-abi3-win32.whl", hash = "sha256:cabda5a6d15d620b6dfe711e1af52205266d05b379ea85a8a301b3593c60e962", size = 2330115, upload-time = "2025-06-24T10:24:55.069Z" },
+    { url = "https://files.pythonhosted.org/packages/13/c3/cc2755ee10be859c4338c962a35b9a663788c0c0b50c0bdd8078fb6870cf/tokenizers-0.21.2-cp39-abi3-win_amd64.whl", hash = "sha256:58747bb898acdb1007f37a7bbe614346e98dc28708ffb66a3fd50ce169ac6c98", size = 2509918, upload-time = "2025-06-24T10:24:53.71Z" },
+]
+
+[[package]]
+name = "torch"
+version = "2.7.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "filelock" },
+    { name = "fsspec" },
+    { name = "jinja2" },
+    { name = "networkx" },
+    { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "setuptools", marker = "python_full_version >= '3.12'" },
+    { name = "sympy" },
+    { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "typing-extensions" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6a/27/2e06cb52adf89fe6e020963529d17ed51532fc73c1e6d1b18420ef03338c/torch-2.7.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a103b5d782af5bd119b81dbcc7ffc6fa09904c423ff8db397a1e6ea8fd71508f", size = 99089441, upload-time = "2025-06-04T17:38:48.268Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/7c/0a5b3aee977596459ec45be2220370fde8e017f651fecc40522fd478cb1e/torch-2.7.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:fe955951bdf32d182ee8ead6c3186ad54781492bf03d547d31771a01b3d6fb7d", size = 821154516, upload-time = "2025-06-04T17:36:28.556Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/91/3d709cfc5e15995fb3fe7a6b564ce42280d3a55676dad672205e94f34ac9/torch-2.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:885453d6fba67d9991132143bf7fa06b79b24352f4506fd4d10b309f53454162", size = 216093147, upload-time = "2025-06-04T17:39:38.132Z" },
+    { url = "https://files.pythonhosted.org/packages/92/f6/5da3918414e07da9866ecb9330fe6ffdebe15cb9a4c5ada7d4b6e0a6654d/torch-2.7.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:d72acfdb86cee2a32c0ce0101606f3758f0d8bb5f8f31e7920dc2809e963aa7c", size = 68630914, upload-time = "2025-06-04T17:39:31.162Z" },
+    { url = "https://files.pythonhosted.org/packages/11/56/2eae3494e3d375533034a8e8cf0ba163363e996d85f0629441fa9d9843fe/torch-2.7.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:236f501f2e383f1cb861337bdf057712182f910f10aeaf509065d54d339e49b2", size = 99093039, upload-time = "2025-06-04T17:39:06.963Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/94/34b80bd172d0072c9979708ccd279c2da2f55c3ef318eceec276ab9544a4/torch-2.7.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:06eea61f859436622e78dd0cdd51dbc8f8c6d76917a9cf0555a333f9eac31ec1", size = 821174704, upload-time = "2025-06-04T17:37:03.799Z" },
+    { url = "https://files.pythonhosted.org/packages/50/9e/acf04ff375b0b49a45511c55d188bcea5c942da2aaf293096676110086d1/torch-2.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:8273145a2e0a3c6f9fd2ac36762d6ee89c26d430e612b95a99885df083b04e52", size = 216095937, upload-time = "2025-06-04T17:39:24.83Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/2b/d36d57c66ff031f93b4fa432e86802f84991477e522adcdffd314454326b/torch-2.7.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:aea4fc1bf433d12843eb2c6b2204861f43d8364597697074c8d38ae2507f8730", size = 68640034, upload-time = "2025-06-04T17:39:17.989Z" },
+    { url = "https://files.pythonhosted.org/packages/87/93/fb505a5022a2e908d81fe9a5e0aa84c86c0d5f408173be71c6018836f34e/torch-2.7.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:27ea1e518df4c9de73af7e8a720770f3628e7f667280bce2be7a16292697e3fa", size = 98948276, upload-time = "2025-06-04T17:39:12.852Z" },
+    { url = "https://files.pythonhosted.org/packages/56/7e/67c3fe2b8c33f40af06326a3d6ae7776b3e3a01daa8f71d125d78594d874/torch-2.7.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c33360cfc2edd976c2633b3b66c769bdcbbf0e0b6550606d188431c81e7dd1fc", size = 821025792, upload-time = "2025-06-04T17:34:58.747Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/37/a37495502bc7a23bf34f89584fa5a78e25bae7b8da513bc1b8f97afb7009/torch-2.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:d8bf6e1856ddd1807e79dc57e54d3335f2b62e6f316ed13ed3ecfe1fc1df3d8b", size = 216050349, upload-time = "2025-06-04T17:38:59.709Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/60/04b77281c730bb13460628e518c52721257814ac6c298acd25757f6a175c/torch-2.7.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:787687087412c4bd68d315e39bc1223f08aae1d16a9e9771d95eabbb04ae98fb", size = 68645146, upload-time = "2025-06-04T17:38:52.97Z" },
+    { url = "https://files.pythonhosted.org/packages/66/81/e48c9edb655ee8eb8c2a6026abdb6f8d2146abd1f150979ede807bb75dcb/torch-2.7.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:03563603d931e70722dce0e11999d53aa80a375a3d78e6b39b9f6805ea0a8d28", size = 98946649, upload-time = "2025-06-04T17:38:43.031Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/24/efe2f520d75274fc06b695c616415a1e8a1021d87a13c68ff9dce733d088/torch-2.7.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:d632f5417b6980f61404a125b999ca6ebd0b8b4bbdbb5fbbba44374ab619a412", size = 821033192, upload-time = "2025-06-04T17:38:09.146Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/d9/9c24d230333ff4e9b6807274f6f8d52a864210b52ec794c5def7925f4495/torch-2.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:23660443e13995ee93e3d844786701ea4ca69f337027b05182f5ba053ce43b38", size = 216055668, upload-time = "2025-06-04T17:38:36.253Z" },
+    { url = "https://files.pythonhosted.org/packages/95/bf/e086ee36ddcef9299f6e708d3b6c8487c1651787bb9ee2939eb2a7f74911/torch-2.7.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:0da4f4dba9f65d0d203794e619fe7ca3247a55ffdcbd17ae8fb83c8b2dc9b585", size = 68925988, upload-time = "2025-06-04T17:38:29.273Z" },
+    { url = "https://files.pythonhosted.org/packages/69/6a/67090dcfe1cf9048448b31555af6efb149f7afa0a310a366adbdada32105/torch-2.7.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e08d7e6f21a617fe38eeb46dd2213ded43f27c072e9165dc27300c9ef9570934", size = 99028857, upload-time = "2025-06-04T17:37:50.956Z" },
+    { url = "https://files.pythonhosted.org/packages/90/1c/48b988870823d1cc381f15ec4e70ed3d65e043f43f919329b0045ae83529/torch-2.7.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:30207f672328a42df4f2174b8f426f354b2baa0b7cca3a0adb3d6ab5daf00dc8", size = 821098066, upload-time = "2025-06-04T17:37:33.939Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/eb/10050d61c9d5140c5dc04a89ed3257ef1a6b93e49dd91b95363d757071e0/torch-2.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:79042feca1c634aaf6603fe6feea8c6b30dfa140a6bbc0b973e2260c7e79a22e", size = 216336310, upload-time = "2025-06-04T17:36:09.862Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/29/beb45cdf5c4fc3ebe282bf5eafc8dfd925ead7299b3c97491900fe5ed844/torch-2.7.1-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:988b0cbc4333618a1056d2ebad9eb10089637b659eb645434d0809d8d937b946", size = 68645708, upload-time = "2025-06-04T17:34:39.852Z" },
+]
+
+[[package]]
+name = "tqdm"
+version = "4.67.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" },
+]
+
+[[package]]
+name = "transformers"
+version = "4.53.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "filelock" },
+    { name = "huggingface-hub" },
+    { name = "numpy" },
+    { name = "packaging" },
+    { name = "pyyaml" },
+    { name = "regex" },
+    { name = "requests" },
+    { name = "safetensors" },
+    { name = "tokenizers" },
+    { name = "tqdm" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9f/2c/68a0024c311db41bb92d4ec17d22e90b7406a4d28aa18d87662f2bbebcd9/transformers-4.53.1.tar.gz", hash = "sha256:da5a9f66ad480bc2a7f75bc32eaf735fd20ac56af4325ca4ce994021ceb37710", size = 9192189, upload-time = "2025-07-04T08:28:40.571Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8d/10/8cef2288810a3210659eb3a20711e8387cc35a881a7762ae387806e2d651/transformers-4.53.1-py3-none-any.whl", hash = "sha256:c84f3c3e41c71fdf2c60c8a893e1cd31191b0cb463385f4c276302d2052d837b", size = 10825681, upload-time = "2025-07-04T08:28:37.318Z" },
+]
+
+[[package]]
+name = "trimesh"
+version = "4.7.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/05/07/0b5ecadb7c355c4fbe6c1c7038c57c5d472b509f9a4f5ff7996c1fed47a2/trimesh-4.7.0.tar.gz", hash = "sha256:13eb97c1f417f744b682d24ab77f86a21c6ccd1e3fb6ce3aa173a94ca982deee", size = 800555, upload-time = "2025-07-09T19:01:31.493Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/35/a5/95d40e832d38584a17bf5882a58b69ecf90f864b0b048dbc02b50146cbc1/trimesh-4.7.0-py3-none-any.whl", hash = "sha256:198d5b30c97d991fd47069cb4e7058ef8431de77a467148f0efde25f171ce73b", size = 708809, upload-time = "2025-07-09T19:01:28.622Z" },
+]
+
+[[package]]
+name = "trio"
+version = "0.30.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "attrs" },
+    { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" },
+    { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
+    { name = "idna" },
+    { name = "outcome" },
+    { name = "sniffio" },
+    { name = "sortedcontainers" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/01/c1/68d582b4d3a1c1f8118e18042464bb12a7c1b75d64d75111b297687041e3/trio-0.30.0.tar.gz", hash = "sha256:0781c857c0c81f8f51e0089929a26b5bb63d57f927728a5586f7e36171f064df", size = 593776, upload-time = "2025-04-21T00:48:19.507Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/69/8e/3f6dfda475ecd940e786defe6df6c500734e686c9cd0a0f8ef6821e9b2f2/trio-0.30.0-py3-none-any.whl", hash = "sha256:3bf4f06b8decf8d3cf00af85f40a89824669e2d033bb32469d34840edcfc22a5", size = 499194, upload-time = "2025-04-21T00:48:17.167Z" },
+]
+
+[[package]]
+name = "trio-websocket"
+version = "0.12.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
+    { name = "outcome" },
+    { name = "trio" },
+    { name = "wsproto" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d1/3c/8b4358e81f2f2cfe71b66a267f023a91db20a817b9425dd964873796980a/trio_websocket-0.12.2.tar.gz", hash = "sha256:22c72c436f3d1e264d0910a3951934798dcc5b00ae56fc4ee079d46c7cf20fae", size = 33549, upload-time = "2025-02-25T05:16:58.947Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c7/19/eb640a397bba49ba49ef9dbe2e7e5c04202ba045b6ce2ec36e9cadc51e04/trio_websocket-0.12.2-py3-none-any.whl", hash = "sha256:df605665f1db533f4a386c94525870851096a223adcb97f72a07e8b4beba45b6", size = 21221, upload-time = "2025-02-25T05:16:57.545Z" },
+]
+
+[[package]]
+name = "triton"
+version = "3.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "setuptools" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8d/a9/549e51e9b1b2c9b854fd761a1d23df0ba2fbc60bd0c13b489ffa518cfcb7/triton-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b74db445b1c562844d3cfad6e9679c72e93fdfb1a90a24052b03bb5c49d1242e", size = 155600257, upload-time = "2025-05-29T23:39:36.085Z" },
+    { url = "https://files.pythonhosted.org/packages/21/2f/3e56ea7b58f80ff68899b1dbe810ff257c9d177d288c6b0f55bf2fe4eb50/triton-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b", size = 155689937, upload-time = "2025-05-29T23:39:44.182Z" },
+    { url = "https://files.pythonhosted.org/packages/24/5f/950fb373bf9c01ad4eb5a8cd5eaf32cdf9e238c02f9293557a2129b9c4ac/triton-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9999e83aba21e1a78c1f36f21bce621b77bcaa530277a50484a7cb4a822f6e43", size = 155669138, upload-time = "2025-05-29T23:39:51.771Z" },
+    { url = "https://files.pythonhosted.org/packages/74/1f/dfb531f90a2d367d914adfee771babbd3f1a5b26c3f5fbc458dee21daa78/triton-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b89d846b5a4198317fec27a5d3a609ea96b6d557ff44b56c23176546023c4240", size = 155673035, upload-time = "2025-05-29T23:40:02.468Z" },
+    { url = "https://files.pythonhosted.org/packages/28/71/bd20ffcb7a64c753dc2463489a61bf69d531f308e390ad06390268c4ea04/triton-3.3.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3198adb9d78b77818a5388bff89fa72ff36f9da0bc689db2f0a651a67ce6a42", size = 155735832, upload-time = "2025-05-29T23:40:10.522Z" },
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.14.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" },
+]
+
+[[package]]
+name = "typing-inspection"
+version = "0.4.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f8/b1/0c11f5058406b3af7609f121aaa6b609744687f1d158b3c3a5bf4cc94238/typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28", size = 75726, upload-time = "2025-05-21T18:55:23.885Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" },
+]
+
+[[package]]
+name = "urllib3"
+version = "2.5.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },
+]
+
+[package.optional-dependencies]
+socks = [
+    { name = "pysocks" },
+]
+
+[[package]]
+name = "websocket-client"
+version = "1.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e6/30/fba0d96b4b5fbf5948ed3f4681f7da2f9f64512e1d303f94b4cc174c24a5/websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da", size = 54648, upload-time = "2024-04-23T22:16:16.976Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826, upload-time = "2024-04-23T22:16:14.422Z" },
+]
+
+[[package]]
+name = "wsproto"
+version = "1.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c9/4a/44d3c295350d776427904d73c189e10aeae66d7f555bb2feee16d1e4ba5a/wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", size = 53425, upload-time = "2022-08-23T19:58:21.447Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/78/58/e860788190eba3bcce367f74d29c4675466ce8dddfba85f7827588416f01/wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736", size = 24226, upload-time = "2022-08-23T19:58:19.96Z" },
+]
+
+[[package]]
+name = "xxhash"
+version = "3.5.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/00/5e/d6e5258d69df8b4ed8c83b6664f2b47d30d2dec551a29ad72a6c69eafd31/xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f", size = 84241, upload-time = "2024-08-17T09:20:38.972Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bb/8a/0e9feca390d512d293afd844d31670e25608c4a901e10202aa98785eab09/xxhash-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212", size = 31970, upload-time = "2024-08-17T09:17:35.675Z" },
+    { url = "https://files.pythonhosted.org/packages/16/e6/be5aa49580cd064a18200ab78e29b88b1127e1a8c7955eb8ecf81f2626eb/xxhash-3.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3171f693dbc2cef6477054a665dc255d996646b4023fe56cb4db80e26f4cc520", size = 30801, upload-time = "2024-08-17T09:17:37.353Z" },
+    { url = "https://files.pythonhosted.org/packages/20/ee/b8a99ebbc6d1113b3a3f09e747fa318c3cde5b04bd9c197688fadf0eeae8/xxhash-3.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c5d3e570ef46adaf93fc81b44aca6002b5a4d8ca11bd0580c07eac537f36680", size = 220927, upload-time = "2024-08-17T09:17:38.835Z" },
+    { url = "https://files.pythonhosted.org/packages/58/62/15d10582ef159283a5c2b47f6d799fc3303fe3911d5bb0bcc820e1ef7ff4/xxhash-3.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7cb29a034301e2982df8b1fe6328a84f4b676106a13e9135a0d7e0c3e9f806da", size = 200360, upload-time = "2024-08-17T09:17:40.851Z" },
+    { url = "https://files.pythonhosted.org/packages/23/41/61202663ea9b1bd8e53673b8ec9e2619989353dba8cfb68e59a9cbd9ffe3/xxhash-3.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0d307d27099bb0cbeea7260eb39ed4fdb99c5542e21e94bb6fd29e49c57a23", size = 428528, upload-time = "2024-08-17T09:17:42.545Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/07/d9a3059f702dec5b3b703737afb6dda32f304f6e9da181a229dafd052c29/xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0342aafd421795d740e514bc9858ebddfc705a75a8c5046ac56d85fe97bf196", size = 194149, upload-time = "2024-08-17T09:17:44.361Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/58/27caadf78226ecf1d62dbd0c01d152ed381c14c1ee4ad01f0d460fc40eac/xxhash-3.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3dbbd9892c5ebffeca1ed620cf0ade13eb55a0d8c84e0751a6653adc6ac40d0c", size = 207703, upload-time = "2024-08-17T09:17:46.656Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/08/32d558ce23e1e068453c39aed7b3c1cdc690c177873ec0ca3a90d5808765/xxhash-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4cc2d67fdb4d057730c75a64c5923abfa17775ae234a71b0200346bfb0a7f482", size = 216255, upload-time = "2024-08-17T09:17:48.031Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/d4/2b971e2d2b0a61045f842b622ef11e94096cf1f12cd448b6fd426e80e0e2/xxhash-3.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ec28adb204b759306a3d64358a5e5c07d7b1dd0ccbce04aa76cb9377b7b70296", size = 202744, upload-time = "2024-08-17T09:17:50.045Z" },
+    { url = "https://files.pythonhosted.org/packages/19/ae/6a6438864a8c4c39915d7b65effd85392ebe22710412902487e51769146d/xxhash-3.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1328f6d8cca2b86acb14104e381225a3d7b42c92c4b86ceae814e5c400dbb415", size = 210115, upload-time = "2024-08-17T09:17:51.834Z" },
+    { url = "https://files.pythonhosted.org/packages/48/7d/b3c27c27d1fc868094d02fe4498ccce8cec9fcc591825c01d6bcb0b4fc49/xxhash-3.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8d47ebd9f5d9607fd039c1fbf4994e3b071ea23eff42f4ecef246ab2b7334198", size = 414247, upload-time = "2024-08-17T09:17:53.094Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/05/918f9e7d2fbbd334b829997045d341d6239b563c44e683b9a7ef8fe50f5d/xxhash-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b96d559e0fcddd3343c510a0fe2b127fbff16bf346dd76280b82292567523442", size = 191419, upload-time = "2024-08-17T09:17:54.906Z" },
+    { url = "https://files.pythonhosted.org/packages/08/29/dfe393805b2f86bfc47c290b275f0b7c189dc2f4e136fd4754f32eb18a8d/xxhash-3.5.0-cp310-cp310-win32.whl", hash = "sha256:61c722ed8d49ac9bc26c7071eeaa1f6ff24053d553146d5df031802deffd03da", size = 30114, upload-time = "2024-08-17T09:17:56.566Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/d7/aa0b22c4ebb7c3ccb993d4c565132abc641cd11164f8952d89eb6a501909/xxhash-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:9bed5144c6923cc902cd14bb8963f2d5e034def4486ab0bbe1f58f03f042f9a9", size = 30003, upload-time = "2024-08-17T09:17:57.596Z" },
+    { url = "https://files.pythonhosted.org/packages/69/12/f969b81541ee91b55f1ce469d7ab55079593c80d04fd01691b550e535000/xxhash-3.5.0-cp310-cp310-win_arm64.whl", hash = "sha256:893074d651cf25c1cc14e3bea4fceefd67f2921b1bb8e40fcfeba56820de80c6", size = 26773, upload-time = "2024-08-17T09:17:59.169Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/c7/afed0f131fbda960ff15eee7f304fa0eeb2d58770fade99897984852ef23/xxhash-3.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02c2e816896dc6f85922ced60097bcf6f008dedfc5073dcba32f9c8dd786f3c1", size = 31969, upload-time = "2024-08-17T09:18:00.852Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/0c/7c3bc6d87e5235672fcc2fb42fd5ad79fe1033925f71bf549ee068c7d1ca/xxhash-3.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6027dcd885e21581e46d3c7f682cfb2b870942feeed58a21c29583512c3f09f8", size = 30800, upload-time = "2024-08-17T09:18:01.863Z" },
+    { url = "https://files.pythonhosted.org/packages/04/9e/01067981d98069eec1c20201f8c145367698e9056f8bc295346e4ea32dd1/xxhash-3.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1308fa542bbdbf2fa85e9e66b1077eea3a88bef38ee8a06270b4298a7a62a166", size = 221566, upload-time = "2024-08-17T09:18:03.461Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/09/d4996de4059c3ce5342b6e1e6a77c9d6c91acce31f6ed979891872dd162b/xxhash-3.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c28b2fdcee797e1c1961cd3bcd3d545cab22ad202c846235197935e1df2f8ef7", size = 201214, upload-time = "2024-08-17T09:18:05.616Z" },
+    { url = "https://files.pythonhosted.org/packages/62/f5/6d2dc9f8d55a7ce0f5e7bfef916e67536f01b85d32a9fbf137d4cadbee38/xxhash-3.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:924361811732ddad75ff23e90efd9ccfda4f664132feecb90895bade6a1b4623", size = 429433, upload-time = "2024-08-17T09:18:06.957Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/72/9256303f10e41ab004799a4aa74b80b3c5977d6383ae4550548b24bd1971/xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89997aa1c4b6a5b1e5b588979d1da048a3c6f15e55c11d117a56b75c84531f5a", size = 194822, upload-time = "2024-08-17T09:18:08.331Z" },
+    { url = "https://files.pythonhosted.org/packages/34/92/1a3a29acd08248a34b0e6a94f4e0ed9b8379a4ff471f1668e4dce7bdbaa8/xxhash-3.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:685c4f4e8c59837de103344eb1c8a3851f670309eb5c361f746805c5471b8c88", size = 208538, upload-time = "2024-08-17T09:18:10.332Z" },
+    { url = "https://files.pythonhosted.org/packages/53/ad/7fa1a109663366de42f724a1cdb8e796a260dbac45047bce153bc1e18abf/xxhash-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbd2ecfbfee70bc1a4acb7461fa6af7748ec2ab08ac0fa298f281c51518f982c", size = 216953, upload-time = "2024-08-17T09:18:11.707Z" },
+    { url = "https://files.pythonhosted.org/packages/35/02/137300e24203bf2b2a49b48ce898ecce6fd01789c0fcd9c686c0a002d129/xxhash-3.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25b5a51dc3dfb20a10833c8eee25903fd2e14059e9afcd329c9da20609a307b2", size = 203594, upload-time = "2024-08-17T09:18:13.799Z" },
+    { url = "https://files.pythonhosted.org/packages/23/03/aeceb273933d7eee248c4322b98b8e971f06cc3880e5f7602c94e5578af5/xxhash-3.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a8fb786fb754ef6ff8c120cb96629fb518f8eb5a61a16aac3a979a9dbd40a084", size = 210971, upload-time = "2024-08-17T09:18:15.824Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/64/ed82ec09489474cbb35c716b189ddc1521d8b3de12b1b5ab41ce7f70253c/xxhash-3.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a905ad00ad1e1c34fe4e9d7c1d949ab09c6fa90c919860c1534ff479f40fd12d", size = 415050, upload-time = "2024-08-17T09:18:17.142Z" },
+    { url = "https://files.pythonhosted.org/packages/71/43/6db4c02dcb488ad4e03bc86d70506c3d40a384ee73c9b5c93338eb1f3c23/xxhash-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:963be41bcd49f53af6d795f65c0da9b4cc518c0dd9c47145c98f61cb464f4839", size = 192216, upload-time = "2024-08-17T09:18:18.779Z" },
+    { url = "https://files.pythonhosted.org/packages/22/6d/db4abec29e7a567455344433d095fdb39c97db6955bb4a2c432e486b4d28/xxhash-3.5.0-cp311-cp311-win32.whl", hash = "sha256:109b436096d0a2dd039c355fa3414160ec4d843dfecc64a14077332a00aeb7da", size = 30120, upload-time = "2024-08-17T09:18:20.009Z" },
+    { url = "https://files.pythonhosted.org/packages/52/1c/fa3b61c0cf03e1da4767213672efe186b1dfa4fc901a4a694fb184a513d1/xxhash-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:b702f806693201ad6c0a05ddbbe4c8f359626d0b3305f766077d51388a6bac58", size = 30003, upload-time = "2024-08-17T09:18:21.052Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/8e/9e6fc572acf6e1cc7ccb01973c213f895cb8668a9d4c2b58a99350da14b7/xxhash-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:c4dcb4120d0cc3cc448624147dba64e9021b278c63e34a38789b688fd0da9bf3", size = 26777, upload-time = "2024-08-17T09:18:22.809Z" },
+    { url = "https://files.pythonhosted.org/packages/07/0e/1bfce2502c57d7e2e787600b31c83535af83746885aa1a5f153d8c8059d6/xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00", size = 31969, upload-time = "2024-08-17T09:18:24.025Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/d6/8ca450d6fe5b71ce521b4e5db69622383d039e2b253e9b2f24f93265b52c/xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9", size = 30787, upload-time = "2024-08-17T09:18:25.318Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/84/de7c89bc6ef63d750159086a6ada6416cc4349eab23f76ab870407178b93/xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84", size = 220959, upload-time = "2024-08-17T09:18:26.518Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/86/51258d3e8a8545ff26468c977101964c14d56a8a37f5835bc0082426c672/xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793", size = 200006, upload-time = "2024-08-17T09:18:27.905Z" },
+    { url = "https://files.pythonhosted.org/packages/02/0a/96973bd325412feccf23cf3680fd2246aebf4b789122f938d5557c54a6b2/xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be", size = 428326, upload-time = "2024-08-17T09:18:29.335Z" },
+    { url = "https://files.pythonhosted.org/packages/11/a7/81dba5010f7e733de88af9555725146fc133be97ce36533867f4c7e75066/xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6", size = 194380, upload-time = "2024-08-17T09:18:30.706Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/7d/f29006ab398a173f4501c0e4977ba288f1c621d878ec217b4ff516810c04/xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90", size = 207934, upload-time = "2024-08-17T09:18:32.133Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/6e/6e88b8f24612510e73d4d70d9b0c7dff62a2e78451b9f0d042a5462c8d03/xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27", size = 216301, upload-time = "2024-08-17T09:18:33.474Z" },
+    { url = "https://files.pythonhosted.org/packages/af/51/7862f4fa4b75a25c3b4163c8a873f070532fe5f2d3f9b3fc869c8337a398/xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2", size = 203351, upload-time = "2024-08-17T09:18:34.889Z" },
+    { url = "https://files.pythonhosted.org/packages/22/61/8d6a40f288f791cf79ed5bb113159abf0c81d6efb86e734334f698eb4c59/xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d", size = 210294, upload-time = "2024-08-17T09:18:36.355Z" },
+    { url = "https://files.pythonhosted.org/packages/17/02/215c4698955762d45a8158117190261b2dbefe9ae7e5b906768c09d8bc74/xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab", size = 414674, upload-time = "2024-08-17T09:18:38.536Z" },
+    { url = "https://files.pythonhosted.org/packages/31/5c/b7a8db8a3237cff3d535261325d95de509f6a8ae439a5a7a4ffcff478189/xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e", size = 192022, upload-time = "2024-08-17T09:18:40.138Z" },
+    { url = "https://files.pythonhosted.org/packages/78/e3/dd76659b2811b3fd06892a8beb850e1996b63e9235af5a86ea348f053e9e/xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8", size = 30170, upload-time = "2024-08-17T09:18:42.163Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/6b/1c443fe6cfeb4ad1dcf231cdec96eb94fb43d6498b4469ed8b51f8b59a37/xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e", size = 30040, upload-time = "2024-08-17T09:18:43.699Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/eb/04405305f290173acc0350eba6d2f1a794b57925df0398861a20fbafa415/xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2", size = 26796, upload-time = "2024-08-17T09:18:45.29Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/b8/e4b3ad92d249be5c83fa72916c9091b0965cb0faeff05d9a0a3870ae6bff/xxhash-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:37889a0d13b0b7d739cfc128b1c902f04e32de17b33d74b637ad42f1c55101f6", size = 31795, upload-time = "2024-08-17T09:18:46.813Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/d8/b3627a0aebfbfa4c12a41e22af3742cf08c8ea84f5cc3367b5de2d039cce/xxhash-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97a662338797c660178e682f3bc180277b9569a59abfb5925e8620fba00b9fc5", size = 30792, upload-time = "2024-08-17T09:18:47.862Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/cc/762312960691da989c7cd0545cb120ba2a4148741c6ba458aa723c00a3f8/xxhash-3.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f85e0108d51092bdda90672476c7d909c04ada6923c14ff9d913c4f7dc8a3bc", size = 220950, upload-time = "2024-08-17T09:18:49.06Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/e9/cc266f1042c3c13750e86a535496b58beb12bf8c50a915c336136f6168dc/xxhash-3.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2fd827b0ba763ac919440042302315c564fdb797294d86e8cdd4578e3bc7f3", size = 199980, upload-time = "2024-08-17T09:18:50.445Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/85/a836cd0dc5cc20376de26b346858d0ac9656f8f730998ca4324921a010b9/xxhash-3.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82085c2abec437abebf457c1d12fccb30cc8b3774a0814872511f0f0562c768c", size = 428324, upload-time = "2024-08-17T09:18:51.988Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/0e/15c243775342ce840b9ba34aceace06a1148fa1630cd8ca269e3223987f5/xxhash-3.5.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07fda5de378626e502b42b311b049848c2ef38784d0d67b6f30bb5008642f8eb", size = 194370, upload-time = "2024-08-17T09:18:54.164Z" },
+    { url = "https://files.pythonhosted.org/packages/87/a1/b028bb02636dfdc190da01951d0703b3d904301ed0ef6094d948983bef0e/xxhash-3.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c279f0d2b34ef15f922b77966640ade58b4ccdfef1c4d94b20f2a364617a493f", size = 207911, upload-time = "2024-08-17T09:18:55.509Z" },
+    { url = "https://files.pythonhosted.org/packages/80/d5/73c73b03fc0ac73dacf069fdf6036c9abad82de0a47549e9912c955ab449/xxhash-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:89e66ceed67b213dec5a773e2f7a9e8c58f64daeb38c7859d8815d2c89f39ad7", size = 216352, upload-time = "2024-08-17T09:18:57.073Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/2a/5043dba5ddbe35b4fe6ea0a111280ad9c3d4ba477dd0f2d1fe1129bda9d0/xxhash-3.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bcd51708a633410737111e998ceb3b45d3dbc98c0931f743d9bb0a209033a326", size = 203410, upload-time = "2024-08-17T09:18:58.54Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/b2/9a8ded888b7b190aed75b484eb5c853ddd48aa2896e7b59bbfbce442f0a1/xxhash-3.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ff2c0a34eae7df88c868be53a8dd56fbdf592109e21d4bfa092a27b0bf4a7bf", size = 210322, upload-time = "2024-08-17T09:18:59.943Z" },
+    { url = "https://files.pythonhosted.org/packages/98/62/440083fafbc917bf3e4b67c2ade621920dd905517e85631c10aac955c1d2/xxhash-3.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4e28503dccc7d32e0b9817aa0cbfc1f45f563b2c995b7a66c4c8a0d232e840c7", size = 414725, upload-time = "2024-08-17T09:19:01.332Z" },
+    { url = "https://files.pythonhosted.org/packages/75/db/009206f7076ad60a517e016bb0058381d96a007ce3f79fa91d3010f49cc2/xxhash-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a6c50017518329ed65a9e4829154626f008916d36295b6a3ba336e2458824c8c", size = 192070, upload-time = "2024-08-17T09:19:03.007Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/6d/c61e0668943a034abc3a569cdc5aeae37d686d9da7e39cf2ed621d533e36/xxhash-3.5.0-cp313-cp313-win32.whl", hash = "sha256:53a068fe70301ec30d868ece566ac90d873e3bb059cf83c32e76012c889b8637", size = 30172, upload-time = "2024-08-17T09:19:04.355Z" },
+    { url = "https://files.pythonhosted.org/packages/96/14/8416dce965f35e3d24722cdf79361ae154fa23e2ab730e5323aa98d7919e/xxhash-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:80babcc30e7a1a484eab952d76a4f4673ff601f54d5142c26826502740e70b43", size = 30041, upload-time = "2024-08-17T09:19:05.435Z" },
+    { url = "https://files.pythonhosted.org/packages/27/ee/518b72faa2073f5aa8e3262408d284892cb79cf2754ba0c3a5870645ef73/xxhash-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:4811336f1ce11cac89dcbd18f3a25c527c16311709a89313c3acaf771def2d4b", size = 26801, upload-time = "2024-08-17T09:19:06.547Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/9a/233606bada5bd6f50b2b72c45de3d9868ad551e83893d2ac86dc7bb8553a/xxhash-3.5.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:2014c5b3ff15e64feecb6b713af12093f75b7926049e26a580e94dcad3c73d8c", size = 29732, upload-time = "2024-08-17T09:20:11.175Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/67/f75276ca39e2c6604e3bee6c84e9db8a56a4973fde9bf35989787cf6e8aa/xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fab81ef75003eda96239a23eda4e4543cedc22e34c373edcaf744e721a163986", size = 36214, upload-time = "2024-08-17T09:20:12.335Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/f8/f6c61fd794229cc3848d144f73754a0c107854372d7261419dcbbd286299/xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e2febf914ace002132aa09169cc572e0d8959d0f305f93d5828c4836f9bc5a6", size = 32020, upload-time = "2024-08-17T09:20:13.537Z" },
+    { url = "https://files.pythonhosted.org/packages/79/d3/c029c99801526f859e6b38d34ab87c08993bf3dcea34b11275775001638a/xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d3a10609c51da2a1c0ea0293fc3968ca0a18bd73838455b5bca3069d7f8e32b", size = 40515, upload-time = "2024-08-17T09:20:14.669Z" },
+    { url = "https://files.pythonhosted.org/packages/62/e3/bef7b82c1997579c94de9ac5ea7626d01ae5858aa22bf4fcb38bf220cb3e/xxhash-3.5.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da", size = 30064, upload-time = "2024-08-17T09:20:15.925Z" },
+]
+
+[[package]]
+name = "yarl"
+version = "1.20.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "idna" },
+    { name = "multidict" },
+    { name = "propcache" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/65/7fed0d774abf47487c64be14e9223749468922817b5e8792b8a64792a1bb/yarl-1.20.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6032e6da6abd41e4acda34d75a816012717000fa6839f37124a47fcefc49bec4", size = 132910, upload-time = "2025-06-10T00:42:31.108Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/7b/988f55a52da99df9e56dc733b8e4e5a6ae2090081dc2754fc8fd34e60aa0/yarl-1.20.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c7b34d804b8cf9b214f05015c4fee2ebe7ed05cf581e7192c06555c71f4446a", size = 90644, upload-time = "2025-06-10T00:42:33.851Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/de/30d98f03e95d30c7e3cc093759982d038c8833ec2451001d45ef4854edc1/yarl-1.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c869f2651cc77465f6cd01d938d91a11d9ea5d798738c1dc077f3de0b5e5fed", size = 89322, upload-time = "2025-06-10T00:42:35.688Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/7a/f2f314f5ebfe9200724b0b748de2186b927acb334cf964fd312eb86fc286/yarl-1.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62915e6688eb4d180d93840cda4110995ad50c459bf931b8b3775b37c264af1e", size = 323786, upload-time = "2025-06-10T00:42:37.817Z" },
+    { url = "https://files.pythonhosted.org/packages/15/3f/718d26f189db96d993d14b984ce91de52e76309d0fd1d4296f34039856aa/yarl-1.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41ebd28167bc6af8abb97fec1a399f412eec5fd61a3ccbe2305a18b84fb4ca73", size = 319627, upload-time = "2025-06-10T00:42:39.937Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/76/8fcfbf5fa2369157b9898962a4a7d96764b287b085b5b3d9ffae69cdefd1/yarl-1.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21242b4288a6d56f04ea193adde174b7e347ac46ce6bc84989ff7c1b1ecea84e", size = 339149, upload-time = "2025-06-10T00:42:42.627Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/95/d7fc301cc4661785967acc04f54a4a42d5124905e27db27bb578aac49b5c/yarl-1.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bea21cdae6c7eb02ba02a475f37463abfe0a01f5d7200121b03e605d6a0439f8", size = 333327, upload-time = "2025-06-10T00:42:44.842Z" },
+    { url = "https://files.pythonhosted.org/packages/65/94/e21269718349582eee81efc5c1c08ee71c816bfc1585b77d0ec3f58089eb/yarl-1.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f8a891e4a22a89f5dde7862994485e19db246b70bb288d3ce73a34422e55b23", size = 326054, upload-time = "2025-06-10T00:42:47.149Z" },
+    { url = "https://files.pythonhosted.org/packages/32/ae/8616d1f07853704523519f6131d21f092e567c5af93de7e3e94b38d7f065/yarl-1.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd803820d44c8853a109a34e3660e5a61beae12970da479cf44aa2954019bf70", size = 315035, upload-time = "2025-06-10T00:42:48.852Z" },
+    { url = "https://files.pythonhosted.org/packages/48/aa/0ace06280861ef055855333707db5e49c6e3a08840a7ce62682259d0a6c0/yarl-1.20.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b982fa7f74c80d5c0c7b5b38f908971e513380a10fecea528091405f519b9ebb", size = 338962, upload-time = "2025-06-10T00:42:51.024Z" },
+    { url = "https://files.pythonhosted.org/packages/20/52/1e9d0e6916f45a8fb50e6844f01cb34692455f1acd548606cbda8134cd1e/yarl-1.20.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:33f29ecfe0330c570d997bcf1afd304377f2e48f61447f37e846a6058a4d33b2", size = 335399, upload-time = "2025-06-10T00:42:53.007Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/65/60452df742952c630e82f394cd409de10610481d9043aa14c61bf846b7b1/yarl-1.20.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:835ab2cfc74d5eb4a6a528c57f05688099da41cf4957cf08cad38647e4a83b30", size = 338649, upload-time = "2025-06-10T00:42:54.964Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/f5/6cd4ff38dcde57a70f23719a838665ee17079640c77087404c3d34da6727/yarl-1.20.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:46b5e0ccf1943a9a6e766b2c2b8c732c55b34e28be57d8daa2b3c1d1d4009309", size = 358563, upload-time = "2025-06-10T00:42:57.28Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/90/c42eefd79d0d8222cb3227bdd51b640c0c1d0aa33fe4cc86c36eccba77d3/yarl-1.20.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:df47c55f7d74127d1b11251fe6397d84afdde0d53b90bedb46a23c0e534f9d24", size = 357609, upload-time = "2025-06-10T00:42:59.055Z" },
+    { url = "https://files.pythonhosted.org/packages/03/c8/cea6b232cb4617514232e0f8a718153a95b5d82b5290711b201545825532/yarl-1.20.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76d12524d05841276b0e22573f28d5fbcb67589836772ae9244d90dd7d66aa13", size = 350224, upload-time = "2025-06-10T00:43:01.248Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/a3/eaa0ab9712f1f3d01faf43cf6f1f7210ce4ea4a7e9b28b489a2261ca8db9/yarl-1.20.1-cp310-cp310-win32.whl", hash = "sha256:6c4fbf6b02d70e512d7ade4b1f998f237137f1417ab07ec06358ea04f69134f8", size = 81753, upload-time = "2025-06-10T00:43:03.486Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/34/e4abde70a9256465fe31c88ed02c3f8502b7b5dead693a4f350a06413f28/yarl-1.20.1-cp310-cp310-win_amd64.whl", hash = "sha256:aef6c4d69554d44b7f9d923245f8ad9a707d971e6209d51279196d8e8fe1ae16", size = 86817, upload-time = "2025-06-10T00:43:05.231Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/18/893b50efc2350e47a874c5c2d67e55a0ea5df91186b2a6f5ac52eff887cd/yarl-1.20.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:47ee6188fea634bdfaeb2cc420f5b3b17332e6225ce88149a17c413c77ff269e", size = 133833, upload-time = "2025-06-10T00:43:07.393Z" },
+    { url = "https://files.pythonhosted.org/packages/89/ed/b8773448030e6fc47fa797f099ab9eab151a43a25717f9ac043844ad5ea3/yarl-1.20.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0f6500f69e8402d513e5eedb77a4e1818691e8f45e6b687147963514d84b44b", size = 91070, upload-time = "2025-06-10T00:43:09.538Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/e3/409bd17b1e42619bf69f60e4f031ce1ccb29bd7380117a55529e76933464/yarl-1.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a8900a42fcdaad568de58887c7b2f602962356908eedb7628eaf6021a6e435b", size = 89818, upload-time = "2025-06-10T00:43:11.575Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/77/64d8431a4d77c856eb2d82aa3de2ad6741365245a29b3a9543cd598ed8c5/yarl-1.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bad6d131fda8ef508b36be3ece16d0902e80b88ea7200f030a0f6c11d9e508d4", size = 347003, upload-time = "2025-06-10T00:43:14.088Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/d2/0c7e4def093dcef0bd9fa22d4d24b023788b0a33b8d0088b51aa51e21e99/yarl-1.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:df018d92fe22aaebb679a7f89fe0c0f368ec497e3dda6cb81a567610f04501f1", size = 336537, upload-time = "2025-06-10T00:43:16.431Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/f3/fc514f4b2cf02cb59d10cbfe228691d25929ce8f72a38db07d3febc3f706/yarl-1.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f969afbb0a9b63c18d0feecf0db09d164b7a44a053e78a7d05f5df163e43833", size = 362358, upload-time = "2025-06-10T00:43:18.704Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/6d/a313ac8d8391381ff9006ac05f1d4331cee3b1efaa833a53d12253733255/yarl-1.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:812303eb4aa98e302886ccda58d6b099e3576b1b9276161469c25803a8db277d", size = 357362, upload-time = "2025-06-10T00:43:20.888Z" },
+    { url = "https://files.pythonhosted.org/packages/00/70/8f78a95d6935a70263d46caa3dd18e1f223cf2f2ff2037baa01a22bc5b22/yarl-1.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98c4a7d166635147924aa0bf9bfe8d8abad6fffa6102de9c99ea04a1376f91e8", size = 348979, upload-time = "2025-06-10T00:43:23.169Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/05/42773027968968f4f15143553970ee36ead27038d627f457cc44bbbeecf3/yarl-1.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12e768f966538e81e6e7550f9086a6236b16e26cd964cf4df35349970f3551cf", size = 337274, upload-time = "2025-06-10T00:43:27.111Z" },
+    { url = "https://files.pythonhosted.org/packages/05/be/665634aa196954156741ea591d2f946f1b78ceee8bb8f28488bf28c0dd62/yarl-1.20.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe41919b9d899661c5c28a8b4b0acf704510b88f27f0934ac7a7bebdd8938d5e", size = 363294, upload-time = "2025-06-10T00:43:28.96Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/90/73448401d36fa4e210ece5579895731f190d5119c4b66b43b52182e88cd5/yarl-1.20.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8601bc010d1d7780592f3fc1bdc6c72e2b6466ea34569778422943e1a1f3c389", size = 358169, upload-time = "2025-06-10T00:43:30.701Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/b0/fce922d46dc1eb43c811f1889f7daa6001b27a4005587e94878570300881/yarl-1.20.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:daadbdc1f2a9033a2399c42646fbd46da7992e868a5fe9513860122d7fe7a73f", size = 362776, upload-time = "2025-06-10T00:43:32.51Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/0d/b172628fce039dae8977fd22caeff3eeebffd52e86060413f5673767c427/yarl-1.20.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:03aa1e041727cb438ca762628109ef1333498b122e4c76dd858d186a37cec845", size = 381341, upload-time = "2025-06-10T00:43:34.543Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/9b/5b886d7671f4580209e855974fe1cecec409aa4a89ea58b8f0560dc529b1/yarl-1.20.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:642980ef5e0fa1de5fa96d905c7e00cb2c47cb468bfcac5a18c58e27dbf8d8d1", size = 379988, upload-time = "2025-06-10T00:43:36.489Z" },
+    { url = "https://files.pythonhosted.org/packages/73/be/75ef5fd0fcd8f083a5d13f78fd3f009528132a1f2a1d7c925c39fa20aa79/yarl-1.20.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:86971e2795584fe8c002356d3b97ef6c61862720eeff03db2a7c86b678d85b3e", size = 371113, upload-time = "2025-06-10T00:43:38.592Z" },
+    { url = "https://files.pythonhosted.org/packages/50/4f/62faab3b479dfdcb741fe9e3f0323e2a7d5cd1ab2edc73221d57ad4834b2/yarl-1.20.1-cp311-cp311-win32.whl", hash = "sha256:597f40615b8d25812f14562699e287f0dcc035d25eb74da72cae043bb884d773", size = 81485, upload-time = "2025-06-10T00:43:41.038Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/09/d9c7942f8f05c32ec72cd5c8e041c8b29b5807328b68b4801ff2511d4d5e/yarl-1.20.1-cp311-cp311-win_amd64.whl", hash = "sha256:26ef53a9e726e61e9cd1cda6b478f17e350fb5800b4bd1cd9fe81c4d91cfeb2e", size = 86686, upload-time = "2025-06-10T00:43:42.692Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667, upload-time = "2025-06-10T00:43:44.369Z" },
+    { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025, upload-time = "2025-06-10T00:43:46.295Z" },
+    { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709, upload-time = "2025-06-10T00:43:48.22Z" },
+    { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287, upload-time = "2025-06-10T00:43:49.924Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429, upload-time = "2025-06-10T00:43:51.7Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429, upload-time = "2025-06-10T00:43:53.494Z" },
+    { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862, upload-time = "2025-06-10T00:43:55.766Z" },
+    { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616, upload-time = "2025-06-10T00:43:58.056Z" },
+    { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954, upload-time = "2025-06-10T00:43:59.773Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575, upload-time = "2025-06-10T00:44:02.051Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061, upload-time = "2025-06-10T00:44:04.196Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142, upload-time = "2025-06-10T00:44:06.527Z" },
+    { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894, upload-time = "2025-06-10T00:44:08.379Z" },
+    { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378, upload-time = "2025-06-10T00:44:10.51Z" },
+    { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069, upload-time = "2025-06-10T00:44:12.834Z" },
+    { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249, upload-time = "2025-06-10T00:44:14.731Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710, upload-time = "2025-06-10T00:44:16.716Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/e1/2411b6d7f769a07687acee88a062af5833cf1966b7266f3d8dfb3d3dc7d3/yarl-1.20.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:0b5ff0fbb7c9f1b1b5ab53330acbfc5247893069e7716840c8e7d5bb7355038a", size = 131811, upload-time = "2025-06-10T00:44:18.933Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/27/584394e1cb76fb771371770eccad35de400e7b434ce3142c2dd27392c968/yarl-1.20.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:14f326acd845c2b2e2eb38fb1346c94f7f3b01a4f5c788f8144f9b630bfff9a3", size = 90078, upload-time = "2025-06-10T00:44:20.635Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/9a/3246ae92d4049099f52d9b0fe3486e3b500e29b7ea872d0f152966fc209d/yarl-1.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f60e4ad5db23f0b96e49c018596707c3ae89f5d0bd97f0ad3684bcbad899f1e7", size = 88748, upload-time = "2025-06-10T00:44:22.34Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/25/35afe384e31115a1a801fbcf84012d7a066d89035befae7c5d4284df1e03/yarl-1.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49bdd1b8e00ce57e68ba51916e4bb04461746e794e7c4d4bbc42ba2f18297691", size = 349595, upload-time = "2025-06-10T00:44:24.314Z" },
+    { url = "https://files.pythonhosted.org/packages/28/2d/8aca6cb2cabc8f12efcb82749b9cefecbccfc7b0384e56cd71058ccee433/yarl-1.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:66252d780b45189975abfed839616e8fd2dbacbdc262105ad7742c6ae58f3e31", size = 342616, upload-time = "2025-06-10T00:44:26.167Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/e9/1312633d16b31acf0098d30440ca855e3492d66623dafb8e25b03d00c3da/yarl-1.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59174e7332f5d153d8f7452a102b103e2e74035ad085f404df2e40e663a22b28", size = 361324, upload-time = "2025-06-10T00:44:27.915Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/a0/688cc99463f12f7669eec7c8acc71ef56a1521b99eab7cd3abb75af887b0/yarl-1.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3968ec7d92a0c0f9ac34d5ecfd03869ec0cab0697c91a45db3fbbd95fe1b653", size = 359676, upload-time = "2025-06-10T00:44:30.041Z" },
+    { url = "https://files.pythonhosted.org/packages/af/44/46407d7f7a56e9a85a4c207724c9f2c545c060380718eea9088f222ba697/yarl-1.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a4fbb50e14396ba3d375f68bfe02215d8e7bc3ec49da8341fe3157f59d2ff5", size = 352614, upload-time = "2025-06-10T00:44:32.171Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/91/31163295e82b8d5485d31d9cf7754d973d41915cadce070491778d9c9825/yarl-1.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11a62c839c3a8eac2410e951301309426f368388ff2f33799052787035793b02", size = 336766, upload-time = "2025-06-10T00:44:34.494Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/8e/c41a5bc482121f51c083c4c2bcd16b9e01e1cf8729e380273a952513a21f/yarl-1.20.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:041eaa14f73ff5a8986b4388ac6bb43a77f2ea09bf1913df7a35d4646db69e53", size = 364615, upload-time = "2025-06-10T00:44:36.856Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/5b/61a3b054238d33d70ea06ebba7e58597891b71c699e247df35cc984ab393/yarl-1.20.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:377fae2fef158e8fd9d60b4c8751387b8d1fb121d3d0b8e9b0be07d1b41e83dc", size = 360982, upload-time = "2025-06-10T00:44:39.141Z" },
+    { url = "https://files.pythonhosted.org/packages/df/a3/6a72fb83f8d478cb201d14927bc8040af901811a88e0ff2da7842dd0ed19/yarl-1.20.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1c92f4390e407513f619d49319023664643d3339bd5e5a56a3bebe01bc67ec04", size = 369792, upload-time = "2025-06-10T00:44:40.934Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/af/4cc3c36dfc7c077f8dedb561eb21f69e1e9f2456b91b593882b0b18c19dc/yarl-1.20.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d25ddcf954df1754ab0f86bb696af765c5bfaba39b74095f27eececa049ef9a4", size = 382049, upload-time = "2025-06-10T00:44:42.854Z" },
+    { url = "https://files.pythonhosted.org/packages/19/3a/e54e2c4752160115183a66dc9ee75a153f81f3ab2ba4bf79c3c53b33de34/yarl-1.20.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:909313577e9619dcff8c31a0ea2aa0a2a828341d92673015456b3ae492e7317b", size = 384774, upload-time = "2025-06-10T00:44:45.275Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/20/200ae86dabfca89060ec6447649f219b4cbd94531e425e50d57e5f5ac330/yarl-1.20.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:793fd0580cb9664548c6b83c63b43c477212c0260891ddf86809e1c06c8b08f1", size = 374252, upload-time = "2025-06-10T00:44:47.31Z" },
+    { url = "https://files.pythonhosted.org/packages/83/75/11ee332f2f516b3d094e89448da73d557687f7d137d5a0f48c40ff211487/yarl-1.20.1-cp313-cp313-win32.whl", hash = "sha256:468f6e40285de5a5b3c44981ca3a319a4b208ccc07d526b20b12aeedcfa654b7", size = 81198, upload-time = "2025-06-10T00:44:49.164Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/ba/39b1ecbf51620b40ab402b0fc817f0ff750f6d92712b44689c2c215be89d/yarl-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:495b4ef2fea40596bfc0affe3837411d6aa3371abcf31aac0ccc4bdd64d4ef5c", size = 86346, upload-time = "2025-06-10T00:44:51.182Z" },
+    { url = "https://files.pythonhosted.org/packages/43/c7/669c52519dca4c95153c8ad96dd123c79f354a376346b198f438e56ffeb4/yarl-1.20.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f60233b98423aab21d249a30eb27c389c14929f47be8430efa7dbd91493a729d", size = 138826, upload-time = "2025-06-10T00:44:52.883Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/42/fc0053719b44f6ad04a75d7f05e0e9674d45ef62f2d9ad2c1163e5c05827/yarl-1.20.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6f3eff4cc3f03d650d8755c6eefc844edde99d641d0dcf4da3ab27141a5f8ddf", size = 93217, upload-time = "2025-06-10T00:44:54.658Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/7f/fa59c4c27e2a076bba0d959386e26eba77eb52ea4a0aac48e3515c186b4c/yarl-1.20.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:69ff8439d8ba832d6bed88af2c2b3445977eba9a4588b787b32945871c2444e3", size = 92700, upload-time = "2025-06-10T00:44:56.784Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/d4/062b2f48e7c93481e88eff97a6312dca15ea200e959f23e96d8ab898c5b8/yarl-1.20.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf34efa60eb81dd2645a2e13e00bb98b76c35ab5061a3989c7a70f78c85006d", size = 347644, upload-time = "2025-06-10T00:44:59.071Z" },
+    { url = "https://files.pythonhosted.org/packages/89/47/78b7f40d13c8f62b499cc702fdf69e090455518ae544c00a3bf4afc9fc77/yarl-1.20.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8e0fe9364ad0fddab2688ce72cb7a8e61ea42eff3c7caeeb83874a5d479c896c", size = 323452, upload-time = "2025-06-10T00:45:01.605Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/2b/490d3b2dc66f52987d4ee0d3090a147ea67732ce6b4d61e362c1846d0d32/yarl-1.20.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f64fbf81878ba914562c672024089e3401974a39767747691c65080a67b18c1", size = 346378, upload-time = "2025-06-10T00:45:03.946Z" },
+    { url = "https://files.pythonhosted.org/packages/66/ad/775da9c8a94ce925d1537f939a4f17d782efef1f973039d821cbe4bcc211/yarl-1.20.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6342d643bf9a1de97e512e45e4b9560a043347e779a173250824f8b254bd5ce", size = 353261, upload-time = "2025-06-10T00:45:05.992Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/23/0ed0922b47a4f5c6eb9065d5ff1e459747226ddce5c6a4c111e728c9f701/yarl-1.20.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56dac5f452ed25eef0f6e3c6a066c6ab68971d96a9fb441791cad0efba6140d3", size = 335987, upload-time = "2025-06-10T00:45:08.227Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/49/bc728a7fe7d0e9336e2b78f0958a2d6b288ba89f25a1762407a222bf53c3/yarl-1.20.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7d7f497126d65e2cad8dc5f97d34c27b19199b6414a40cb36b52f41b79014be", size = 329361, upload-time = "2025-06-10T00:45:10.11Z" },
+    { url = "https://files.pythonhosted.org/packages/93/8f/b811b9d1f617c83c907e7082a76e2b92b655400e61730cd61a1f67178393/yarl-1.20.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:67e708dfb8e78d8a19169818eeb5c7a80717562de9051bf2413aca8e3696bf16", size = 346460, upload-time = "2025-06-10T00:45:12.055Z" },
+    { url = "https://files.pythonhosted.org/packages/70/fd/af94f04f275f95da2c3b8b5e1d49e3e79f1ed8b6ceb0f1664cbd902773ff/yarl-1.20.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:595c07bc79af2494365cc96ddeb772f76272364ef7c80fb892ef9d0649586513", size = 334486, upload-time = "2025-06-10T00:45:13.995Z" },
+    { url = "https://files.pythonhosted.org/packages/84/65/04c62e82704e7dd0a9b3f61dbaa8447f8507655fd16c51da0637b39b2910/yarl-1.20.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7bdd2f80f4a7df852ab9ab49484a4dee8030023aa536df41f2d922fd57bf023f", size = 342219, upload-time = "2025-06-10T00:45:16.479Z" },
+    { url = "https://files.pythonhosted.org/packages/91/95/459ca62eb958381b342d94ab9a4b6aec1ddec1f7057c487e926f03c06d30/yarl-1.20.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c03bfebc4ae8d862f853a9757199677ab74ec25424d0ebd68a0027e9c639a390", size = 350693, upload-time = "2025-06-10T00:45:18.399Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/00/d393e82dd955ad20617abc546a8f1aee40534d599ff555ea053d0ec9bf03/yarl-1.20.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:344d1103e9c1523f32a5ed704d576172d2cabed3122ea90b1d4e11fe17c66458", size = 355803, upload-time = "2025-06-10T00:45:20.677Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/ed/c5fb04869b99b717985e244fd93029c7a8e8febdfcffa06093e32d7d44e7/yarl-1.20.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:88cab98aa4e13e1ade8c141daeedd300a4603b7132819c484841bb7af3edce9e", size = 341709, upload-time = "2025-06-10T00:45:23.221Z" },
+    { url = "https://files.pythonhosted.org/packages/24/fd/725b8e73ac2a50e78a4534ac43c6addf5c1c2d65380dd48a9169cc6739a9/yarl-1.20.1-cp313-cp313t-win32.whl", hash = "sha256:b121ff6a7cbd4abc28985b6028235491941b9fe8fe226e6fdc539c977ea1739d", size = 86591, upload-time = "2025-06-10T00:45:25.793Z" },
+    { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003, upload-time = "2025-06-10T00:45:27.752Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" },
+]
+
+[[package]]
+name = "zipp"
+version = "3.23.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" },
+]